Merge pull request #3078 from akhilmhdh/feat/batch-upsert

Batch upsert operation

backend/e2e-test/routes/v3/secrets-v2.spec.ts

@@ -535,6 +535,107 @@ describe.each([{ auth: AuthMode.JWT }, { auth: AuthMode.IDENTITY_ACCESS_TOKEN }]
       );
     });
 
+    test.each(secretTestCases)("Bulk upsert secrets in path $path", async ({ secret, path }) => {
+      const updateSharedSecRes = await testServer.inject({
+        method: "PATCH",
+        url: `/api/v3/secrets/batch/raw`,
+        headers: {
+          authorization: `Bearer ${authToken}`
+        },
+        body: {
+          workspaceId: seedData1.projectV3.id,
+          environment: seedData1.environment.slug,
+          secretPath: path,
+          mode: "upsert",
+          secrets: Array.from(Array(5)).map((_e, i) => ({
+            secretKey: `BULK-${secret.key}-${i + 1}`,
+            secretValue: "update-value",
+            secretComment: secret.comment
+          }))
+        }
+      });
+      expect(updateSharedSecRes.statusCode).toBe(200);
+      const updateSharedSecPayload = JSON.parse(updateSharedSecRes.payload);
+      expect(updateSharedSecPayload).toHaveProperty("secrets");
+
+      // bulk ones should exist
+      const secrets = await getSecrets(seedData1.environment.slug, path);
+      expect(secrets).toEqual(
+        expect.arrayContaining(
+          Array.from(Array(5)).map((_e, i) =>
+            expect.objectContaining({
+              secretKey: `BULK-${secret.key}-${i + 1}`,
+              secretValue: "update-value",
+              type: SecretType.Shared
+            })
+          )
+        )
+      );
+      await Promise.all(
+        Array.from(Array(5)).map((_e, i) => deleteSecret({ path, key: `BULK-${secret.key}-${i + 1}` }))
+      );
+    });
+
+    test("Bulk upsert secrets in path multiple paths", async () => {
+      const firstBatchSecrets = Array.from(Array(5)).map((_e, i) => ({
+        secretKey: `BULK-KEY-${secretTestCases[0].secret.key}-${i + 1}`,
+        secretValue: "update-value",
+        secretComment: "comment",
+        secretPath: secretTestCases[0].path
+      }));
+      const secondBatchSecrets = Array.from(Array(5)).map((_e, i) => ({
+        secretKey: `BULK-KEY-${secretTestCases[1].secret.key}-${i + 1}`,
+        secretValue: "update-value",
+        secretComment: "comment",
+        secretPath: secretTestCases[1].path
+      }));
+      const testSecrets = [...firstBatchSecrets, ...secondBatchSecrets];
+
+      const updateSharedSecRes = await testServer.inject({
+        method: "PATCH",
+        url: `/api/v3/secrets/batch/raw`,
+        headers: {
+          authorization: `Bearer ${authToken}`
+        },
+        body: {
+          workspaceId: seedData1.projectV3.id,
+          environment: seedData1.environment.slug,
+          mode: "upsert",
+          secrets: testSecrets
+        }
+      });
+      expect(updateSharedSecRes.statusCode).toBe(200);
+      const updateSharedSecPayload = JSON.parse(updateSharedSecRes.payload);
+      expect(updateSharedSecPayload).toHaveProperty("secrets");
+
+      // bulk ones should exist
+      const firstBatchSecretsOnInfisical = await getSecrets(seedData1.environment.slug, secretTestCases[0].path);
+      expect(firstBatchSecretsOnInfisical).toEqual(
+        expect.arrayContaining(
+          firstBatchSecrets.map((el) =>
+            expect.objectContaining({
+              secretKey: el.secretKey,
+              secretValue: "update-value",
+              type: SecretType.Shared
+            })
+          )
+        )
+      );
+      const secondBatchSecretsOnInfisical = await getSecrets(seedData1.environment.slug, secretTestCases[1].path);
+      expect(secondBatchSecretsOnInfisical).toEqual(
+        expect.arrayContaining(
+          secondBatchSecrets.map((el) =>
+            expect.objectContaining({
+              secretKey: el.secretKey,
+              secretValue: "update-value",
+              type: SecretType.Shared
+            })
+          )
+        )
+      );
+      await Promise.all(testSecrets.map((el) => deleteSecret({ path: el.secretPath, key: el.secretKey })));
+    });
+
     test.each(secretTestCases)("Bulk delete secrets in path $path", async ({ secret, path }) => {
       await Promise.all(
         Array.from(Array(5)).map((_e, i) => createSecret({ ...secret, key: `BULK-${secret.key}-${i + 1}`, path }))

backend/src/ee/services/audit-log/audit-log-types.ts

@@ -352,6 +352,7 @@ interface CreateSecretBatchEvent {
     secrets: Array<{
       secretId: string;
       secretKey: string;
+      secretPath?: string;
       secretVersion: number;
       secretMetadata?: TSecretMetadata;
     }>;
@@ -374,8 +375,14 @@ interface UpdateSecretBatchEvent {
   type: EventType.UPDATE_SECRETS;
   metadata: {
     environment: string;
-    secretPath: string;
+    secretPath?: string;
-    secrets: Array<{ secretId: string; secretKey: string; secretVersion: number; secretMetadata?: TSecretMetadata }>;
+    secrets: Array<{
+      secretId: string;
+      secretKey: string;
+      secretVersion: number;
+      secretMetadata?: TSecretMetadata;
+      secretPath?: string;
+    }>;
   };
 }
 

backend/src/lib/api-docs/constants.ts

@@ -721,7 +721,8 @@ export const RAW_SECRETS = {
     secretName: "The name of the secret to update.",
     secretComment: "Update comment to the secret.",
     environment: "The slug of the environment where the secret is located.",
-    secretPath: "The path of the secret to update.",
+    mode: "Defines how the system should handle missing secrets during an update.",
+    secretPath: "The default path for secrets to update or upsert, if not provided in the secret details.",
     secretValue: "The new value of the secret.",
     skipMultilineEncoding: "Skip multiline encoding for the secret value.",
     type: "The type of the secret to update.",

backend/src/server/routes/v3/secret-router.ts

@@ -20,6 +20,7 @@ import { ActorType, AuthMode } from "@app/services/auth/auth-type";
 import { ProjectFilterType } from "@app/services/project/project-types";
 import { ResourceMetadataSchema } from "@app/services/resource-metadata/resource-metadata-schema";
 import { SecretOperations, SecretProtectionType } from "@app/services/secret/secret-types";
+import { SecretUpdateMode } from "@app/services/secret-v2-bridge/secret-v2-bridge-types";
 import { PostHogEventTypes } from "@app/services/telemetry/telemetry-types";
 
 import { secretRawSchema } from "../sanitizedSchemas";
@@ -2030,6 +2031,11 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => {
           .default("/")
           .transform(removeTrailingSlash)
           .describe(RAW_SECRETS.UPDATE.secretPath),
+        mode: z
+          .nativeEnum(SecretUpdateMode)
+          .optional()
+          .default(SecretUpdateMode.FailOnNotFound)
+          .describe(RAW_SECRETS.UPDATE.mode),
         secrets: z
           .object({
             secretKey: SecretNameSchema.describe(RAW_SECRETS.UPDATE.secretName),
@@ -2037,6 +2043,12 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => {
               .string()
               .transform((val) => (val.at(-1) === "\n" ? `${val.trim()}\n` : val.trim()))
               .describe(RAW_SECRETS.UPDATE.secretValue),
+            secretPath: z
+              .string()
+              .trim()
+              .transform(removeTrailingSlash)
+              .optional()
+              .describe(RAW_SECRETS.UPDATE.secretPath),
             secretComment: z.string().trim().optional().describe(RAW_SECRETS.UPDATE.secretComment),
             skipMultilineEncoding: z.boolean().optional().describe(RAW_SECRETS.UPDATE.skipMultilineEncoding),
             newSecretName: SecretNameSchema.optional().describe(RAW_SECRETS.UPDATE.newSecretName),
@@ -2073,7 +2085,8 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => {
         environment,
         projectSlug,
         projectId: req.body.workspaceId,
-        secrets: inputSecrets
+        secrets: inputSecrets,
+        mode: req.body.mode
       });
       if (secretOperation.type === SecretProtectionType.Approval) {
         return { approval: secretOperation.approval };
@@ -2092,15 +2105,39 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => {
           metadata: {
             environment: req.body.environment,
             secretPath: req.body.secretPath,
-            secrets: secrets.map((secret) => ({
+            secrets: secrets
-              secretId: secret.id,
+              .filter((el) => el.version > 1)
-              secretKey: secret.secretKey,
+              .map((secret) => ({
-              secretVersion: secret.version,
+                secretId: secret.id,
-              secretMetadata: secretMetadataMap.get(secret.secretKey)
+                secretPath: secret.secretPath,
-            }))
+                secretKey: secret.secretKey,
+                secretVersion: secret.version,
+                secretMetadata: secretMetadataMap.get(secret.secretKey)
+              }))
           }
         }
       });
+      const createdSecrets = secrets.filter((el) => el.version === 1);
+      if (createdSecrets.length) {
+        await server.services.auditLog.createAuditLog({
+          projectId: secrets[0].workspace,
+          ...req.auditLogInfo,
+          event: {
+            type: EventType.CREATE_SECRETS,
+            metadata: {
+              environment: req.body.environment,
+              secretPath: req.body.secretPath,
+              secrets: createdSecrets.map((secret) => ({
+                secretId: secret.id,
+                secretPath: secret.secretPath,
+                secretKey: secret.secretKey,
+                secretVersion: secret.version,
+                secretMetadata: secretMetadataMap.get(secret.secretKey)
+              }))
+            }
+          }
+        });
+      }
 
       await server.services.telemetry.sendPostHogEvents({
         event: PostHogEventTypes.SecretUpdated,

backend/src/services/secret-v2-bridge/secret-v2-bridge-service.ts

@@ -1,7 +1,15 @@
 import { ForbiddenError, PureAbility, subject } from "@casl/ability";
+import { Knex } from "knex";
 import { z } from "zod";
 
-import { ActionProjectType, ProjectMembershipRole, SecretsV2Schema, SecretType, TableName } from "@app/db/schemas";
+import {
+  ActionProjectType,
+  ProjectMembershipRole,
+  SecretsV2Schema,
+  SecretType,
+  TableName,
+  TSecretsV2
+} from "@app/db/schemas";
 import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service";
 import { ProjectPermissionActions, ProjectPermissionSub } from "@app/ee/services/permission/project-permission";
 import { TSecretApprovalPolicyServiceFactory } from "@app/ee/services/secret-approval-policy/secret-approval-policy-service";
@@ -36,6 +44,7 @@ import {
 } from "./secret-v2-bridge-fns";
 import {
   SecretOperations,
+  SecretUpdateMode,
   TBackFillSecretReferencesDTO,
   TCreateManySecretDTO,
   TCreateSecretDTO,
@@ -103,12 +112,13 @@ export const secretV2BridgeServiceFactory = ({
   const $validateSecretReferences = async (
     projectId: string,
     permission: PureAbility,
-    references: ReturnType<typeof getAllSecretReferences>["nestedReferences"]
+    references: ReturnType<typeof getAllSecretReferences>["nestedReferences"],
+    tx?: Knex
   ) => {
     if (!references.length) return;
 
     const uniqueReferenceEnvironmentSlugs = Array.from(new Set(references.map((el) => el.environment)));
-    const referencesEnvironments = await projectEnvDAL.findBySlugs(projectId, uniqueReferenceEnvironmentSlugs);
+    const referencesEnvironments = await projectEnvDAL.findBySlugs(projectId, uniqueReferenceEnvironmentSlugs, tx);
     if (referencesEnvironments.length !== uniqueReferenceEnvironmentSlugs.length)
       throw new BadRequestError({
         message: `Referenced environment not found. Missing ${diff(
@@ -122,36 +132,41 @@ export const secretV2BridgeServiceFactory = ({
       references.map((el) => ({
         secretPath: el.secretPath,
         envId: referencesEnvironmentGroupBySlug[el.environment][0].id
-      }))
+      })),
+      tx
     );
     const referencesFolderGroupByPath = groupBy(referredFolders.filter(Boolean), (i) => `${i?.envId}-${i?.path}`);
-    const referredSecrets = await secretDAL.find({
+    const referredSecrets = await secretDAL.find(
-      $complex: {
+      {
-        operator: "or",
+        $complex: {
-        value: references.map((el) => {
+          operator: "or",
-          const folderId =
+          value: references.map((el) => {
-            referencesFolderGroupByPath[`${referencesEnvironmentGroupBySlug[el.environment][0].id}-${el.secretPath}`][0]
+            const folderId =
-              ?.id;
+              referencesFolderGroupByPath[
-          if (!folderId) throw new BadRequestError({ message: `Referenced path ${el.secretPath} doesn't exist` });
+                `${referencesEnvironmentGroupBySlug[el.environment][0].id}-${el.secretPath}`
+              ][0]?.id;
+            if (!folderId) throw new BadRequestError({ message: `Referenced path ${el.secretPath} doesn't exist` });
 
-          return {
+            return {
-            operator: "and",
+              operator: "and",
-            value: [
+              value: [
-              {
+                {
-                operator: "eq",
+                  operator: "eq",
-                field: "folderId",
+                  field: "folderId",
-                value: folderId
+                  value: folderId
-              },
+                },
-              {
+                {
-                operator: "eq",
+                  operator: "eq",
-                field: `${TableName.SecretV2}.key` as "key",
+                  field: `${TableName.SecretV2}.key` as "key",
-                value: el.secretKey
+                  value: el.secretKey
-              }
+                }
-            ]
+              ]
-          };
+            };
-        })
+          })
-      }
+        }
-    });
+      },
+      { tx }
+    );
 
     if (
       referredSecrets.length !==
@@ -1245,8 +1260,9 @@ export const secretV2BridgeServiceFactory = ({
     actorAuthMethod,
     environment,
     projectId,
-    secretPath,
+    secretPath: defaultSecretPath = "/",
-    secrets: inputSecrets
+    secrets: inputSecrets,
+    mode: updateMode
   }: TUpdateManySecretDTO) => {
     const { permission } = await permissionService.getProjectPermission({
       actor,
@@ -1257,196 +1273,280 @@ export const secretV2BridgeServiceFactory = ({
       actionProjectType: ActionProjectType.SecretManager
     });
 
-    const folder = await folderDAL.findBySecretPath(projectId, environment, secretPath);
+    const secretsToUpdateGroupByPath = groupBy(inputSecrets, (el) => el.secretPath || defaultSecretPath);
-    if (!folder)
+    const projectEnvironment = await projectEnvDAL.findOne({ projectId, slug: environment });
+    if (!projectEnvironment) {
       throw new NotFoundError({
-        message: `Folder with path '${secretPath}' in environment with slug '${environment}' not found`,
+        message: `Environment with slug '${environment}' in project with ID '${projectId}' not found`
+      });
+    }
+
+    const folders = await folderDAL.findByManySecretPath(
+      Object.keys(secretsToUpdateGroupByPath).map((el) => ({ envId: projectEnvironment.id, secretPath: el }))
+    );
+    if (folders.length !== Object.keys(secretsToUpdateGroupByPath).length)
+      throw new NotFoundError({
+        message: `Folder with path '${null}' in environment with slug '${environment}' not found`,
         name: "UpdateManySecret"
       });
-    const folderId = folder.id;
-
-    const secretsToUpdate = await secretDAL.find({
-      folderId,
-      $complex: {
-        operator: "and",
-        value: [
-          {
-            operator: "or",
-            value: inputSecrets.map((el) => ({
-              operator: "and",
-              value: [
-                {
-                  operator: "eq",
-                  field: `${TableName.SecretV2}.key` as "key",
-                  value: el.secretKey
-                },
-                {
-                  operator: "eq",
-                  field: "type",
-                  value: SecretType.Shared
-                }
-              ]
-            }))
-          }
-        ]
-      }
-    });
-    if (secretsToUpdate.length !== inputSecrets.length) {
-      const secretsToUpdateNames = secretsToUpdate.map((secret) => secret.key);
-      const invalidSecrets = inputSecrets.filter((secret) => !secretsToUpdateNames.includes(secret.secretKey));
-      throw new NotFoundError({
-        message: `Secret does not exist: ${invalidSecrets.map((el) => el.secretKey).join(",")}`
-      });
-    }
-    const secretsToUpdateInDBGroupedByKey = groupBy(secretsToUpdate, (i) => i.key);
-
-    secretsToUpdate.forEach((el) => {
-      ForbiddenError.from(permission).throwUnlessCan(
-        ProjectPermissionActions.Edit,
-        subject(ProjectPermissionSub.Secrets, {
-          environment,
-          secretPath,
-          secretName: el.key,
-          secretTags: el.tags.map((i) => i.slug)
-        })
-      );
-    });
-
-    // get all tags
-    const sanitizedTagIds = inputSecrets.flatMap(({ tagIds = [] }) => tagIds);
-    const tags = sanitizedTagIds.length ? await secretTagDAL.findManyTagsById(projectId, sanitizedTagIds) : [];
-    if (tags.length !== sanitizedTagIds.length) throw new NotFoundError({ message: "Tag not found" });
-    const tagsGroupByID = groupBy(tags, (i) => i.id);
-
-    // check again to avoid non authorized tags are removed
-    inputSecrets.forEach((el) => {
-      ForbiddenError.from(permission).throwUnlessCan(
-        ProjectPermissionActions.Edit,
-        subject(ProjectPermissionSub.Secrets, {
-          environment,
-          secretPath,
-          secretName: el.secretKey,
-          secretTags: (el.tagIds || []).map((i) => tagsGroupByID[i][0].slug)
-        })
-      );
-    });
-
-    // now find any secret that needs to update its name
-    // same process as above
-    const secretsWithNewName = inputSecrets.filter(({ newSecretName }) => Boolean(newSecretName));
-    if (secretsWithNewName.length) {
-      const secrets = await secretDAL.find({
-        folderId,
-        $complex: {
-          operator: "and",
-          value: [
-            {
-              operator: "or",
-              value: secretsWithNewName.map((el) => ({
-                operator: "and",
-                value: [
-                  {
-                    operator: "eq",
-                    field: `${TableName.SecretV2}.key` as "key",
-                    value: el.secretKey
-                  },
-                  {
-                    operator: "eq",
-                    field: "type",
-                    value: SecretType.Shared
-                  }
-                ]
-              }))
-            }
-          ]
-        }
-      });
-      if (secrets.length)
-        throw new BadRequestError({
-          message: `Secret with new name already exists: ${secretsWithNewName.map((el) => el.newSecretName).join(",")}`
-        });
-
-      secretsWithNewName.forEach((el) => {
-        ForbiddenError.from(permission).throwUnlessCan(
-          ProjectPermissionActions.Create,
-          subject(ProjectPermissionSub.Secrets, {
-            environment,
-            secretPath,
-            secretName: el.newSecretName as string,
-            secretTags: (el.tagIds || []).map((i) => tagsGroupByID[i][0].slug)
-          })
-        );
-      });
-    }
-    // now get all secret references made and validate the permission
-    const secretReferencesGroupByInputSecretKey: Record<string, ReturnType<typeof getAllSecretReferences>> = {};
-    const secretReferences: TSecretReference[] = [];
-    inputSecrets.forEach((el) => {
-      if (el.secretValue) {
-        const references = getAllSecretReferences(el.secretValue);
-        secretReferencesGroupByInputSecretKey[el.secretKey] = references;
-        secretReferences.push(...references.nestedReferences);
-        references.localReferences.forEach((localRefKey) => {
-          secretReferences.push({ secretKey: localRefKey, secretPath, environment });
-        });
-      }
-    });
-    await $validateSecretReferences(projectId, permission, secretReferences);
 
     const { encryptor: secretManagerEncryptor, decryptor: secretManagerDecryptor } =
       await kmsService.createCipherPairWithDataKey({ type: KmsDataKey.SecretManager, projectId });
 
-    const secrets = await secretDAL.transaction(async (tx) =>
+    const updatedSecrets: Array<TSecretsV2 & { secretPath: string }> = [];
-      fnSecretBulkUpdate({
+    await secretDAL.transaction(async (tx) => {
-        folderId,
+      for await (const folder of folders) {
-        orgId: actorOrgId,
+        if (!folder) throw new NotFoundError({ message: "Folder not found" });
-        tx,
-        inputSecrets: inputSecrets.map((el) => {
-          const originalSecret = secretsToUpdateInDBGroupedByKey[el.secretKey][0];
-          const encryptedValue =
-            typeof el.secretValue !== "undefined"
-              ? {
-                  encryptedValue: secretManagerEncryptor({ plainText: Buffer.from(el.secretValue) }).cipherTextBlob,
-                  references: secretReferencesGroupByInputSecretKey[el.secretKey]?.nestedReferences
-                }
-              : {};
 
-          return {
+        const folderId = folder.id;
-            filter: { id: originalSecret.id, type: SecretType.Shared },
+        const secretPath = folder.path;
-            data: {
+        let secretsToUpdate = secretsToUpdateGroupByPath[secretPath];
-              reminderRepeatDays: el.secretReminderRepeatDays,
+        const secretsToUpdateInDB = await secretDAL.find(
-              encryptedComment: setKnexStringValue(
+          {
-                el.secretComment,
+            folderId,
-                (value) => secretManagerEncryptor({ plainText: Buffer.from(value) }).cipherTextBlob
+            $complex: {
-              ),
+              operator: "and",
-              reminderNote: el.secretReminderNote,
+              value: [
-              skipMultilineEncoding: el.skipMultilineEncoding,
+                {
-              key: el.newSecretName || el.secretKey,
+                  operator: "or",
-              tags: el.tagIds,
+                  value: secretsToUpdate.map((el) => ({
-              secretMetadata: el.secretMetadata,
+                    operator: "and",
-              ...encryptedValue
+                    value: [
+                      {
+                        operator: "eq",
+                        field: `${TableName.SecretV2}.key` as "key",
+                        value: el.secretKey
+                      },
+                      {
+                        operator: "eq",
+                        field: "type",
+                        value: SecretType.Shared
+                      }
+                    ]
+                  }))
+                }
+              ]
             }
-          };
+          },
-        }),
+          { tx }
-        secretDAL,
+        );
-        secretVersionDAL,
+        if (secretsToUpdateInDB.length !== secretsToUpdate.length && updateMode === SecretUpdateMode.FailOnNotFound)
-        secretTagDAL,
+          throw new NotFoundError({
-        secretVersionTagDAL,
+            message: `Secret does not exist: ${diff(
-        resourceMetadataDAL
+              secretsToUpdate.map((el) => el.secretKey),
-      })
+              secretsToUpdateInDB.map((el) => el.key)
-    );
+            ).join(", ")} in path ${folder.path}`
-    await snapshotService.performSnapshot(folderId);
+          });
-    await secretQueueService.syncSecrets({
+
-      actor,
+        const secretsToUpdateInDBGroupedByKey = groupBy(secretsToUpdateInDB, (i) => i.key);
-      actorId,
+        const secretsToCreate = secretsToUpdate.filter((el) => !secretsToUpdateInDBGroupedByKey?.[el.secretKey]);
-      secretPath,
+        secretsToUpdate = secretsToUpdate.filter((el) => secretsToUpdateInDBGroupedByKey?.[el.secretKey]);
-      projectId,
+
-      orgId: actorOrgId,
+        secretsToUpdateInDB.forEach((el) => {
-      environmentSlug: folder.environment.slug
+          ForbiddenError.from(permission).throwUnlessCan(
+            ProjectPermissionActions.Edit,
+            subject(ProjectPermissionSub.Secrets, {
+              environment,
+              secretPath,
+              secretName: el.key,
+              secretTags: el.tags.map((i) => i.slug)
+            })
+          );
+        });
+
+        // get all tags
+        const sanitizedTagIds = secretsToUpdate.flatMap(({ tagIds = [] }) => tagIds);
+        const tags = sanitizedTagIds.length ? await secretTagDAL.findManyTagsById(projectId, sanitizedTagIds, tx) : [];
+        if (tags.length !== sanitizedTagIds.length) throw new NotFoundError({ message: "Tag not found" });
+        const tagsGroupByID = groupBy(tags, (i) => i.id);
+
+        // check create permission allowed in upsert mode
+        if (updateMode === SecretUpdateMode.Upsert) {
+          secretsToCreate.forEach((el) => {
+            ForbiddenError.from(permission).throwUnlessCan(
+              ProjectPermissionActions.Create,
+              subject(ProjectPermissionSub.Secrets, {
+                environment,
+                secretPath,
+                secretName: el.secretKey,
+                secretTags: (el.tagIds || []).map((i) => tagsGroupByID[i][0].slug)
+              })
+            );
+          });
+        }
+
+        // check again to avoid non authorized tags are removed
+        secretsToUpdate.forEach((el) => {
+          ForbiddenError.from(permission).throwUnlessCan(
+            ProjectPermissionActions.Edit,
+            subject(ProjectPermissionSub.Secrets, {
+              environment,
+              secretPath,
+              secretName: el.secretKey,
+              secretTags: (el.tagIds || []).map((i) => tagsGroupByID[i][0].slug)
+            })
+          );
+        });
+
+        // now find any secret that needs to update its name
+        // same process as above
+        const secretsWithNewName = secretsToUpdate.filter(({ newSecretName }) => Boolean(newSecretName));
+        if (secretsWithNewName.length) {
+          const secrets = await secretDAL.find(
+            {
+              folderId,
+              $complex: {
+                operator: "and",
+                value: [
+                  {
+                    operator: "or",
+                    value: secretsWithNewName.map((el) => ({
+                      operator: "and",
+                      value: [
+                        {
+                          operator: "eq",
+                          field: `${TableName.SecretV2}.key` as "key",
+                          value: el.secretKey
+                        },
+                        {
+                          operator: "eq",
+                          field: "type",
+                          value: SecretType.Shared
+                        }
+                      ]
+                    }))
+                  }
+                ]
+              }
+            },
+            { tx }
+          );
+          if (secrets.length)
+            throw new BadRequestError({
+              message: `Secret with new name already exists: ${secretsWithNewName
+                .map((el) => el.newSecretName)
+                .join(", ")}`
+            });
+
+          secretsWithNewName.forEach((el) => {
+            ForbiddenError.from(permission).throwUnlessCan(
+              ProjectPermissionActions.Create,
+              subject(ProjectPermissionSub.Secrets, {
+                environment,
+                secretPath,
+                secretName: el.newSecretName as string,
+                secretTags: (el.tagIds || []).map((i) => tagsGroupByID[i][0].slug)
+              })
+            );
+          });
+        }
+        // now get all secret references made and validate the permission
+        const secretReferencesGroupByInputSecretKey: Record<string, ReturnType<typeof getAllSecretReferences>> = {};
+        const secretReferences: TSecretReference[] = [];
+        secretsToUpdate.concat(SecretUpdateMode.Upsert === updateMode ? secretsToCreate : []).forEach((el) => {
+          if (el.secretValue) {
+            const references = getAllSecretReferences(el.secretValue);
+            secretReferencesGroupByInputSecretKey[el.secretKey] = references;
+            secretReferences.push(...references.nestedReferences);
+            references.localReferences.forEach((localRefKey) => {
+              secretReferences.push({ secretKey: localRefKey, secretPath, environment });
+            });
+          }
+        });
+        await $validateSecretReferences(projectId, permission, secretReferences, tx);
+
+        const bulkUpdatedSecrets = await fnSecretBulkUpdate({
+          folderId,
+          orgId: actorOrgId,
+          tx,
+          inputSecrets: secretsToUpdate.map((el) => {
+            const originalSecret = secretsToUpdateInDBGroupedByKey[el.secretKey][0];
+            const encryptedValue =
+              typeof el.secretValue !== "undefined"
+                ? {
+                    encryptedValue: secretManagerEncryptor({ plainText: Buffer.from(el.secretValue) }).cipherTextBlob,
+                    references: secretReferencesGroupByInputSecretKey[el.secretKey]?.nestedReferences
+                  }
+                : {};
+
+            return {
+              filter: { id: originalSecret.id, type: SecretType.Shared },
+              data: {
+                reminderRepeatDays: el.secretReminderRepeatDays,
+                encryptedComment: setKnexStringValue(
+                  el.secretComment,
+                  (value) => secretManagerEncryptor({ plainText: Buffer.from(value) }).cipherTextBlob
+                ),
+                reminderNote: el.secretReminderNote,
+                skipMultilineEncoding: el.skipMultilineEncoding,
+                key: el.newSecretName || el.secretKey,
+                tags: el.tagIds,
+                secretMetadata: el.secretMetadata,
+                ...encryptedValue
+              }
+            };
+          }),
+          secretDAL,
+          secretVersionDAL,
+          secretTagDAL,
+          secretVersionTagDAL,
+          resourceMetadataDAL
+        });
+        updatedSecrets.push(...bulkUpdatedSecrets.map((el) => ({ ...el, secretPath: folder.path })));
+        if (updateMode === SecretUpdateMode.Upsert) {
+          const bulkInsertedSecrets = await fnSecretBulkInsert({
+            inputSecrets: secretsToCreate.map((el) => {
+              const references = secretReferencesGroupByInputSecretKey[el.secretKey]?.nestedReferences;
+
+              return {
+                version: 1,
+                encryptedComment: setKnexStringValue(
+                  el.secretComment,
+                  (value) => secretManagerEncryptor({ plainText: Buffer.from(value) }).cipherTextBlob
+                ),
+                encryptedValue: el.secretValue
+                  ? secretManagerEncryptor({ plainText: Buffer.from(el.secretValue) }).cipherTextBlob
+                  : undefined,
+                skipMultilineEncoding: el.skipMultilineEncoding,
+                key: el.secretKey,
+                tagIds: el.tagIds,
+                references,
+                secretMetadata: el.secretMetadata,
+                type: SecretType.Shared
+              };
+            }),
+            folderId,
+            orgId: actorOrgId,
+            secretDAL,
+            resourceMetadataDAL,
+            secretVersionDAL,
+            secretTagDAL,
+            secretVersionTagDAL,
+            tx
+          });
+          updatedSecrets.push(...bulkInsertedSecrets.map((el) => ({ ...el, secretPath: folder.path })));
+        }
+      }
     });
 
-    return secrets.map((el) =>
+    await Promise.allSettled(folders.map((el) => (el?.id ? snapshotService.performSnapshot(el.id) : undefined)));
-      reshapeBridgeSecret(projectId, environment, secretPath, {
+    await Promise.allSettled(
+      folders.map((el) =>
+        el
+          ? secretQueueService.syncSecrets({
+              actor,
+              actorId,
+              secretPath: el.path,
+              projectId,
+              orgId: actorOrgId,
+              environmentSlug: environment
+            })
+          : undefined
+      )
+    );
+
+    return updatedSecrets.map((el) =>
+      reshapeBridgeSecret(projectId, environment, el.secretPath, {
         ...el,
         value: el.encryptedValue ? secretManagerDecryptor({ cipherTextBlob: el.encryptedValue }).toString() : "",
         comment: el.encryptedComment ? secretManagerDecryptor({ cipherTextBlob: el.encryptedComment }).toString() : ""

backend/src/services/secret-v2-bridge/secret-v2-bridge-types.ts

@@ -23,6 +23,12 @@ export type TSecretReferenceDTO = {
   secretKey: string;
 };
 
+export enum SecretUpdateMode {
+  Ignore = "ignore",
+  Upsert = "upsert",
+  FailOnNotFound = "failOnNotFound"
+}
+
 export type TGetSecretsDTO = {
   expandSecretReferences?: boolean;
   path: string;
@@ -113,6 +119,7 @@ export type TUpdateManySecretDTO = Omit<TProjectPermission, "projectId"> & {
   secretPath: string;
   projectId: string;
   environment: string;
+  mode: SecretUpdateMode;
   secrets: {
     secretKey: string;
     newSecretName?: string;
@@ -123,6 +130,7 @@ export type TUpdateManySecretDTO = Omit<TProjectPermission, "projectId"> & {
     secretReminderRepeatDays?: number | null;
     secretReminderNote?: string | null;
     secretMetadata?: ResourceMetadataDTO;
+    secretPath?: string;
   }[];
 };
 

backend/src/services/secret/secret-service.ts

@@ -30,7 +30,10 @@ import { groupBy, pick } from "@app/lib/fn";
 import { logger } from "@app/lib/logger";
 import { alphaNumericNanoId } from "@app/lib/nanoid";
 import { OrgServiceActor } from "@app/lib/types";
-import { TGetSecretsRawByFolderMappingsDTO } from "@app/services/secret-v2-bridge/secret-v2-bridge-types";
+import {
+  SecretUpdateMode,
+  TGetSecretsRawByFolderMappingsDTO
+} from "@app/services/secret-v2-bridge/secret-v2-bridge-types";
 
 import { ActorType } from "../auth/auth-type";
 import { TProjectDALFactory } from "../project/project-dal";
@@ -2012,6 +2015,7 @@ export const secretServiceFactory = ({
     actorOrgId,
     actorAuthMethod,
     secretPath,
+    mode = SecretUpdateMode.FailOnNotFound,
     secrets: inputSecrets = []
   }: TUpdateManySecretRawDTO) => {
     if (!projectSlug && !optionalProjectId)
@@ -2076,7 +2080,8 @@ export const secretServiceFactory = ({
         actorOrgId,
         actor,
         actorId,
-        secrets: inputSecrets
+        secrets: inputSecrets,
+        mode
       });
       return { type: SecretProtectionType.Direct as const, secrets };
     }

backend/src/services/secret/secret-types.ts

@@ -17,6 +17,7 @@ import { TKmsServiceFactory } from "../kms/kms-service";
 import { TResourceMetadataDALFactory } from "../resource-metadata/resource-metadata-dal";
 import { ResourceMetadataDTO } from "../resource-metadata/resource-metadata-schema";
 import { TSecretV2BridgeDALFactory } from "../secret-v2-bridge/secret-v2-bridge-dal";
+import { SecretUpdateMode } from "../secret-v2-bridge/secret-v2-bridge-types";
 import { TSecretVersionV2DALFactory } from "../secret-v2-bridge/secret-version-dal";
 import { TSecretVersionV2TagDALFactory } from "../secret-v2-bridge/secret-version-tag-dal";
 
@@ -274,6 +275,7 @@ export type TUpdateManySecretRawDTO = Omit<TProjectPermission, "projectId"> & {
   projectId?: string;
   projectSlug?: string;
   environment: string;
+  mode: SecretUpdateMode;
   secrets: {
     secretKey: string;
     newSecretName?: string;