Make sync behavior apply on first sync only, finish MVP create/update bidirectional sync for Heroku

backend/src/db/migrations/20240307232900_integration-last-used.ts

@@ -0,0 +1,15 @@
+import { Knex } from "knex";
+
+import { TableName } from "../schemas";
+
+export async function up(knex: Knex): Promise<void> {
+  await knex.schema.alterTable(TableName.Integration, (t) => {
+    t.datetime("lastUsed");
+  });
+}
+
+export async function down(knex: Knex): Promise<void> {
+  await knex.schema.alterTable(TableName.Integration, (t) => {
+    t.dropColumn("lastUsed");
+  });
+}

backend/src/db/schemas/integrations.ts

@@ -27,7 +27,8 @@ export const IntegrationsSchema = z.object({
   envId: z.string().uuid(),
   secretPath: z.string().default("/"),
   createdAt: z.date(),
-  updatedAt: z.date()
+  updatedAt: z.date(),
+  lastUsed: z.date().nullable().optional()
 });
 
 export type TIntegrations = z.infer<typeof IntegrationsSchema>;

backend/src/ee/services/secret-approval-request/secret-approval-request-service.ts

@@ -12,14 +12,14 @@ import { groupBy, pick, unique } from "@app/lib/fn";
 import { alphaNumericNanoId } from "@app/lib/nanoid";
 import { ActorType } from "@app/services/auth/auth-type";
 import { TProjectDALFactory } from "@app/services/project/project-dal";
+import { TSecretDALFactory } from "@app/services/secret/secret-dal";
 import { TSecretQueueFactory } from "@app/services/secret/secret-queue";
 import { TSecretServiceFactory } from "@app/services/secret/secret-service";
 import { TSecretVersionDALFactory } from "@app/services/secret/secret-version-dal";
+import { TSecretVersionTagDALFactory } from "@app/services/secret/secret-version-tag-dal";
 import { TSecretBlindIndexDALFactory } from "@app/services/secret-blind-index/secret-blind-index-dal";
 import { TSecretFolderDALFactory } from "@app/services/secret-folder/secret-folder-dal";
 import { TSecretTagDALFactory } from "@app/services/secret-tag/secret-tag-dal";
-import { TSecretDALFactory } from "@app/services/secret/secret-dal";
-import { TSecretVersionTagDALFactory } from "@app/services/secret/secret-version-tag-dal";
 
 import { TPermissionServiceFactory } from "../permission/permission-service";
 import { ProjectPermissionActions, ProjectPermissionSub } from "../permission/project-permission";
@@ -47,7 +47,7 @@ type TSecretApprovalRequestServiceFactoryDep = {
   secretApprovalRequestReviewerDAL: TSecretApprovalRequestReviewerDALFactory;
   folderDAL: Pick<TSecretFolderDALFactory, "findBySecretPath" | "findById" | "findSecretPathByFolderIds">;
   secretDAL: TSecretDALFactory;
-  secretTagDAL: Pick<TSecretTagDALFactory, "findManyTagsById" | "saveTagsToSecret">;
+  secretTagDAL: Pick<TSecretTagDALFactory, "findManyTagsById" | "saveTagsToSecret" | "deleteTagsManySecret">;
   secretBlindIndexDAL: Pick<TSecretBlindIndexDALFactory, "findOne">;
   snapshotService: Pick<TSecretSnapshotServiceFactory, "performSnapshot">;
   secretVersionDAL: Pick<TSecretVersionDALFactory, "findLatestVersionMany" | "insertMany">;
@@ -377,7 +377,11 @@ export const secretApprovalRequestServiceFactory = ({
                   "secretBlindIndex"
                 ])
               }
-            }))
+            })),
+            secretDAL,
+            secretVersionDAL,
+            secretTagDAL,
+            secretVersionTagDAL
           })
         : [];
       const deletedSecret = secretDeletionCommits.length

backend/src/services/integration-auth/integration-sync-secret.ts

@@ -20,20 +20,20 @@ import sodium from "libsodium-wrappers";
 import isEqual from "lodash.isequal";
 import { z } from "zod";
 
-import { TIntegrationAuths, TIntegrations, SecretType } from "@app/db/schemas";
+import { SecretType, TIntegrationAuths, TIntegrations } from "@app/db/schemas";
 import { request } from "@app/lib/config/request";
 import { BadRequestError } from "@app/lib/errors";
-
-import { Integrations, IntegrationUrls, IntegrationSyncBehavior } from "./integration-list";
 import { TProjectDALFactory } from "@app/services/project/project-dal";
 import { TSecretDALFactory } from "@app/services/secret/secret-dal";
-import { TSecretBlindIndexDALFactory } from "@app/services/secret-blind-index/secret-blind-index-dal";
-import { TSecretTagDALFactory } from "@app/services/secret-tag/secret-tag-dal";
-import { TSecretFolderDALFactory } from "@app/services/secret-folder/secret-folder-dal";
 import { TSecretVersionDALFactory } from "@app/services/secret/secret-version-dal";
 import { TSecretVersionTagDALFactory } from "@app/services/secret/secret-version-tag-dal";
+import { TSecretBlindIndexDALFactory } from "@app/services/secret-blind-index/secret-blind-index-dal";
+import { TSecretFolderDALFactory } from "@app/services/secret-folder/secret-folder-dal";
+import { TSecretTagDALFactory } from "@app/services/secret-tag/secret-tag-dal";
 
-import { createManySecretsRawHelper } from "../secret/secret-fns";
+import { TIntegrationDALFactory } from "../integration/integration-dal";
+import { createManySecretsRawHelper, updateManySecretsRawHelper } from "../secret/secret-fns";
+import { Integrations, IntegrationSyncBehavior, IntegrationUrls } from "./integration-list";
 
 const getSecretKeyValuePair = (secrets: Record<string, { value: string | null; comment?: string } | null>) =>
   Object.keys(secrets).reduce<Record<string, string | null | undefined>>((prev, key) => {
@@ -589,10 +589,11 @@ const syncSecretsAWSSecretManager = async ({
 
 /**
  * Sync/push [secrets] to Heroku app named [integration.app]
- * server.services... 
+ * server.services...
  */
 const syncSecretsHeroku = async ({
   projectDAL,
+  integrationDAL,
   secretDAL,
   secretVersionDAL,
   secretBlindIndexDAL,
@@ -608,6 +609,7 @@ const syncSecretsHeroku = async ({
   accessToken
 }: {
   projectDAL: TProjectDALFactory;
+  integrationDAL: TIntegrationDALFactory;
   secretDAL: TSecretDALFactory;
   secretVersionDAL: TSecretVersionDALFactory;
   secretBlindIndexDAL: TSecretBlindIndexDALFactory;
@@ -622,7 +624,6 @@ const syncSecretsHeroku = async ({
   secrets: Record<string, { value: string; comment?: string } | null>;
   accessToken: string;
 }) => {
-
   const herokuSecrets = (
     await request.get(`${IntegrationUrls.HEROKU_API_URL}/apps/${integration.app}/config-vars`, {
       headers: {
@@ -632,62 +633,90 @@ const syncSecretsHeroku = async ({
       }
     })
   ).data;
-  
+
-  let secretsToAdd: { [key: string]: string } = {};
+  const secretsToAdd: { [key: string]: string } = {};
-  let secretsToUpdate: { [key: string]: string } = {};
+  const secretsToUpdate: { [key: string]: string } = {};
 
   const metadata = z.record(z.any()).parse(integration.metadata);
 
   Object.keys(herokuSecrets).forEach((key) => {
-    switch (metadata.syncBehavior) {
+    if (!integration.lastUsed) {
-      case IntegrationSyncBehavior.OVERWRITE_TARGET: {
+      // first time using integration
-        if (!(key in secrets)) secrets[key] = null;
+      // -> apply initial sync behavior rule
-        break;
+      switch (metadata.syncBehavior) {
-      };
+        case IntegrationSyncBehavior.OVERWRITE_TARGET: {
-      case IntegrationSyncBehavior.PREFER_TARGET: {
+          if (!(key in secrets)) secrets[key] = null;
-        secrets[key] = herokuSecrets[key];
+          break;
-        if (!(key in secrets)) {
+        }
-          secretsToAdd[key] = herokuSecrets[key];
+        case IntegrationSyncBehavior.PREFER_TARGET: {
-        } else {
+          if (!(key in secrets)) {
-          if (secrets[key] !== herokuSecrets[key]) {
+            secretsToAdd[key] = herokuSecrets[key];
-            secretsToUpdate[key] = secrets[key]?.value as string;
+          } else if (secrets[key]?.value !== herokuSecrets[key]) {
+            secretsToUpdate[key] = herokuSecrets[key];
           }
+          secrets[key] = {
+            value: herokuSecrets[key]
+          };
+          break;
         }
-        break;
+        case IntegrationSyncBehavior.PREFER_SOURCE: {
-      };
+          if (!(key in secrets)) {
-      case IntegrationSyncBehavior.PREFER_SOURCE: {
+            secrets[key] = herokuSecrets[key];
-        if(!(key in secrets)) {
+            secretsToAdd[key] = herokuSecrets[key];
-          secrets[key] = herokuSecrets[key];
+          }
-          secretsToAdd[key] = herokuSecrets[key];
+          break;
         }
-        break;
+        default: {
-      };
+          if (!(key in secrets)) secrets[key] = null;
-      default: {
+          break;
-        if (!(key in secrets)) secrets[key] = null;
+        }
-        break;
+      }
-      };
+    } else if (!(key in secrets)) secrets[key] = null;
-    }
   });
 
-  await createManySecretsRawHelper({
+  if (Object.keys(secretsToAdd).length) {
-    botKey,
+    await createManySecretsRawHelper({
-    projectDAL,
+      botKey,
-    secretDAL,
+      projectDAL,
-    secretVersionDAL,
+      secretDAL,
-    secretBlindIndexDAL,
+      secretVersionDAL,
-    secretTagDAL,
+      secretBlindIndexDAL,
-    secretVersionTagDAL,
+      secretTagDAL,
-    folderDAL,
+      secretVersionTagDAL,
-    projectId,
+      folderDAL,
-    environment,
+      projectId,
-    path: secretPath,
+      environment,
-    secrets: Object.keys(secretsToAdd).map((key) => ({
+      path: secretPath,
-      secretName: key,
+      secrets: Object.keys(secretsToAdd).map((key) => ({
-      secretValue: secretsToAdd[key],
+        secretName: key,
-      type: SecretType.Shared,
+        secretValue: secretsToAdd[key],
-      secretComment: ""
+        type: SecretType.Shared,
-    }))
+        secretComment: ""
-  });
+      }))
+    });
+  }
+
+  if (Object.keys(secretsToUpdate).length) {
+    await updateManySecretsRawHelper({
+      projectId,
+      environment,
+      path: secretPath,
+      secrets: Object.keys(secretsToUpdate).map((key) => ({
+        secretName: key,
+        secretValue: secretsToUpdate[key],
+        type: SecretType.Shared,
+        secretComment: ""
+      })),
+      botKey, // TODO: consider getting botKey inside this fn
+      projectDAL,
+      secretDAL,
+      secretVersionDAL,
+      secretBlindIndexDAL,
+      secretTagDAL,
+      secretVersionTagDAL,
+      folderDAL
+    });
+  }
 
   await request.patch(
     `${IntegrationUrls.HEROKU_API_URL}/apps/${integration.app}/config-vars`,
@@ -700,6 +729,10 @@ const syncSecretsHeroku = async ({
       }
     }
   );
+
+  await integrationDAL.updateById(integration.id, {
+    lastUsed: new Date()
+  });
 };
 
 /**
@@ -3013,12 +3046,13 @@ const syncSecretsHasuraCloud = async ({
 
 /**
  * Sync/push [secrets] to [app] in integration named [integration]
- * 
+ *
  * Do this in terms of DAL
- * 
+ *
  */
 export const syncIntegrationSecrets = async ({
   projectDAL,
+  integrationDAL,
   secretDAL,
   secretVersionDAL,
   secretBlindIndexDAL,
@@ -3037,6 +3071,7 @@ export const syncIntegrationSecrets = async ({
   appendices
 }: {
   projectDAL: TProjectDALFactory;
+  integrationDAL: TIntegrationDALFactory;
   secretDAL: TSecretDALFactory;
   secretVersionDAL: TSecretVersionDALFactory;
   secretBlindIndexDAL: TSecretBlindIndexDALFactory;
@@ -3088,6 +3123,7 @@ export const syncIntegrationSecrets = async ({
     case Integrations.HEROKU:
       await syncSecretsHeroku({
         projectDAL,
+        integrationDAL,
         secretDAL,
         secretVersionDAL,
         secretBlindIndexDAL,
@@ -3100,7 +3136,7 @@ export const syncIntegrationSecrets = async ({
         secretPath,
         integration,
         secrets,
-        accessToken,
+        accessToken
       });
       break;
     case Integrations.VERCEL:

backend/src/services/secret/secret-fns.ts

@@ -1,32 +1,33 @@
 /* eslint-disable no-await-in-loop */
 import path from "path";
 
-import { 
+import {
-  SecretKeyEncoding, 
-  TSecretBlindIndexes, 
-  TSecrets,
   SecretEncryptionAlgo,
+  SecretKeyEncoding,
   SecretType,
-  TableName
+  TableName,
+  TSecretBlindIndexes,
+  TSecrets
 } from "@app/db/schemas";
 import { getConfig } from "@app/lib/config/env";
-import { 
+import {
-  buildSecretBlindIndexFromName, 
+  buildSecretBlindIndexFromName,
-  encryptSymmetric128BitHexKeyUTF8,
+  decryptSymmetric128BitHexKeyUTF8,
-  decryptSymmetric128BitHexKeyUTF8 
+  encryptSymmetric128BitHexKeyUTF8
 } from "@app/lib/crypto";
+import { BadRequestError } from "@app/lib/errors";
+import { groupBy, unique } from "@app/lib/fn";
 
 import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal";
 import { TSecretDALFactory } from "./secret-dal";
-import { 
+import {
   TCreateManySecretsRawHelper,
+  TFnSecretBlindIndexCheck,
   TFnSecretBulkInsert,
-  TFnSecretBlindIndexCheck
+  TFnSecretBulkUpdate,
+  TUpdateManySecretsRawHelper
 } from "./secret-types";
 
-import { groupBy, unique } from "@app/lib/fn";
-import { BadRequestError } from "@app/lib/errors";
-
 export const generateSecretBlindIndexBySalt = async (secretName: string, secretBlindIndexDoc: TSecretBlindIndexes) => {
   const appCfg = getConfig();
   const secretBlindIndex = await buildSecretBlindIndexFromName({
@@ -308,14 +309,15 @@ export const fnSecretBlindIndexCheck = async ({
 
 // these functions are special functions shared by a couple of resources
 // used by secret approval, rotation or anywhere in which secret needs to modified
-export const fnSecretBulkInsert = async ({ // TODO: Pick types here 
+export const fnSecretBulkInsert = async ({
-  folderId, 
+  // TODO: Pick types here
-  inputSecrets, 
+  folderId,
+  inputSecrets,
   secretDAL,
   secretVersionDAL,
   secretTagDAL,
   secretVersionTagDAL,
-  tx 
+  tx
 }: TFnSecretBulkInsert) => {
   const newSecrets = await secretDAL.insertMany(
     inputSecrets.map(({ tags, ...el }) => ({ ...el, folderId })),
@@ -349,10 +351,63 @@ export const fnSecretBulkInsert = async ({ // TODO: Pick types here
   return newSecrets.map((secret) => ({ ...secret, _id: secret.id }));
 };
 
-export const createManySecretsRawHelper = async ({ // TODO: place on top
+export const fnSecretBulkUpdate = async ({
+  tx,
+  inputSecrets,
+  folderId,
+  projectId,
+  secretDAL,
+  secretVersionDAL,
+  secretTagDAL,
+  secretVersionTagDAL
+}: TFnSecretBulkUpdate) => {
+  const newSecrets = await secretDAL.bulkUpdate(
+    inputSecrets.map(({ filter, data: { tags, ...data } }) => ({
+      filter: { ...filter, folderId },
+      data
+    })),
+    tx
+  );
+  const secretVersions = await secretVersionDAL.insertMany(
+    newSecrets.map(({ id, createdAt, updatedAt, ...el }) => ({
+      ...el,
+      secretId: id
+    })),
+    tx
+  );
+  const secsUpdatedTag = inputSecrets.flatMap(({ data: { tags } }, i) =>
+    tags !== undefined ? { tags, secretId: newSecrets[i].id } : []
+  );
+  if (secsUpdatedTag.length) {
+    await secretTagDAL.deleteTagsManySecret(
+      projectId,
+      secsUpdatedTag.map(({ secretId }) => secretId),
+      tx
+    );
+    const newSecretTags = secsUpdatedTag.flatMap(({ tags: secretTags = [], secretId }) =>
+      secretTags.map((tag) => ({
+        [`${TableName.SecretTag}Id` as const]: tag,
+        [`${TableName.Secret}Id` as const]: secretId
+      }))
+    );
+    if (newSecretTags.length) {
+      const secTags = await secretTagDAL.saveTagsToSecret(newSecretTags, tx);
+      const secVersionsGroupBySecId = groupBy(secretVersions, (i) => i.secretId);
+      const newSecretVersionTags = secTags.flatMap(({ secretsId, secret_tagsId }) => ({
+        [`${TableName.SecretVersion}Id` as const]: secVersionsGroupBySecId[secretsId][0].id,
+        [`${TableName.SecretTag}Id` as const]: secret_tagsId
+      }));
+      await secretVersionTagDAL.insertMany(newSecretVersionTags, tx);
+    }
+  }
+
+  return newSecrets.map((secret) => ({ ...secret, _id: secret.id }));
+};
+
+export const createManySecretsRawHelper = async ({
   projectId,
   environment,
-  path,
+  path: secretPath,
   secrets,
   userId,
   botKey, // TODO: consider getting botKey inside this fn
@@ -364,67 +419,66 @@ export const createManySecretsRawHelper = async ({ // TODO: place on top
   secretVersionTagDAL,
   folderDAL
 }: TCreateManySecretsRawHelper) => {
-await projectDAL.checkProjectUpgradeStatus(projectId);
+  await projectDAL.checkProjectUpgradeStatus(projectId);
 
-const folder = await folderDAL.findBySecretPath(projectId, environment, path);
+  const folder = await folderDAL.findBySecretPath(projectId, environment, secretPath);
-if (!folder) throw new BadRequestError({ message: "Folder not  found", name: "Create secret" });
+  if (!folder) throw new BadRequestError({ message: "Folder not  found", name: "Create secret" });
-const folderId = folder.id;
+  const folderId = folder.id;
 
-const blindIndexCfg = await secretBlindIndexDAL.findOne({ projectId });
+  const blindIndexCfg = await secretBlindIndexDAL.findOne({ projectId });
-if (!blindIndexCfg) throw new BadRequestError({ message: "Blind index not found", name: "Update secret" });
+  if (!blindIndexCfg) throw new BadRequestError({ message: "Blind index not found", name: "Create secret" });
-
-const inputSecrets = await Promise.all(
-  secrets.map(async (secret) => {
-    const secretKeyEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretName, botKey);
-    const secretValueEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretValue || "", botKey);
-    const secretCommentEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretComment || "", botKey);
-
-    if (secret.type === SecretType.Personal) {
-      const sharedExist = await secretDAL.findOne({
-        secretBlindIndex: keyName2BlindIndex[secret.secretName],
-        folderId,
-        type: SecretType.Shared
-      });
-
-      if (!sharedExist)
-        throw new BadRequestError({
-          message: "Failed to create personal secret override for no corresponding shared secret"
-        });
-      
-      if (!userId) throw new BadRequestError({ message: "Missing user id for personal secret" });
-    }
-
-    const tags = secret.tags ? await secretTagDAL.findManyTagsById(projectId, secret.tags) : [];
-    if ((secret.tags || []).length !== tags.length) throw new BadRequestError({ message: "Tag not found" });
-
-    return ({
-      type: secret.type,
-      userId: secret.type === SecretType.Personal ? userId : null,
-      secretName: secret.secretName,
-      secretKeyCiphertext: secretKeyEncrypted.ciphertext,
-      secretKeyIV: secretKeyEncrypted.iv,
-      secretKeyTag: secretKeyEncrypted.tag,
-      secretValueCiphertext: secretValueEncrypted.ciphertext,
-      secretValueIV: secretValueEncrypted.iv,
-      secretValueTag: secretValueEncrypted.tag,
-      secretCommentCiphertext: secretCommentEncrypted.ciphertext,
-      secretCommentIV: secretCommentEncrypted.iv,
-      secretCommentTag: secretCommentEncrypted.tag,
-      skipMultilineEncoding: secret.skipMultilineEncoding,
-      tags: secret.tags
-    });
-  })
-);
 
   // insert operation
   const { keyName2BlindIndex } = await fnSecretBlindIndexCheck({
-    inputSecrets,
+    inputSecrets: secrets,
     folderId,
     isNew: true,
     blindIndexCfg,
     secretDAL
   });
 
+  const inputSecrets = await Promise.all(
+    secrets.map(async (secret) => {
+      const secretKeyEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretName, botKey);
+      const secretValueEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretValue || "", botKey);
+      const secretCommentEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretComment || "", botKey);
+
+      if (secret.type === SecretType.Personal) {
+        if (!userId) throw new BadRequestError({ message: "Missing user id for personal secret" });
+        const sharedExist = await secretDAL.findOne({
+          secretBlindIndex: keyName2BlindIndex[secret.secretName],
+          folderId,
+          type: SecretType.Shared
+        });
+
+        if (!sharedExist)
+          throw new BadRequestError({
+            message: "Failed to create personal secret override for no corresponding shared secret"
+          });
+      }
+
+      const tags = secret.tags ? await secretTagDAL.findManyTagsById(projectId, secret.tags) : [];
+      if ((secret.tags || []).length !== tags.length) throw new BadRequestError({ message: "Tag not found" });
+
+      return {
+        type: secret.type,
+        userId: secret.type === SecretType.Personal ? userId : null,
+        secretName: secret.secretName,
+        secretKeyCiphertext: secretKeyEncrypted.ciphertext,
+        secretKeyIV: secretKeyEncrypted.iv,
+        secretKeyTag: secretKeyEncrypted.tag,
+        secretValueCiphertext: secretValueEncrypted.ciphertext,
+        secretValueIV: secretValueEncrypted.iv,
+        secretValueTag: secretValueEncrypted.tag,
+        secretCommentCiphertext: secretCommentEncrypted.ciphertext,
+        secretCommentIV: secretCommentEncrypted.iv,
+        secretCommentTag: secretCommentEncrypted.tag,
+        skipMultilineEncoding: secret.skipMultilineEncoding,
+        tags: secret.tags
+      };
+    })
+  );
+
   const newSecrets = await secretDAL.transaction(async (tx) =>
     fnSecretBulkInsert({
       inputSecrets: inputSecrets.map(({ secretName, ...el }) => ({
@@ -444,4 +498,130 @@ const inputSecrets = await Promise.all(
   );
 
   return newSecrets;
-}
+};
+
+export const updateManySecretsRawHelper = async ({
+  projectId,
+  environment,
+  path: secretPath,
+  secrets,
+  userId,
+  botKey, // TODO: consider getting botKey inside this fn
+  projectDAL,
+  secretDAL,
+  secretVersionDAL,
+  secretBlindIndexDAL,
+  secretTagDAL,
+  secretVersionTagDAL,
+  folderDAL
+}: TUpdateManySecretsRawHelper) => {
+  await projectDAL.checkProjectUpgradeStatus(projectId);
+
+  const folder = await folderDAL.findBySecretPath(projectId, environment, secretPath);
+  if (!folder) throw new BadRequestError({ message: "Folder not  found", name: "Update secret" });
+  const folderId = folder.id;
+
+  const blindIndexCfg = await secretBlindIndexDAL.findOne({ projectId });
+  if (!blindIndexCfg) throw new BadRequestError({ message: "Blind index not found", name: "Update secret" });
+
+  const { keyName2BlindIndex } = await fnSecretBlindIndexCheck({
+    inputSecrets: secrets,
+    folderId,
+    isNew: false,
+    blindIndexCfg,
+    secretDAL,
+    userId
+  });
+
+  const inputSecrets = await Promise.all(
+    secrets.map(async (secret) => {
+      if (secret.newSecretName === "") {
+        throw new BadRequestError({ message: "New secret name cannot be empty" });
+      }
+
+      const secretKeyEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretName, botKey);
+      const secretValueEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretValue || "", botKey);
+      const secretCommentEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretComment || "", botKey);
+
+      if (secret.type === SecretType.Personal) {
+        if (!userId) throw new BadRequestError({ message: "Missing user id for personal secret" });
+
+        const sharedExist = await secretDAL.findOne({
+          secretBlindIndex: keyName2BlindIndex[secret.secretName],
+          folderId,
+          type: SecretType.Shared
+        });
+
+        if (!sharedExist)
+          throw new BadRequestError({
+            message: "Failed to update personal secret override for no corresponding shared secret"
+          });
+
+        if (secret.newSecretName) throw new BadRequestError({ message: "Personal secret cannot change the key name" });
+      }
+
+      const tags = secret.tags ? await secretTagDAL.findManyTagsById(projectId, secret.tags) : [];
+      if ((secret.tags || []).length !== tags.length) throw new BadRequestError({ message: "Tag not found" });
+
+      return {
+        type: secret.type,
+        userId: secret.type === SecretType.Personal ? userId : null,
+        secretName: secret.secretName,
+        newSecretName: secret.newSecretName,
+        secretKeyCiphertext: secretKeyEncrypted.ciphertext,
+        secretKeyIV: secretKeyEncrypted.iv,
+        secretKeyTag: secretKeyEncrypted.tag,
+        secretValueCiphertext: secretValueEncrypted.ciphertext,
+        secretValueIV: secretValueEncrypted.iv,
+        secretValueTag: secretValueEncrypted.tag,
+        secretCommentCiphertext: secretCommentEncrypted.ciphertext,
+        secretCommentIV: secretCommentEncrypted.iv,
+        secretCommentTag: secretCommentEncrypted.tag,
+        skipMultilineEncoding: secret.skipMultilineEncoding,
+        tags: secret.tags
+      };
+    })
+  );
+
+  const tagIds = inputSecrets.flatMap(({ tags = [] }) => tags);
+  const tags = tagIds.length ? await secretTagDAL.findManyTagsById(projectId, tagIds) : [];
+  if (tagIds.length !== tags.length) throw new BadRequestError({ message: "Tag not found" });
+
+  // now find any secret that needs to update its name
+  // same process as above
+  const nameUpdatedSecrets = inputSecrets.filter(({ newSecretName }) => Boolean(newSecretName));
+  const { keyName2BlindIndex: newKeyName2BlindIndex } = await fnSecretBlindIndexCheck({
+    inputSecrets: nameUpdatedSecrets,
+    folderId,
+    isNew: true,
+    blindIndexCfg,
+    secretDAL
+  });
+
+  const updatedSecrets = await secretDAL.transaction(async (tx) =>
+    fnSecretBulkUpdate({
+      folderId,
+      projectId,
+      tx,
+      inputSecrets: inputSecrets.map(({ secretName, newSecretName, ...el }) => ({
+        filter: { secretBlindIndex: keyName2BlindIndex[secretName], type: SecretType.Shared },
+        data: {
+          ...el,
+          folderId,
+          secretBlindIndex:
+            newSecretName && newKeyName2BlindIndex[newSecretName]
+              ? newKeyName2BlindIndex[newSecretName]
+              : keyName2BlindIndex[secretName],
+          algorithm: SecretEncryptionAlgo.AES_256_GCM,
+          keyEncoding: SecretKeyEncoding.UTF8
+        }
+      })),
+      secretDAL,
+      secretVersionDAL,
+      secretTagDAL,
+      secretVersionTagDAL
+    })
+  );
+
+  return updatedSecrets;
+};

backend/src/services/secret/secret-queue.ts

@@ -6,6 +6,10 @@ import { BadRequestError } from "@app/lib/errors";
 import { isSamePath } from "@app/lib/fn";
 import { logger } from "@app/lib/logger";
 import { QueueJobs, QueueName, TQueueServiceFactory } from "@app/queue";
+import { TSecretVersionDALFactory } from "@app/services/secret/secret-version-dal";
+import { TSecretVersionTagDALFactory } from "@app/services/secret/secret-version-tag-dal";
+import { TSecretBlindIndexDALFactory } from "@app/services/secret-blind-index/secret-blind-index-dal";
+import { TSecretTagDALFactory } from "@app/services/secret-tag/secret-tag-dal";
 
 import { TIntegrationDALFactory } from "../integration/integration-dal";
 import { TIntegrationAuthServiceFactory } from "../integration-auth/integration-auth-service";
@@ -25,16 +29,11 @@ import { TSecretDALFactory } from "./secret-dal";
 import { interpolateSecrets } from "./secret-fns";
 import { TCreateSecretReminderDTO, THandleReminderDTO, TRemoveSecretReminderDTO } from "./secret-types";
 
-import { TSecretBlindIndexDALFactory } from "@app/services/secret-blind-index/secret-blind-index-dal";
-import { TSecretTagDALFactory } from "@app/services/secret-tag/secret-tag-dal";
-import { TSecretVersionDALFactory } from "@app/services/secret/secret-version-dal";
-import { TSecretVersionTagDALFactory } from "@app/services/secret/secret-version-tag-dal";
-
 export type TSecretQueueFactory = ReturnType<typeof secretQueueFactory>;
 
 type TSecretQueueFactoryDep = {
   queueService: TQueueServiceFactory;
-  integrationDAL: Pick<TIntegrationDALFactory, "findByProjectIdV2">;
+  integrationDAL: Pick<TIntegrationDALFactory, "findByProjectIdV2" | "updateById">;
   projectBotService: Pick<TProjectBotServiceFactory, "getBotKey">;
   integrationAuthService: Pick<TIntegrationAuthServiceFactory, "getIntegrationAccessToken">;
   folderDAL: TSecretFolderDALFactory;
@@ -75,7 +74,7 @@ export const secretQueueFactory = ({
   secretVersionDAL,
   secretBlindIndexDAL,
   secretTagDAL,
-  secretVersionTagDAL,
+  secretVersionTagDAL
 }: TSecretQueueFactoryDep) => {
   const syncIntegrations = async (dto: TGetSecrets) => {
     await queueService.queue(QueueName.IntegrationSync, QueueJobs.IntegrationSync, dto, {
@@ -321,6 +320,7 @@ export const secretQueueFactory = ({
 
       await syncIntegrationSecrets({
         projectDAL,
+        integrationDAL,
         secretDAL,
         secretVersionDAL,
         secretBlindIndexDAL,

backend/src/services/secret/secret-service.ts

@@ -1,6 +1,6 @@
 import { ForbiddenError, subject } from "@casl/ability";
 
-import { SecretEncryptionAlgo, SecretKeyEncoding, SecretsSchema, SecretType, TableName } from "@app/db/schemas";
+import { SecretEncryptionAlgo, SecretKeyEncoding, SecretsSchema, SecretType } from "@app/db/schemas";
 import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service";
 import { ProjectPermissionActions, ProjectPermissionSub } from "@app/ee/services/permission/project-permission";
 import { TSecretSnapshotServiceFactory } from "@app/ee/services/secret-snapshot/secret-snapshot-service";
@@ -19,14 +19,9 @@ import { TSecretImportDALFactory } from "../secret-import/secret-import-dal";
 import { fnSecretsFromImports } from "../secret-import/secret-import-fns";
 import { TSecretTagDALFactory } from "../secret-tag/secret-tag-dal";
 import { TSecretDALFactory } from "./secret-dal";
-import { 
+import { decryptSecretRaw, fnSecretBlindIndexCheck, fnSecretBulkInsert, fnSecretBulkUpdate } from "./secret-fns";
-  decryptSecretRaw, 
-  fnSecretBlindIndexCheck,
-  fnSecretBulkInsert
-} from "./secret-fns";
 import { TSecretQueueFactory } from "./secret-queue";
 import {
-  TCreateManySecretsRawHelper,
   TCreateBulkSecretDTO,
   TCreateSecretDTO,
   TCreateSecretRawDTO,
@@ -35,7 +30,6 @@ import {
   TDeleteSecretRawDTO,
   TFnSecretBlindIndexCheckV2,
   TFnSecretBulkDelete,
-  TFnSecretBulkUpdate,
   TGetASecretDTO,
   TGetASecretRawDTO,
   TGetSecretsDTO,
@@ -98,50 +92,6 @@ export const secretServiceFactory = ({
     return secretBlindIndex;
   };
 
-  const fnSecretBulkUpdate = async ({ tx, inputSecrets, folderId, projectId }: TFnSecretBulkUpdate) => {
-    const newSecrets = await secretDAL.bulkUpdate(
-      inputSecrets.map(({ filter, data: { tags, ...data } }) => ({
-        filter: { ...filter, folderId },
-        data
-      })),
-      tx
-    );
-    const secretVersions = await secretVersionDAL.insertMany(
-      newSecrets.map(({ id, createdAt, updatedAt, ...el }) => ({
-        ...el,
-        secretId: id
-      })),
-      tx
-    );
-    const secsUpdatedTag = inputSecrets.flatMap(({ data: { tags } }, i) =>
-      tags !== undefined ? { tags, secretId: newSecrets[i].id } : []
-    );
-    if (secsUpdatedTag.length) {
-      await secretTagDAL.deleteTagsManySecret(
-        projectId,
-        secsUpdatedTag.map(({ secretId }) => secretId),
-        tx
-      );
-      const newSecretTags = secsUpdatedTag.flatMap(({ tags: secretTags = [], secretId }) =>
-        secretTags.map((tag) => ({
-          [`${TableName.SecretTag}Id` as const]: tag,
-          [`${TableName.Secret}Id` as const]: secretId
-        }))
-      );
-      if (newSecretTags.length) {
-        const secTags = await secretTagDAL.saveTagsToSecret(newSecretTags, tx);
-        const secVersionsGroupBySecId = groupBy(secretVersions, (i) => i.secretId);
-        const newSecretVersionTags = secTags.flatMap(({ secretsId, secret_tagsId }) => ({
-          [`${TableName.SecretVersion}Id` as const]: secVersionsGroupBySecId[secretsId][0].id,
-          [`${TableName.SecretTag}Id` as const]: secret_tagsId
-        }));
-        await secretVersionTagDAL.insertMany(newSecretVersionTags, tx);
-      }
-    }
-
-    return newSecrets.map((secret) => ({ ...secret, _id: secret.id }));
-  };
-
   const fnSecretBulkDelete = async ({ folderId, inputSecrets, tx, actorId }: TFnSecretBulkDelete) => {
     const deletedSecrets = await secretDAL.deleteMany(
       inputSecrets.map(({ type, secretBlindIndex }) => ({
@@ -372,6 +322,10 @@ export const secretServiceFactory = ({
             }
           }
         ],
+        secretDAL,
+        secretVersionDAL,
+        secretTagDAL,
+        secretVersionTagDAL,
         tx
       })
     );
@@ -598,7 +552,7 @@ export const secretServiceFactory = ({
     const folderId = folder.id;
 
     const blindIndexCfg = await secretBlindIndexDAL.findOne({ projectId });
-    if (!blindIndexCfg) throw new BadRequestError({ message: "Blind index not found", name: "Update secret" });
+    if (!blindIndexCfg) throw new BadRequestError({ message: "Blind index not found", name: "Create secret" });
 
     const { keyName2BlindIndex } = await fnSecretBlindIndexCheck({
       inputSecrets,
@@ -656,7 +610,7 @@ export const secretServiceFactory = ({
     await projectDAL.checkProjectUpgradeStatus(projectId);
 
     const folder = await folderDAL.findBySecretPath(projectId, environment, path);
-    if (!folder) throw new BadRequestError({ message: "Folder not  found", name: "Create secret" });
+    if (!folder) throw new BadRequestError({ message: "Folder not  found", name: "Update secret" });
     const folderId = folder.id;
 
     const blindIndexCfg = await secretBlindIndexDAL.findOne({ projectId });
@@ -703,7 +657,11 @@ export const secretServiceFactory = ({
             algorithm: SecretEncryptionAlgo.AES_256_GCM,
             keyEncoding: SecretKeyEncoding.UTF8
           }
-        }))
+        })),
+        secretDAL,
+        secretVersionDAL,
+        secretTagDAL,
+        secretVersionTagDAL
       })
     );
 
@@ -826,98 +784,6 @@ export const secretServiceFactory = ({
     });
     return decryptSecretRaw(secret, botKey);
   };
-  
-  const createManySecretsRawHelper = async ({ // TODO: place on top
-    projectId,
-    environment,
-    path,
-    secrets,
-    userId
-  }: TCreateManySecretsRawHelper) => {
-    const botKey = await projectBotService.getBotKey(projectId);
-    if (!botKey) throw new BadRequestError({ message: "Project bot not found", name: "bot_not_found_error" });
-
-    await projectDAL.checkProjectUpgradeStatus(projectId);
-
-    const folder = await folderDAL.findBySecretPath(projectId, environment, path);
-    if (!folder) throw new BadRequestError({ message: "Folder not  found", name: "Create secret" });
-    const folderId = folder.id;
-
-    const blindIndexCfg = await secretBlindIndexDAL.findOne({ projectId });
-    if (!blindIndexCfg) throw new BadRequestError({ message: "Blind index not found", name: "Update secret" });
-
-    const inputSecrets = await Promise.all(
-      secrets.map(async (secret) => {
-        const secretKeyEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretName, botKey);
-        const secretValueEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretValue || "", botKey);
-        const secretCommentEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretComment || "", botKey);
-
-        if (secret.type === SecretType.Personal) {
-          const sharedExist = await secretDAL.findOne({
-            secretBlindIndex: keyName2BlindIndex[secret.secretName],
-            folderId,
-            type: SecretType.Shared
-          });
-
-          if (!sharedExist)
-            throw new BadRequestError({
-              message: "Failed to create personal secret override for no corresponding shared secret"
-            });
-          
-          if (!userId) throw new BadRequestError({ message: "Missing user id for personal secret" });
-        }
-
-        const tags = secret.tags ? await secretTagDAL.findManyTagsById(projectId, secret.tags) : [];
-        if ((secret.tags || []).length !== tags.length) throw new BadRequestError({ message: "Tag not found" });
-
-        return ({
-          type: secret.type,
-          userId: secret.type === SecretType.Personal ? userId : null,
-          secretName: secret.secretName,
-          secretKeyCiphertext: secretKeyEncrypted.ciphertext,
-          secretKeyIV: secretKeyEncrypted.iv,
-          secretKeyTag: secretKeyEncrypted.tag,
-          secretValueCiphertext: secretValueEncrypted.ciphertext,
-          secretValueIV: secretValueEncrypted.iv,
-          secretValueTag: secretValueEncrypted.tag,
-          secretCommentCiphertext: secretCommentEncrypted.ciphertext,
-          secretCommentIV: secretCommentEncrypted.iv,
-          secretCommentTag: secretCommentEncrypted.tag,
-          skipMultilineEncoding: secret.skipMultilineEncoding,
-          tags: secret.tags
-        });
-      })
-    );
-
-    // insert operation
-    const { keyName2BlindIndex } = await fnSecretBlindIndexCheck({
-      inputSecrets,
-      folderId,
-      isNew: true,
-      blindIndexCfg,
-      secretDAL
-    });
-
-    const newSecrets = await secretDAL.transaction(async (tx) =>
-      fnSecretBulkInsert({
-        inputSecrets: inputSecrets.map(({ secretName, ...el }) => ({
-          ...el,
-          version: 0,
-          secretBlindIndex: keyName2BlindIndex[secretName],
-          algorithm: SecretEncryptionAlgo.AES_256_GCM,
-          keyEncoding: SecretKeyEncoding.UTF8
-        })),
-        folderId,
-        secretDAL,
-        secretVersionDAL,
-        secretTagDAL,
-        secretVersionTagDAL,
-        tx
-      })
-    );
-
-    return newSecrets;
-  }
 
   const createSecretRaw = async ({
     secretName,

backend/src/services/secret/secret-types.ts

@@ -4,11 +4,11 @@ import { SecretType, TSecretBlindIndexes, TSecrets, TSecretsInsert, TSecretsUpda
 import { TProjectPermission } from "@app/lib/types";
 import { TProjectDALFactory } from "@app/services/project/project-dal";
 import { TSecretDALFactory } from "@app/services/secret/secret-dal";
-import { TSecretBlindIndexDALFactory } from "@app/services/secret-blind-index/secret-blind-index-dal";
-import { TSecretTagDALFactory } from "@app/services/secret-tag/secret-tag-dal";
-import { TSecretFolderDALFactory } from "@app/services/secret-folder/secret-folder-dal";
 import { TSecretVersionDALFactory } from "@app/services/secret/secret-version-dal";
 import { TSecretVersionTagDALFactory } from "@app/services/secret/secret-version-tag-dal";
+import { TSecretBlindIndexDALFactory } from "@app/services/secret-blind-index/secret-blind-index-dal";
+import { TSecretFolderDALFactory } from "@app/services/secret-folder/secret-folder-dal";
+import { TSecretTagDALFactory } from "@app/services/secret-tag/secret-tag-dal";
 
 type TPartialSecret = Pick<TSecrets, "id" | "secretReminderRepeatDays" | "secretReminderNote">;
 
@@ -198,6 +198,10 @@ export type TFnSecretBulkUpdate = {
   folderId: string;
   projectId: string;
   inputSecrets: { filter: Partial<TSecrets>; data: TSecretsUpdate & { tags?: string[] } }[];
+  secretDAL: Pick<TSecretDALFactory, "bulkUpdate">;
+  secretVersionDAL: Pick<TSecretVersionDALFactory, "insertMany">;
+  secretTagDAL: Pick<TSecretTagDALFactory, "saveTagsToSecret" | "deleteTagsManySecret">;
+  secretVersionTagDAL: Pick<TSecretVersionTagDALFactory, "insertMany">;
   tx?: Knex;
 };
 
@@ -257,7 +261,7 @@ export type TCreateManySecretsRawHelper = {
     tags?: string[];
     metadata?: {
       source?: string;
-    }
+    };
   }[];
   userId?: string; // only relevant for personal secret(s)
   botKey: string;
@@ -268,4 +272,33 @@ export type TCreateManySecretsRawHelper = {
   secretTagDAL: TSecretTagDALFactory;
   secretVersionTagDAL: TSecretVersionTagDALFactory;
   folderDAL: TSecretFolderDALFactory;
-}
+};
+
+export type TUpdateManySecretsRawHelper = {
+  projectId: string;
+  environment: string;
+  path: string;
+  secrets: {
+    secretName: string;
+    newSecretName?: string;
+    secretValue: string;
+    type: SecretType;
+    secretComment?: string;
+    skipMultilineEncoding?: boolean;
+    secretReminderRepeatDays?: number | null;
+    secretReminderNote?: string | null;
+    tags?: string[];
+    metadata?: {
+      source?: string;
+    };
+  }[];
+  userId?: string; // only relevant for personal secret(s)
+  botKey: string;
+  projectDAL: TProjectDALFactory;
+  secretDAL: TSecretDALFactory;
+  secretVersionDAL: TSecretVersionDALFactory;
+  secretBlindIndexDAL: TSecretBlindIndexDALFactory;
+  secretTagDAL: TSecretTagDALFactory;
+  secretVersionTagDAL: TSecretVersionTagDALFactory;
+  folderDAL: TSecretFolderDALFactory;
+};