MVP preliminary idea for initial sync behavior

backend/src/services/secret/secret-fns.ts

@@ -1,12 +1,31 @@
 /* eslint-disable no-await-in-loop */
 import path from "path";
 
-import { SecretKeyEncoding, TSecretBlindIndexes, TSecrets } from "@app/db/schemas";
+import { 
+  SecretKeyEncoding, 
+  TSecretBlindIndexes, 
+  TSecrets,
+  SecretEncryptionAlgo,
+  SecretType,
+  TableName
+} from "@app/db/schemas";
 import { getConfig } from "@app/lib/config/env";
-import { buildSecretBlindIndexFromName, decryptSymmetric128BitHexKeyUTF8 } from "@app/lib/crypto";
+import { 
+  buildSecretBlindIndexFromName, 
+  encryptSymmetric128BitHexKeyUTF8,
+  decryptSymmetric128BitHexKeyUTF8 
+} from "@app/lib/crypto";
 
 import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal";
 import { TSecretDALFactory } from "./secret-dal";
+import { 
+  TCreateManySecretsRawHelper,
+  TFnSecretBulkInsert,
+  TFnSecretBlindIndexCheck
+} from "./secret-types";
+
+import { groupBy, unique } from "@app/lib/fn";
+import { BadRequestError } from "@app/lib/errors";
 
 export const generateSecretBlindIndexBySalt = async (secretName: string, secretBlindIndexDoc: TSecretBlindIndexes) => {
   const appCfg = getConfig();
@@ -228,3 +247,201 @@ export const decryptSecretRaw = (secret: TSecrets & { workspace: string; environ
     user: secret.userId
   };
 };
+
+/**
+ * Checks and handles secrets using a blind index method.
+ * The function generates mappings between secret names and their blind indexes, validates user IDs for personal secrets, and retrieves secrets from the database based on their blind indexes.
+ * For new secrets (isNew = true), it ensures they don't already exist in the database.
+ * For existing secrets, it verifies their presence in the database.
+ * If discrepancies are found, errors are thrown. The function returns mappings and the fetched secrets.
+ */
+export const fnSecretBlindIndexCheck = async ({
+  inputSecrets,
+  folderId,
+  isNew,
+  userId,
+  blindIndexCfg,
+  secretDAL
+}: TFnSecretBlindIndexCheck) => {
+  const blindIndex2KeyName: Record<string, string> = {}; // used at audit log point
+  const keyName2BlindIndex = await Promise.all(
+    inputSecrets.map(({ secretName }) => generateSecretBlindIndexBySalt(secretName, blindIndexCfg))
+  ).then((blindIndexes) =>
+    blindIndexes.reduce<Record<string, string>>((prev, curr, i) => {
+      // eslint-disable-next-line
+      prev[inputSecrets[i].secretName] = curr;
+      blindIndex2KeyName[curr] = inputSecrets[i].secretName;
+      return prev;
+    }, {})
+  );
+
+  if (inputSecrets.some(({ type }) => type === SecretType.Personal) && !userId) {
+    throw new BadRequestError({ message: "Missing user id for personal secret" });
+  }
+
+  const secrets = await secretDAL.findByBlindIndexes(
+    folderId,
+    inputSecrets.map(({ secretName, type }) => ({
+      blindIndex: keyName2BlindIndex[secretName],
+      type: type || SecretType.Shared
+    })),
+    userId
+  );
+
+  if (isNew) {
+    if (secrets.length) throw new BadRequestError({ message: "Secret already exist" });
+  } else {
+    const secretKeysInDB = unique(secrets, (el) => el.secretBlindIndex as string).map(
+      (el) => blindIndex2KeyName[el.secretBlindIndex as string]
+    );
+    const hasUnknownSecretsProvided = secretKeysInDB.length !== inputSecrets.length;
+    if (hasUnknownSecretsProvided) {
+      const keysMissingInDB = Object.keys(keyName2BlindIndex).filter((key) => !secretKeysInDB.includes(key));
+      throw new BadRequestError({
+        message: `Secret not found: blind index ${keysMissingInDB.join(",")}`
+      });
+    }
+  }
+
+  return { blindIndex2KeyName, keyName2BlindIndex, secrets };
+};
+
+// these functions are special functions shared by a couple of resources
+// used by secret approval, rotation or anywhere in which secret needs to modified
+export const fnSecretBulkInsert = async ({ // TODO: Pick types here 
+  folderId, 
+  inputSecrets, 
+  secretDAL,
+  secretVersionDAL,
+  secretTagDAL,
+  secretVersionTagDAL,
+  tx 
+}: TFnSecretBulkInsert) => {
+  const newSecrets = await secretDAL.insertMany(
+    inputSecrets.map(({ tags, ...el }) => ({ ...el, folderId })),
+    tx
+  );
+  const newSecretGroupByBlindIndex = groupBy(newSecrets, (item) => item.secretBlindIndex as string);
+  const newSecretTags = inputSecrets.flatMap(({ tags: secretTags = [], secretBlindIndex }) =>
+    secretTags.map((tag) => ({
+      [`${TableName.SecretTag}Id` as const]: tag,
+      [`${TableName.Secret}Id` as const]: newSecretGroupByBlindIndex[secretBlindIndex as string][0].id
+    }))
+  );
+  const secretVersions = await secretVersionDAL.insertMany(
+    inputSecrets.map(({ tags, ...el }) => ({
+      ...el,
+      folderId,
+      secretId: newSecretGroupByBlindIndex[el.secretBlindIndex as string][0].id
+    })),
+    tx
+  );
+  if (newSecretTags.length) {
+    const secTags = await secretTagDAL.saveTagsToSecret(newSecretTags, tx);
+    const secVersionsGroupBySecId = groupBy(secretVersions, (i) => i.secretId);
+    const newSecretVersionTags = secTags.flatMap(({ secretsId, secret_tagsId }) => ({
+      [`${TableName.SecretVersion}Id` as const]: secVersionsGroupBySecId[secretsId][0].id,
+      [`${TableName.SecretTag}Id` as const]: secret_tagsId
+    }));
+    await secretVersionTagDAL.insertMany(newSecretVersionTags, tx);
+  }
+
+  return newSecrets.map((secret) => ({ ...secret, _id: secret.id }));
+};
+
+export const createManySecretsRawHelper = async ({ // TODO: place on top
+  projectId,
+  environment,
+  path,
+  secrets,
+  userId,
+  botKey, // TODO: consider getting botKey inside this fn
+  projectDAL,
+  secretDAL,
+  secretVersionDAL,
+  secretBlindIndexDAL,
+  secretTagDAL,
+  secretVersionTagDAL,
+  folderDAL
+}: TCreateManySecretsRawHelper) => {
+await projectDAL.checkProjectUpgradeStatus(projectId);
+
+const folder = await folderDAL.findBySecretPath(projectId, environment, path);
+if (!folder) throw new BadRequestError({ message: "Folder not  found", name: "Create secret" });
+const folderId = folder.id;
+
+const blindIndexCfg = await secretBlindIndexDAL.findOne({ projectId });
+if (!blindIndexCfg) throw new BadRequestError({ message: "Blind index not found", name: "Update secret" });
+
+const inputSecrets = await Promise.all(
+  secrets.map(async (secret) => {
+    const secretKeyEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretName, botKey);
+    const secretValueEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretValue || "", botKey);
+    const secretCommentEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretComment || "", botKey);
+
+    if (secret.type === SecretType.Personal) {
+      const sharedExist = await secretDAL.findOne({
+        secretBlindIndex: keyName2BlindIndex[secret.secretName],
+        folderId,
+        type: SecretType.Shared
+      });
+
+      if (!sharedExist)
+        throw new BadRequestError({
+          message: "Failed to create personal secret override for no corresponding shared secret"
+        });
+      
+      if (!userId) throw new BadRequestError({ message: "Missing user id for personal secret" });
+    }
+
+    const tags = secret.tags ? await secretTagDAL.findManyTagsById(projectId, secret.tags) : [];
+    if ((secret.tags || []).length !== tags.length) throw new BadRequestError({ message: "Tag not found" });
+
+    return ({
+      type: secret.type,
+      userId: secret.type === SecretType.Personal ? userId : null,
+      secretName: secret.secretName,
+      secretKeyCiphertext: secretKeyEncrypted.ciphertext,
+      secretKeyIV: secretKeyEncrypted.iv,
+      secretKeyTag: secretKeyEncrypted.tag,
+      secretValueCiphertext: secretValueEncrypted.ciphertext,
+      secretValueIV: secretValueEncrypted.iv,
+      secretValueTag: secretValueEncrypted.tag,
+      secretCommentCiphertext: secretCommentEncrypted.ciphertext,
+      secretCommentIV: secretCommentEncrypted.iv,
+      secretCommentTag: secretCommentEncrypted.tag,
+      skipMultilineEncoding: secret.skipMultilineEncoding,
+      tags: secret.tags
+    });
+  })
+);
+
+  // insert operation
+  const { keyName2BlindIndex } = await fnSecretBlindIndexCheck({
+    inputSecrets,
+    folderId,
+    isNew: true,
+    blindIndexCfg,
+    secretDAL
+  });
+
+  const newSecrets = await secretDAL.transaction(async (tx) =>
+    fnSecretBulkInsert({
+      inputSecrets: inputSecrets.map(({ secretName, ...el }) => ({
+        ...el,
+        version: 0,
+        secretBlindIndex: keyName2BlindIndex[secretName],
+        algorithm: SecretEncryptionAlgo.AES_256_GCM,
+        keyEncoding: SecretKeyEncoding.UTF8
+      })),
+      folderId,
+      secretDAL,
+      secretVersionDAL,
+      secretTagDAL,
+      secretVersionTagDAL,
+      tx
+    })
+  );
+
+  return newSecrets;
+}