feat: created base for secret v2 bridge and plugged it to secret-router

backend/src/server/routes/index.ts

@@ -182,6 +182,10 @@ import { registerSecretScannerGhApp } from "../plugins/secret-scanner";
 import { registerV1Routes } from "./v1";
 import { registerV2Routes } from "./v2";
 import { registerV3Routes } from "./v3";
+import { secretV2BridgeDALFactory } from "@app/services/secret-v2-bridge/secret-v2-bridge-dal";
+import { secretVersionV2BridgeDALFactory } from "@app/services/secret-v2-bridge/secret-version-dal";
+import { secretVersionV2TagBridgeDALFactory } from "@app/services/secret-v2-bridge/secret-version-tag-dal";
+import { secretV2BridgeServiceFactory } from "@app/services/secret-v2-bridge/secret-v2-bridge-service";
 
 export const registerRoutes = async (
   server: FastifyZodProvider,
@@ -229,6 +233,10 @@ export const registerRoutes = async (
   const secretVersionTagDAL = secretVersionTagDALFactory(db);
   const secretBlindIndexDAL = secretBlindIndexDALFactory(db);
 
+  const secretV2BridgeDAL = secretV2BridgeDALFactory(db);
+  const secretVersionV2BridgeDAL = secretVersionV2BridgeDALFactory(db);
+  const secretVersionV2TagBridgeDAL = secretVersionV2TagBridgeDALFactory(db);
+
   const integrationDAL = integrationDALFactory(db);
   const integrationAuthDAL = integrationAuthDALFactory(db);
   const webhookDAL = webhookDALFactory(db);
@@ -719,6 +727,22 @@ export const registerRoutes = async (
     secretDAL,
     secretBlindIndexDAL
   });
+
+  const secretV2BridgeService = secretV2BridgeServiceFactory({
+    folderDAL,
+    secretVersionDAL: secretVersionV2BridgeDAL,
+    secretQueueService,
+    secretDAL: secretV2BridgeDAL,
+    permissionService,
+    secretVersionTagDAL: secretVersionV2TagBridgeDAL,
+    secretTagDAL,
+    projectEnvDAL,
+    secretImportDAL,
+    secretApprovalRequestDAL,
+    secretApprovalPolicyService,
+    secretApprovalRequestSecretDAL
+  });
+
   const secretService = secretServiceFactory({
     folderDAL,
     secretVersionDAL,
@@ -735,7 +759,8 @@ export const registerRoutes = async (
     projectBotService,
     secretApprovalPolicyService,
     secretApprovalRequestDAL,
-    secretApprovalRequestSecretDAL
+    secretApprovalRequestSecretDAL,
+    secretV2BridgeService
   });
 
   const secretSharingService = secretSharingServiceFactory({

backend/src/server/routes/sanitizedSchemas.ts

@@ -62,7 +62,7 @@ export const secretRawSchema = z.object({
   version: z.number(),
   type: z.string(),
   secretKey: z.string(),
-  secretValue: z.string(),
+  secretValue: z.string().optional(),
   secretComment: z.string().optional(),
   secretReminderNote: z.string().nullable().optional(),
   secretReminderRepeatDays: z.number().nullable().optional(),

backend/src/services/project-bot/project-bot-fns.ts

@@ -22,6 +22,10 @@ export const getBotKeyFnFactory = (
     const project = await projectDAL.findById(projectId);
     if (!project) throw new BadRequestError({ message: "Project not found during bot lookup." });
 
+    if (project.version === 3) {
+      return { project, shouldUseSecretV2Bridge: true };
+    }
+
     const bot = await projectBotDAL.findOne({ projectId: project.id });
 
     if (!bot) throw new BadRequestError({ message: "Failed to find bot key", name: "bot_not_found_error" });
@@ -31,12 +35,13 @@ export const getBotKeyFnFactory = (
 
     const botPrivateKey = getBotPrivateKey({ bot });
 
-    return decryptAsymmetric({
+    const botKey = decryptAsymmetric({
       ciphertext: bot.encryptedProjectKey,
       privateKey: botPrivateKey,
       nonce: bot.encryptedProjectKeyNonce,
       publicKey: bot.sender.publicKey
     });
+    return { botKey, project, shouldUseSecretV2Bridge: false };
   };
 
   return getBotKeyFn;

backend/src/services/secret-import/secret-import-fns.ts

@@ -1,8 +1,9 @@
-import { SecretType, TSecretImports, TSecrets } from "@app/db/schemas";
+import { SecretType, TSecretImports, TSecrets, TSecretsV2 } from "@app/db/schemas";
 import { groupBy } from "@app/lib/fn";
 
 import { TSecretDALFactory } from "../secret/secret-dal";
 import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal";
+import { TSecretV2BridgeDALFactory } from "../secret-v2-bridge/secret-v2-bridge-dal";
 import { TSecretImportDALFactory } from "./secret-import-dal";
 
 type TSecretImportSecrets = {
@@ -18,6 +19,28 @@ type TSecretImportSecrets = {
   secrets: (TSecrets & { workspace: string; environment: string; _id: string })[];
 };
 
+type TSecretImportSecretsV2 = {
+  secretPath: string;
+  environment: string;
+  environmentInfo: {
+    id: string;
+    slug: string;
+    name: string;
+  };
+  folderId: string | undefined;
+  importFolderId: string;
+  secrets: (TSecretsV2 & {
+    workspace: string;
+    environment: string;
+    _id: string;
+    secretKey: string;
+    // akhilmhdh: yes i know you can put ?.
+    // But for somereason ts consider ? and undefined explicit as different just ts things
+    secretValue: string | undefined;
+    secretComment: string | undefined;
+  })[];
+};
+
 const LEVEL_BREAK = 10;
 const getImportUniqKey = (envSlug: string, path: string) => `${envSlug}=${path}`;
 export const fnSecretsFromImports = async ({
@@ -115,3 +138,101 @@ export const fnSecretsFromImports = async ({
 
   return secrets;
 };
+
+export const fnSecretsV2FromImports = async ({
+  allowedImports: possibleCyclicImports,
+  folderDAL,
+  secretDAL,
+  secretImportDAL,
+  depth = 0,
+  cyclicDetector = new Set()
+}: {
+  allowedImports: (Omit<TSecretImports, "importEnv"> & {
+    importEnv: { id: string; slug: string; name: string };
+  })[];
+  folderDAL: Pick<TSecretFolderDALFactory, "findByManySecretPath">;
+  secretDAL: Pick<TSecretV2BridgeDALFactory, "find">;
+  secretImportDAL: Pick<TSecretImportDALFactory, "findByFolderIds">;
+  depth?: number;
+  cyclicDetector?: Set<string>;
+}) => {
+  // avoid going more than a depth
+  if (depth >= LEVEL_BREAK) return [];
+
+  const allowedImports = possibleCyclicImports.filter(
+    ({ importPath, importEnv }) => !cyclicDetector.has(getImportUniqKey(importEnv.slug, importPath))
+  );
+
+  const importedFolders = (
+    await folderDAL.findByManySecretPath(
+      allowedImports.map(({ importEnv, importPath }) => ({
+        envId: importEnv.id,
+        secretPath: importPath
+      }))
+    )
+  ).filter(Boolean); // remove undefined ones
+  if (!importedFolders.length) {
+    return [];
+  }
+
+  const importedFolderIds = importedFolders.map((el) => el?.id) as string[];
+  const importedFolderGroupBySourceImport = groupBy(importedFolders, (i) => `${i?.envId}-${i?.path}`);
+  const importedSecrets = await secretDAL.find(
+    {
+      $in: { folderId: importedFolderIds },
+      type: SecretType.Shared
+    },
+    {
+      sort: [["id", "asc"]]
+    }
+  );
+
+  const importedSecretsGroupByFolderId = groupBy(importedSecrets, (i) => i.folderId);
+
+  allowedImports.forEach(({ importPath, importEnv }) => {
+    cyclicDetector.add(getImportUniqKey(importEnv.slug, importPath));
+  });
+  // now we need to check recursively deeper imports made inside other imports
+  // we go level wise meaning we take all imports of a tree level and then go deeper ones level by level
+  const deeperImports = await secretImportDAL.findByFolderIds(importedFolderIds);
+  let secretsFromDeeperImports: TSecretImportSecretsV2[] = [];
+  if (deeperImports.length) {
+    secretsFromDeeperImports = await fnSecretsV2FromImports({
+      allowedImports: deeperImports.filter(({ isReplication }) => !isReplication),
+      secretImportDAL,
+      folderDAL,
+      secretDAL,
+      depth: depth + 1,
+      cyclicDetector
+    });
+  }
+  const secretsFromdeeperImportGroupedByFolderId = groupBy(secretsFromDeeperImports, (i) => i.importFolderId);
+
+  const secrets = allowedImports.map(({ importPath, importEnv, id, folderId }, i) => {
+    const sourceImportFolder = importedFolderGroupBySourceImport[`${importEnv.id}-${importPath}`][0];
+    const folderDeeperImportSecrets =
+      secretsFromdeeperImportGroupedByFolderId?.[sourceImportFolder?.id || ""]?.[0]?.secrets || [];
+
+    return {
+      secretPath: importPath,
+      environment: importEnv.slug,
+      environmentInfo: importEnv,
+      folderId: importedFolders?.[i]?.id,
+      id,
+      importFolderId: folderId,
+      secrets: (importedSecretsGroupByFolderId?.[importedFolders?.[i]?.id as string] || [])
+        .map((item) => ({
+          ...item,
+          secretKey: item.key,
+          secretValue: item.encryptedValue?.toString(),
+          secretComment: item.encryptedComment?.toString(),
+          environment: importEnv.slug,
+          workspace: "", // This field should not be used, it's only here to keep the older Python SDK versions backwards compatible with the new Postgres backend.
+          _id: item.id // The old Python SDK depends on the _id field being returned. We return this to keep the older Python SDK versions backwards compatible with the new Postgres backend.
+        }))
+        .concat(folderDeeperImportSecrets)
+    };
+  });
+
+  return secrets;
+};

backend/src/services/secret-tag/secret-tag-dal.ts

@@ -10,6 +10,7 @@ export type TSecretTagDALFactory = ReturnType<typeof secretTagDALFactory>;
 export const secretTagDALFactory = (db: TDbClient) => {
   const secretTagOrm = ormify(db, TableName.SecretTag);
   const secretJnTagOrm = ormify(db, TableName.JnSecretTag);
+  const secretV2JnTagOrm = ormify(db, TableName.SecretV2JnTag);
 
   const findManyTagsById = async (projectId: string, ids: string[], tx?: Knex) => {
     try {
@@ -38,6 +39,8 @@ export const secretTagDALFactory = (db: TDbClient) => {
     ...secretTagOrm,
     saveTagsToSecret: secretJnTagOrm.insertMany,
     deleteTagsToSecret: secretJnTagOrm.delete,
+    saveTagsToSecretV2: secretV2JnTagOrm.insertMany,
+    deleteTagsToSecretV2: secretV2JnTagOrm.delete,
     deleteTagsManySecret,
     findManyTagsById
   };

backend/src/services/secret-v2-bridge/secret-v2-bridge-dal.ts

@@ -0,0 +1,393 @@
+import { Knex } from "knex";
+import { validate as uuidValidate } from "uuid";
+
+import { TDbClient } from "@app/db";
+import { SecretsV2Schema, SecretType, TableName, TSecretsV2, TSecretsV2Update } from "@app/db/schemas";
+import { BadRequestError, DatabaseError } from "@app/lib/errors";
+import { ormify, selectAllTableCols, sqlNestRelationships } from "@app/lib/knex";
+
+export type TSecretV2BridgeDALFactory = ReturnType<typeof secretV2BridgeDALFactory>;
+
+export const secretV2BridgeDALFactory = (db: TDbClient) => {
+  const secretOrm = ormify(db, TableName.SecretV2);
+
+  const update = async (filter: Partial<TSecretsV2>, data: Omit<TSecretsV2Update, "version">, tx?: Knex) => {
+    try {
+      const sec = await (tx || db)(TableName.SecretV2)
+        .where(filter)
+        .update(data)
+        .increment("version", 1)
+        .returning("*");
+      return sec;
+    } catch (error) {
+      throw new DatabaseError({ error, name: "update secret" });
+    }
+  };
+
+  const bulkUpdate = async (
+    data: Array<{ filter: Partial<TSecretsV2>; data: TSecretsV2Update }>,
+
+    tx?: Knex
+  ) => {
+    try {
+      const secs = await Promise.all(
+        data.map(async ({ filter, data: updateData }) => {
+          const [doc] = await (tx || db)(TableName.SecretV2)
+            .where(filter)
+            .update(updateData)
+            .increment("version", 1)
+            .returning("*");
+          if (!doc) throw new BadRequestError({ message: "Failed to update document" });
+          return doc;
+        })
+      );
+      return secs;
+    } catch (error) {
+      throw new DatabaseError({ error, name: "bulk update secret" });
+    }
+  };
+
+  const bulkUpdateNoVersionIncrement = async (data: TSecretsV2[], tx?: Knex) => {
+    try {
+      const existingSecrets = await secretOrm.find(
+        {
+          $in: {
+            id: data.map((el) => el.id)
+          }
+        },
+        { tx }
+      );
+
+      if (existingSecrets.length !== data.length) {
+        throw new BadRequestError({ message: "Some of the secrets do not exist" });
+      }
+
+      if (data.length === 0) return [];
+
+      const updatedSecrets = await (tx || db)(TableName.SecretV2)
+        .insert(data)
+        .onConflict("id") // this will cause a conflict then merge the data
+        .merge() // Merge the data with the existing data
+        .returning("*");
+
+      return updatedSecrets;
+    } catch (error) {
+      throw new DatabaseError({ error, name: "bulk update secret" });
+    }
+  };
+
+  const deleteMany = async (
+    data: Array<{ key: string; type: SecretType }>,
+    folderId: string,
+    userId: string,
+    tx?: Knex
+  ) => {
+    try {
+      const deletedSecrets = await (tx || db)(TableName.SecretV2)
+        .where({ folderId })
+        .where((bd) => {
+          data.forEach((el) => {
+            void bd.orWhere({
+              key: el.key,
+              type: el.type,
+              ...(el.type === SecretType.Personal ? { userId } : {})
+            });
+            // if shared is getting deleted then personal ones also should be deleted
+            if (el.type === SecretType.Shared) {
+              void bd.orWhere({
+                key: el.key,
+                type: SecretType.Personal
+              });
+            }
+          });
+        })
+        .delete()
+        .returning("*");
+      return deletedSecrets;
+    } catch (error) {
+      throw new DatabaseError({ error, name: "delete many secret" });
+    }
+  };
+
+  const findByFolderId = async (folderId: string, userId?: string, tx?: Knex) => {
+    try {
+      // check if not uui then userId id is null (corner case because service token's ID is not UUI in effort to keep backwards compatibility from mongo)
+      if (userId && !uuidValidate(userId)) {
+        // eslint-disable-next-line
+        userId = undefined;
+      }
+
+      const secs = await (tx || db.replicaNode())(TableName.SecretV2)
+        .where({ folderId })
+        .where((bd) => {
+          void bd.whereNull("userId").orWhere({ userId: userId || null });
+        })
+        .leftJoin(
+          TableName.SecretV2JnTag,
+          `${TableName.SecretV2}.id`,
+          `${TableName.SecretV2JnTag}.${TableName.SecretV2}Id`
+        )
+        .leftJoin(
+          TableName.SecretTag,
+          `${TableName.SecretV2JnTag}.${TableName.SecretTag}Id`,
+          `${TableName.SecretTag}.id`
+        )
+        .select(selectAllTableCols(TableName.SecretV2))
+        .select(db.ref("id").withSchema(TableName.SecretTag).as("tagId"))
+        .select(db.ref("color").withSchema(TableName.SecretTag).as("tagColor"))
+        .select(db.ref("slug").withSchema(TableName.SecretTag).as("tagSlug"))
+        .select(db.ref("name").withSchema(TableName.SecretTag).as("tagName"))
+        .orderBy("id", "asc");
+
+      const data = sqlNestRelationships({
+        data: secs,
+        key: "id",
+        parentMapper: (el) => ({ _id: el.id, ...SecretsV2Schema.parse(el) }),
+        childrenMapper: [
+          {
+            key: "tagId",
+            label: "tags" as const,
+            mapper: ({ tagId: id, tagColor: color, tagSlug: slug, tagName: name }) => ({
+              id,
+              color,
+              slug,
+              name
+            })
+          }
+        ]
+      });
+      return data;
+    } catch (error) {
+      throw new DatabaseError({ error, name: "get all secret" });
+    }
+  };
+
+  const getSecretTags = async (secretId: string, tx?: Knex) => {
+    try {
+      const tags = await (tx || db.replicaNode())(TableName.SecretV2JnTag)
+        .join(TableName.SecretTag, `${TableName.SecretV2JnTag}.${TableName.SecretTag}Id`, `${TableName.SecretTag}.id`)
+        .where({ [`${TableName.SecretV2}Id` as const]: secretId })
+        .select(db.ref("id").withSchema(TableName.SecretTag).as("tagId"))
+        .select(db.ref("color").withSchema(TableName.SecretTag).as("tagColor"))
+        .select(db.ref("slug").withSchema(TableName.SecretTag).as("tagSlug"))
+        .select(db.ref("name").withSchema(TableName.SecretTag).as("tagName"));
+
+      return tags.map((el) => ({
+        id: el.tagId,
+        color: el.tagColor,
+        slug: el.tagSlug,
+        name: el.tagName
+      }));
+    } catch (error) {
+      throw new DatabaseError({ error, name: "get secret tags" });
+    }
+  };
+
+  const findByFolderIds = async (folderIds: string[], userId?: string, tx?: Knex) => {
+    try {
+      // check if not uui then userId id is null (corner case because service token's ID is not UUI in effort to keep backwards compatibility from mongo)
+      if (userId && !uuidValidate(userId)) {
+        // eslint-disable-next-line no-param-reassign
+        userId = undefined;
+      }
+
+      const secs = await (tx || db.replicaNode())(TableName.SecretV2)
+        .whereIn("folderId", folderIds)
+        .where((bd) => {
+          void bd.whereNull("userId").orWhere({ userId: userId || null });
+        })
+        .leftJoin(
+          TableName.SecretV2JnTag,
+          `${TableName.SecretV2}.id`,
+          `${TableName.SecretV2JnTag}.${TableName.SecretV2}Id`
+        )
+        .leftJoin(
+          TableName.SecretTag,
+          `${TableName.SecretV2JnTag}.${TableName.SecretTag}Id`,
+          `${TableName.SecretTag}.id`
+        )
+        .select(selectAllTableCols(TableName.SecretV2))
+        .select(db.ref("id").withSchema(TableName.SecretTag).as("tagId"))
+        .select(db.ref("color").withSchema(TableName.SecretTag).as("tagColor"))
+        .select(db.ref("slug").withSchema(TableName.SecretTag).as("tagSlug"))
+        .select(db.ref("name").withSchema(TableName.SecretTag).as("tagName"))
+        .orderBy("id", "asc");
+
+      const data = sqlNestRelationships({
+        data: secs,
+        key: "id",
+        parentMapper: (el) => ({ _id: el.id, ...SecretsV2Schema.parse(el) }),
+        childrenMapper: [
+          {
+            key: "tagId",
+            label: "tags" as const,
+            mapper: ({ tagId: id, tagColor: color, tagSlug: slug, tagName: name }) => ({
+              id,
+              color,
+              slug,
+              name
+            })
+          }
+        ]
+      });
+      return data;
+    } catch (error) {
+      throw new DatabaseError({ error, name: "get all secret" });
+    }
+  };
+
+  const findBySecretKeys = async (
+    folderId: string,
+    query: Array<{ key: string; type: SecretType.Shared } | { key: string; type: SecretType.Personal; userId: string }>,
+    tx?: Knex
+  ) => {
+    if (!query.length) return [];
+    try {
+      const secrets = await (tx || db.replicaNode())(TableName.SecretV2)
+        .where({ folderId })
+        .where((bd) => {
+          query.forEach((el) => {
+            if (el.type === SecretType.Personal && !el.userId) {
+              throw new BadRequestError({ message: "Missing personal user id" });
+            }
+            void bd.orWhere({
+              key: el.key,
+              type: el.type,
+              userId: el.type === SecretType.Personal ? el.userId : null
+            });
+          });
+        });
+      return secrets;
+    } catch (error) {
+      throw new DatabaseError({ error, name: "find by blind indexes" });
+    }
+  };
+
+  const upsertSecretReferences = async (
+    data: {
+      secretId: string;
+      references: Array<{ environment: string; secretPath: string; secretKey: string }>;
+    }[] = [],
+    tx?: Knex
+  ) => {
+    try {
+      if (!data.length) return;
+
+      await (tx || db)(TableName.SecretReferenceV2)
+        .whereIn(
+          "secretId",
+          data.map(({ secretId }) => secretId)
+        )
+        .delete();
+      const newSecretReferences = data
+        .filter(({ references }) => references.length)
+        .flatMap(({ secretId, references }) =>
+          references.map(({ environment, secretPath, secretKey }) => ({
+            secretPath,
+            secretId,
+            environment,
+            secretKey
+          }))
+        );
+      if (!newSecretReferences.length) return;
+      const secretReferences = await (tx || db)(TableName.SecretReferenceV2).insert(newSecretReferences);
+      return secretReferences;
+    } catch (error) {
+      throw new DatabaseError({ error, name: "UpsertSecretReference" });
+    }
+  };
+
+  const findReferencedSecretReferences = async (projectId: string, envSlug: string, secretPath: string, tx?: Knex) => {
+    try {
+      const docs = await (tx || db.replicaNode())(TableName.SecretReferenceV2)
+        .where({
+          secretPath,
+          environment: envSlug
+        })
+        .join(TableName.SecretV2, `${TableName.SecretV2}.id`, `${TableName.SecretReferenceV2}.secretId`)
+        .join(TableName.SecretFolder, `${TableName.SecretV2}.folderId`, `${TableName.SecretFolder}.id`)
+        .join(TableName.Environment, `${TableName.SecretFolder}.envId`, `${TableName.Environment}.id`)
+        .where("projectId", projectId)
+        .select(selectAllTableCols(TableName.SecretReferenceV2))
+        .select("folderId");
+
+      return docs;
+    } catch (error) {
+      throw new DatabaseError({ error, name: "FindReferencedSecretReferences" });
+    }
+  };
+
+  // special query to backfill secret value
+  const findAllProjectSecretValues = async (projectId: string, tx?: Knex) => {
+    try {
+      const docs = await (tx || db.replicaNode())(TableName.SecretV2)
+        .join(TableName.SecretFolder, `${TableName.SecretV2}.folderId`, `${TableName.SecretFolder}.id`)
+        .join(TableName.Environment, `${TableName.SecretFolder}.envId`, `${TableName.Environment}.id`)
+        .where("projectId", projectId)
+        // not empty
+        .whereNotNull("encryptedValue")
+        .select("encryptedValue", `${TableName.SecretV2}.id` as "id");
+      return docs;
+    } catch (error) {
+      throw new DatabaseError({ error, name: "FindAllProjectSecretValues" });
+    }
+  };
+
+  const findOneWithTags = async (filter: Partial<TSecretsV2>, tx?: Knex) => {
+    try {
+      const rawDocs = await (tx || db.replicaNode())(TableName.SecretV2)
+        .where(filter)
+        .leftJoin(
+          TableName.SecretV2JnTag,
+          `${TableName.SecretV2}.id`,
+          `${TableName.SecretV2JnTag}.${TableName.SecretV2}Id`
+        )
+        .leftJoin(
+          TableName.SecretTag,
+          `${TableName.SecretV2JnTag}.${TableName.SecretTag}Id`,
+          `${TableName.SecretTag}.id`
+        )
+        .select(selectAllTableCols(TableName.SecretV2))
+        .select(db.ref("id").withSchema(TableName.SecretTag).as("tagId"))
+        .select(db.ref("color").withSchema(TableName.SecretTag).as("tagColor"))
+        .select(db.ref("slug").withSchema(TableName.SecretTag).as("tagSlug"))
+        .select(db.ref("name").withSchema(TableName.SecretTag).as("tagName"));
+      const docs = sqlNestRelationships({
+        data: rawDocs,
+        key: "id",
+        parentMapper: (el) => ({ _id: el.id, ...SecretsV2Schema.parse(el) }),
+        childrenMapper: [
+          {
+            key: "tagId",
+            label: "tags" as const,
+            mapper: ({ tagId: id, tagColor: color, tagSlug: slug, tagName: name }) => ({
+              id,
+              color,
+              slug,
+              name
+            })
+          }
+        ]
+      });
+      return docs?.[0];
+    } catch (error) {
+      throw new DatabaseError({ error, name: "FindOneWIthTags" });
+    }
+  };
+
+  return {
+    ...secretOrm,
+    update,
+    bulkUpdate,
+    deleteMany,
+    bulkUpdateNoVersionIncrement,
+    getSecretTags,
+    findOneWithTags,
+    findByFolderId,
+    findByFolderIds,
+    findBySecretKeys,
+    upsertSecretReferences,
+    findReferencedSecretReferences,
+    findAllProjectSecretValues
+  };
+};

backend/src/services/secret-v2-bridge/secret-v2-bridge-fns.ts

@@ -0,0 +1,553 @@
+import path from "node:path";
+
+import { TableName, TSecretFolders, TSecretsV2 } from "@app/db/schemas";
+import { groupBy } from "@app/lib/fn";
+import { logger } from "@app/lib/logger";
+
+import { TProjectEnvDALFactory } from "../project-env/project-env-dal";
+import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal";
+import { TSecretV2BridgeDALFactory } from "./secret-v2-bridge-dal";
+import { TFnSecretBulkDelete, TFnSecretBulkInsert, TFnSecretBulkUpdate } from "./secret-v2-bridge-types";
+
+const INTERPOLATION_SYNTAX_REG = /\${([^}]+)}/g;
+
+export const shouldUseSecretV2Bridge = (version: number) => version === 3;
+
+/**
+ * Grabs and processes nested secret references from a string
+ *
+ * This function looks for patterns that match the interpolation syntax in the input string.
+ * It filters out references that include nested paths, splits them into environment and
+ * secret path parts, and then returns an array of objects with the environment and the
+ * joined secret path.
+ * @example
+ * const value = "Hello ${dev.someFolder.OtherFolder.SECRET_NAME} and ${prod.anotherFolder.SECRET_NAME}";
+ * const result = getAllNestedSecretReferences(value);
+ * // result will be:
+ * // [
+ * //   { environment: 'dev', secretPath: '/someFolder/OtherFolder' },
+ * //   { environment: 'prod', secretPath: '/anotherFolder' }
+ * // ]
+ */
+export const getAllNestedSecretReferences = (maybeSecretReference: string) => {
+  const references = Array.from(maybeSecretReference.matchAll(INTERPOLATION_SYNTAX_REG), (m) => m[1]);
+  return references
+    .filter((el) => el.includes("."))
+    .map((el) => {
+      const [environment, ...secretPathList] = el.split(".");
+      return {
+        environment,
+        secretPath: path.join("/", ...secretPathList.slice(0, -1)),
+        secretKey: secretPathList[secretPathList.length - 1]
+      };
+    });
+};
+
+// these functions are special functions shared by a couple of resources
+// used by secret approval, rotation or anywhere in which secret needs to modified
+export const fnSecretBulkInsert = async ({
+  // TODO: Pick types here
+  folderId,
+  inputSecrets,
+  secretDAL,
+  secretVersionDAL,
+  secretTagDAL,
+  secretVersionTagDAL,
+  tx
+}: TFnSecretBulkInsert) => {
+  const sanitizedInputSecrets = inputSecrets.map(
+    ({
+      skipMultilineEncoding,
+      type,
+      key,
+      userId,
+      encryptedComment,
+      version,
+      metadata,
+      reminderNote,
+      encryptedValue,
+      reminderRepeatDays
+    }) => ({
+      skipMultilineEncoding,
+      type,
+      key,
+      userId,
+      encryptedComment,
+      version,
+      metadata,
+      reminderNote,
+      encryptedValue,
+      reminderRepeatDays
+    })
+  );
+
+  const newSecrets = await secretDAL.insertMany(sanitizedInputSecrets.map((el) => ({ ...el, folderId })));
+  const newSecretGroupedByKeyName = groupBy(newSecrets, (item) => item.key);
+  const newSecretTags = inputSecrets.flatMap(({ tagIds: secretTags = [], key }) =>
+    secretTags.map((tag) => ({
+      [`${TableName.SecretTag}Id` as const]: tag,
+      [`${TableName.SecretV2}Id` as const]: newSecretGroupedByKeyName[key][0].id
+    }))
+  );
+  const secretVersions = await secretVersionDAL.insertMany(
+    sanitizedInputSecrets.map((el) => ({
+      ...el,
+      folderId,
+      secretId: newSecretGroupedByKeyName[el.key][0].id
+    })),
+    tx
+  );
+  await secretDAL.upsertSecretReferences(
+    inputSecrets.map(({ references = [], key }) => ({
+      secretId: newSecretGroupedByKeyName[key][0].id,
+      references
+    })),
+    tx
+  );
+  if (newSecretTags.length) {
+    const secTags = await secretTagDAL.saveTagsToSecretV2(newSecretTags, tx);
+    const secVersionsGroupBySecId = groupBy(secretVersions, (i) => i.secretId);
+    const newSecretVersionTags = secTags.flatMap(({ secrets_v2Id, secret_tagsId }) => ({
+      [`${TableName.SecretVersionV2}Id` as const]: secVersionsGroupBySecId[secrets_v2Id][0].id,
+      [`${TableName.SecretTag}Id` as const]: secret_tagsId
+    }));
+    await secretVersionTagDAL.insertMany(newSecretVersionTags, tx);
+  }
+
+  return newSecrets.map((secret) => ({ ...secret, _id: secret.id }));
+};
+
+export const fnSecretBulkUpdate = async ({
+  tx,
+  inputSecrets,
+  folderId,
+  secretDAL,
+  secretVersionDAL,
+  secretTagDAL,
+  secretVersionTagDAL
+}: TFnSecretBulkUpdate) => {
+  const sanitizedInputSecrets = inputSecrets.map(
+    ({
+      filter,
+      data: {
+        skipMultilineEncoding,
+        type,
+        key,
+        encryptedValue,
+        userId,
+        encryptedComment,
+        version,
+        metadata,
+        reminderNote,
+        reminderRepeatDays
+      }
+    }) => ({
+      filter: { ...filter, folderId },
+      data: {
+        skipMultilineEncoding,
+        type,
+        key,
+        userId,
+        encryptedComment,
+        version,
+        metadata,
+        reminderNote,
+        encryptedValue,
+        reminderRepeatDays
+      }
+    })
+  );
+
+  const newSecrets = await secretDAL.bulkUpdate(sanitizedInputSecrets, tx);
+  const secretVersions = await secretVersionDAL.insertMany(
+    newSecrets.map(
+      ({
+        skipMultilineEncoding,
+        type,
+        key,
+        userId,
+        encryptedComment,
+        version,
+        metadata,
+        reminderNote,
+        encryptedValue,
+        reminderRepeatDays,
+        id: secretId
+      }) => ({
+        skipMultilineEncoding,
+        type,
+        key,
+        userId,
+        encryptedComment,
+        version,
+        metadata,
+        reminderNote,
+        encryptedValue,
+        reminderRepeatDays,
+        folderId,
+        secretId
+      })
+    ),
+    tx
+  );
+  await secretDAL.upsertSecretReferences(
+    inputSecrets
+      .filter(({ data: { references } }) => Boolean(references))
+      .map(({ data: { references = [] } }, i) => ({
+        secretId: newSecrets[i].id,
+        references
+      })),
+    tx
+  );
+  const secsUpdatedTag = inputSecrets.flatMap(({ data: { tags } }, i) =>
+    tags !== undefined ? { tags, secretId: newSecrets[i].id } : []
+  );
+  if (secsUpdatedTag.length) {
+    await secretTagDAL.deleteTagsToSecretV2({ $in: { id: secsUpdatedTag.map(({ secretId }) => secretId) } }, tx);
+    const newSecretTags = secsUpdatedTag.flatMap(({ tags: secretTags = [], secretId }) =>
+      secretTags.map((tag) => ({
+        [`${TableName.SecretTag}Id` as const]: tag,
+        [`${TableName.SecretV2}Id` as const]: secretId
+      }))
+    );
+    if (newSecretTags.length) {
+      const secTags = await secretTagDAL.saveTagsToSecretV2(newSecretTags, tx);
+      const secVersionsGroupBySecId = groupBy(secretVersions, (i) => i.secretId);
+      const newSecretVersionTags = secTags.flatMap(({ secrets_v2Id, secret_tagsId }) => ({
+        [`${TableName.SecretVersionV2}Id` as const]: secVersionsGroupBySecId[secrets_v2Id][0].id,
+        [`${TableName.SecretTag}Id` as const]: secret_tagsId
+      }));
+      await secretVersionTagDAL.insertMany(newSecretVersionTags, tx);
+    }
+  }
+
+  return newSecrets.map((secret) => ({ ...secret, _id: secret.id }));
+};
+
+export const fnSecretBulkDelete = async ({
+  folderId,
+  inputSecrets,
+  tx,
+  actorId,
+  secretDAL,
+  secretQueueService
+}: TFnSecretBulkDelete) => {
+  const deletedSecrets = await secretDAL.deleteMany(
+    inputSecrets.map(({ type, secretKey }) => ({
+      key: secretKey,
+      type
+    })),
+    folderId,
+    actorId,
+    tx
+  );
+
+  await Promise.allSettled(
+    deletedSecrets
+      .filter(({ reminderRepeatDays }) => Boolean(reminderRepeatDays))
+      .map(({ id, reminderRepeatDays }) =>
+        secretQueueService.removeSecretReminder({ secretId: id, repeatDays: reminderRepeatDays as number })
+      )
+  );
+
+  return deletedSecrets;
+};
+
+// Introduce a new interface for mapping parent IDs to their children
+interface FolderMap {
+  [parentId: string]: TSecretFolders[];
+}
+const buildHierarchy = (folders: TSecretFolders[]): FolderMap => {
+  const map: FolderMap = {};
+  map.null = []; // Initialize mapping for root directory
+
+  folders.forEach((folder) => {
+    const parentId = folder.parentId || "null";
+    if (!map[parentId]) {
+      map[parentId] = [];
+    }
+    map[parentId].push(folder);
+  });
+
+  return map;
+};
+
+const generatePaths = (
+  map: FolderMap,
+  parentId: string = "null",
+  basePath: string = "",
+  currentDepth: number = 0
+): { path: string; folderId: string }[] => {
+  const children = map[parentId || "null"] || [];
+  let paths: { path: string; folderId: string }[] = [];
+
+  children.forEach((child) => {
+    // Determine if this is the root folder of the environment. If no parentId is present and the name is root, it's the root folder
+    const isRootFolder = child.name === "root" && !child.parentId;
+
+    // Form the current path based on the base path and the current child
+    // eslint-disable-next-line no-nested-ternary
+    const currPath = basePath === "" ? (isRootFolder ? "/" : `/${child.name}`) : `${basePath}/${child.name}`;
+
+    // Add the current path
+    paths.push({
+      path: currPath,
+      folderId: child.id
+    });
+
+    // We make sure that the recursion depth doesn't exceed 20.
+    // We do this to create "circuit break", basically to ensure that we can't encounter any potential memory leaks.
+    if (currentDepth >= 20) {
+      logger.info(`generatePaths: Recursion depth exceeded 20, breaking out of recursion [map=${JSON.stringify(map)}]`);
+      return;
+    }
+    // Recursively generate paths for children, passing down the formatted path
+    const childPaths = generatePaths(map, child.id, currPath, currentDepth + 1);
+    paths = paths.concat(
+      childPaths.map((p) => ({
+        path: p.path,
+        folderId: p.folderId
+      }))
+    );
+  });
+
+  return paths;
+};
+
+type TRecursivelyFetchSecretsFromFoldersArg = {
+  folderDAL: Pick<TSecretFolderDALFactory, "find">;
+  projectEnvDAL: Pick<TProjectEnvDALFactory, "findOne">;
+  projectId: string;
+  environment: string;
+  currentPath: string;
+  hasAccess: (environment: string, secretPath: string) => boolean;
+};
+
+export const recursivelyGetSecretPaths = async ({
+  folderDAL,
+  projectEnvDAL,
+  projectId,
+  environment,
+  currentPath,
+  hasAccess
+}: TRecursivelyFetchSecretsFromFoldersArg) => {
+  const env = await projectEnvDAL.findOne({
+    projectId,
+    slug: environment
+  });
+
+  if (!env) {
+    throw new Error(`'${environment}' environment not found in project with ID ${projectId}`);
+  }
+
+  // Fetch all folders in env once with a single query
+  const folders = await folderDAL.find({
+    envId: env.id,
+    isReserved: false
+  });
+
+  // Build the folder hierarchy map
+  const folderMap = buildHierarchy(folders);
+
+  // Generate the paths paths and normalize the root path to /
+  const paths = generatePaths(folderMap).map((p) => ({
+    path: p.path === "/" ? p.path : p.path.substring(1),
+    folderId: p.folderId
+  }));
+
+  // Filter out paths that the user does not have permission to access, and paths that are not in the current path
+  const allowedPaths = paths.filter(
+    (folder) => hasAccess(environment, folder.path) && folder.path.startsWith(currentPath === "/" ? "" : currentPath)
+  );
+
+  return allowedPaths;
+};
+
+type TInterpolateSecretArg = {
+  projectId: string;
+  decryptSecret: (encryptedValue?: Buffer | null) => string;
+  secretDAL: Pick<TSecretV2BridgeDALFactory, "findByFolderId">;
+  folderDAL: Pick<TSecretFolderDALFactory, "findBySecretPath">;
+};
+
+export const interpolateSecrets = ({ projectId, decryptSecret, secretDAL, folderDAL }: TInterpolateSecretArg) => {
+  const fetchSecretsCrossEnv = () => {
+    const fetchCache: Record<string, Record<string, string>> = {};
+
+    return async (secRefEnv: string, secRefPath: string[], secRefKey: string) => {
+      const secRefPathUrl = path.join("/", ...secRefPath);
+      const uniqKey = `${secRefEnv}-${secRefPathUrl}`;
+
+      if (fetchCache?.[uniqKey]) {
+        return fetchCache[uniqKey][secRefKey];
+      }
+
+      const folder = await folderDAL.findBySecretPath(projectId, secRefEnv, secRefPathUrl);
+      if (!folder) return "";
+      const secrets = await secretDAL.findByFolderId(folder.id);
+
+      const decryptedSec = secrets.reduce<Record<string, string>>((prev, secret) => {
+        // eslint-disable-next-line
+        prev[secret.key] = decryptSecret(secret.encryptedValue);
+        return prev;
+      }, {});
+
+      fetchCache[uniqKey] = decryptedSec;
+
+      return fetchCache[uniqKey][secRefKey];
+    };
+  };
+
+  const recursivelyExpandSecret = async (
+    expandedSec: Record<string, string>,
+    interpolatedSec: Record<string, string>,
+    fetchCrossEnv: (env: string, secPath: string[], secKey: string) => Promise<string>,
+    recursionChainBreaker: Record<string, boolean>,
+    key: string
+  ) => {
+    if (expandedSec?.[key] !== undefined) {
+      return expandedSec[key];
+    }
+    if (recursionChainBreaker?.[key]) {
+      return "";
+    }
+    // eslint-disable-next-line
+    recursionChainBreaker[key] = true;
+
+    let interpolatedValue = interpolatedSec[key];
+    if (!interpolatedValue) {
+      // eslint-disable-next-line no-console
+      console.error(`Couldn't find referenced value - ${key}`);
+      return "";
+    }
+
+    const refs = interpolatedValue.match(INTERPOLATION_SYNTAX_REG);
+    if (refs) {
+      for (const interpolationSyntax of refs) {
+        const interpolationKey = interpolationSyntax.slice(2, interpolationSyntax.length - 1);
+        const entities = interpolationKey.trim().split(".");
+
+        if (entities.length === 1) {
+          // eslint-disable-next-line
+          const val = await recursivelyExpandSecret(
+            expandedSec,
+            interpolatedSec,
+            fetchCrossEnv,
+            recursionChainBreaker,
+            interpolationKey
+          );
+          if (val) {
+            interpolatedValue = interpolatedValue.replaceAll(interpolationSyntax, val);
+          }
+          // eslint-disable-next-line
+          continue;
+        }
+
+        if (entities.length > 1) {
+          const secRefEnv = entities[0];
+          const secRefPath = entities.slice(1, entities.length - 1);
+          const secRefKey = entities[entities.length - 1];
+
+          // eslint-disable-next-line
+          const val = await fetchCrossEnv(secRefEnv, secRefPath, secRefKey);
+          if (val) {
+            interpolatedValue = interpolatedValue.replaceAll(interpolationSyntax, val);
+          }
+        }
+      }
+    }
+
+    // eslint-disable-next-line
+    expandedSec[key] = interpolatedValue;
+    return interpolatedValue;
+  };
+
+  // used to convert multi line ones to quotes ones with \n
+  const formatMultiValueEnv = (val?: string) => {
+    if (!val) return "";
+    if (!val.match("\n")) return val;
+    return `"${val.replace(/\n/g, "\\n")}"`;
+  };
+
+  const expandSecrets = async (
+    secrets: Record<string, { value: string; comment?: string; skipMultilineEncoding?: boolean | null }>
+  ) => {
+    const expandedSec: Record<string, string> = {};
+    const interpolatedSec: Record<string, string> = {};
+
+    const crossSecEnvFetch = fetchSecretsCrossEnv();
+
+    Object.keys(secrets).forEach((key) => {
+      if (secrets[key].value.match(INTERPOLATION_SYNTAX_REG)) {
+        interpolatedSec[key] = secrets[key].value;
+      } else {
+        expandedSec[key] = secrets[key].value;
+      }
+    });
+
+    for (const key of Object.keys(secrets)) {
+      if (expandedSec?.[key]) {
+        // should not do multi line encoding if user has set it to skip
+        // eslint-disable-next-line
+        secrets[key].value = secrets[key].skipMultilineEncoding
+          ? formatMultiValueEnv(expandedSec[key])
+          : expandedSec[key];
+        // eslint-disable-next-line
+        continue;
+      }
+
+      // this is to avoid recursion loop. So the graph should be direct graph rather than cyclic
+      // so for any recursion building if there is an entity two times same key meaning it will be looped
+      const recursionChainBreaker: Record<string, boolean> = {};
+      // eslint-disable-next-line
+      const expandedVal = await recursivelyExpandSecret(
+        expandedSec,
+        interpolatedSec,
+        crossSecEnvFetch,
+        recursionChainBreaker,
+        key
+      );
+
+      // eslint-disable-next-line
+      secrets[key].value = secrets[key].skipMultilineEncoding ? formatMultiValueEnv(expandedVal) : expandedVal;
+    }
+
+    return secrets;
+  };
+  return expandSecrets;
+};
+
+export const reshapeBridgeSecret = (
+  workspaceId: string,
+  environment: string,
+  secretPath: string,
+  secret: Omit<TSecretsV2, "encryptedValue" | "encryptedComment"> & {
+    value?: string;
+    comment?: string;
+    tags?: {
+      id: string;
+      slug: string;
+      color?: string | null;
+      name: string;
+    }[];
+  }
+) => ({
+  secretKey: secret.key,
+  secretPath,
+  workspace: workspaceId,
+  environment,
+  secretValue: secret.value,
+  secretComment: secret.comment,
+  version: secret.version,
+  type: secret.type,
+  _id: secret.id,
+  id: secret.id,
+  user: secret.userId,
+  tags: secret.tags,
+  skipMultilineEncoding: secret.skipMultilineEncoding,
+  secretReminderRepeatDays: secret.reminderRepeatDays,
+  secretReminderNote: secret.reminderNote,
+  metadata: secret.metadata,
+  createdAt: secret.createdAt,
+  updatedAt: secret.updatedAt
+});

backend/src/services/secret-v2-bridge/secret-v2-bridge-types.ts

@@ -0,0 +1,267 @@
+import { Knex } from "knex";
+
+import { SecretType, TSecretsV2, TSecretsV2Insert, TSecretsV2Update } from "@app/db/schemas";
+import { TProjectPermission } from "@app/lib/types";
+import { TProjectDALFactory } from "@app/services/project/project-dal";
+import { TSecretFolderDALFactory } from "@app/services/secret-folder/secret-folder-dal";
+import { TSecretTagDALFactory } from "@app/services/secret-tag/secret-tag-dal";
+
+import { TSecretV2BridgeDALFactory } from "./secret-v2-bridge-dal";
+import { TSecretVersionV2DALFactory } from "./secret-version-dal";
+import { TSecretVersionV2TagDALFactory } from "./secret-version-tag-dal";
+
+type TPartialSecret = Pick<TSecretsV2, "id" | "reminderRepeatDays" | "reminderNote">;
+
+type TPartialInputSecret = Pick<TSecretsV2, "type" | "reminderNote" | "reminderRepeatDays" | "id">;
+
+export type TGetSecretsDTO = {
+  expandSecretReferences?: boolean;
+  path: string;
+  environment: string;
+  includeImports?: boolean;
+  recursive?: boolean;
+} & TProjectPermission;
+
+export type TGetASecretDTO = {
+  secretName: string;
+  path: string;
+  environment: string;
+  expandSecretReferences?: boolean;
+  type: "shared" | "personal";
+  includeImports?: boolean;
+  version?: number;
+  projectId: string;
+} & Omit<TProjectPermission, "projectId">;
+
+export type TCreateSecretDTO = TProjectPermission & {
+  secretName: string;
+  secretPath: string;
+  environment: string;
+  secretValue: string;
+  type: SecretType;
+  tagIds?: string[];
+  secretComment?: string;
+  skipMultilineEncoding?: boolean;
+  secretReminderRepeatDays?: number | null;
+  secretReminderNote?: string | null;
+};
+
+export type TUpdateSecretDTO = TProjectPermission & {
+  secretPath: string;
+  environment: string;
+  secretName: string;
+  secretValue?: string;
+  newSecretName?: string;
+  secretComment?: string;
+  type: SecretType;
+  tagIds?: string[];
+  skipMultilineEncoding?: boolean;
+  secretReminderRepeatDays?: number | null;
+  secretReminderNote?: string | null;
+  metadata?: {
+    source?: string;
+  };
+};
+
+export type TDeleteSecretDTO = TProjectPermission & {
+  secretPath: string;
+  environment: string;
+  secretName: string;
+  type: SecretType;
+};
+
+export type TCreateManySecretDTO = Omit<TProjectPermission, "projectId"> & {
+  secretPath: string;
+  projectId: string;
+  environment: string;
+  secrets: {
+    secretKey: string;
+    secretValue: string;
+    secretComment?: string;
+    skipMultilineEncoding?: boolean;
+    tagIds?: string[];
+    metadata?: {
+      source?: string;
+    };
+  }[];
+};
+
+export type TUpdateManySecretDTO = Omit<TProjectPermission, "projectId"> & {
+  secretPath: string;
+  projectId: string;
+  environment: string;
+  secrets: {
+    secretKey: string;
+    newSecretName?: string;
+    secretValue: string;
+    secretComment?: string;
+    skipMultilineEncoding?: boolean;
+    tagIds?: string[];
+    secretReminderRepeatDays?: number | null;
+    secretReminderNote?: string | null;
+  }[];
+};
+
+export type TDeleteManySecretDTO = Omit<TProjectPermission, "projectId"> & {
+  secretPath: string;
+  projectId: string;
+  environment: string;
+  secrets: {
+    secretKey: string;
+    type?: SecretType;
+  }[];
+};
+
+export type TGetSecretVersionsDTO = Omit<TProjectPermission, "projectId"> & {
+  limit?: number;
+  offset?: number;
+  secretId: string;
+};
+
+export type TSecretReference = { environment: string; secretPath: string; secretKey: string };
+
+export type TFnSecretBulkInsert = {
+  folderId: string;
+  tx?: Knex;
+  inputSecrets: Array<Omit<TSecretsV2Insert, "folderId"> & { tagIds?: string[]; references: TSecretReference[] }>;
+  secretDAL: Pick<TSecretV2BridgeDALFactory, "insertMany" | "upsertSecretReferences">;
+  secretVersionDAL: Pick<TSecretVersionV2DALFactory, "insertMany">;
+  secretTagDAL: Pick<TSecretTagDALFactory, "saveTagsToSecretV2">;
+  secretVersionTagDAL: Pick<TSecretVersionV2TagDALFactory, "insertMany">;
+};
+
+type TRequireReferenceIfValue =
+  | (Omit<TSecretsV2Update, "encryptedValue"> & {
+      encryptedValue: Buffer | null;
+      references: TSecretReference[];
+    })
+  | (Omit<TSecretsV2Update, "encryptedValue"> & {
+      encryptedValue?: never;
+      references?: never;
+    });
+
+export type TFnSecretBulkUpdate = {
+  folderId: string;
+  inputSecrets: {
+    filter: Partial<TSecretsV2>;
+    data: TRequireReferenceIfValue & { tags?: string[] };
+  }[];
+  secretDAL: Pick<TSecretV2BridgeDALFactory, "bulkUpdate" | "upsertSecretReferences">;
+  secretVersionDAL: Pick<TSecretVersionV2DALFactory, "insertMany">;
+  secretTagDAL: Pick<TSecretTagDALFactory, "saveTagsToSecretV2" | "deleteTagsToSecretV2">;
+  secretVersionTagDAL: Pick<TSecretVersionV2TagDALFactory, "insertMany">;
+  tx?: Knex;
+};
+
+export type TFnSecretBulkDelete = {
+  folderId: string;
+  projectId: string;
+  inputSecrets: Array<{ type: SecretType; secretKey: string }>;
+  actorId: string;
+  tx?: Knex;
+  secretDAL: Pick<TSecretV2BridgeDALFactory, "deleteMany">;
+  secretQueueService: {
+    removeSecretReminder: (data: TRemoveSecretReminderDTO) => Promise<void>;
+  };
+};
+
+export type THandleReminderDTO = {
+  newSecret: TPartialInputSecret;
+  oldSecret: TPartialSecret;
+  projectId: string;
+};
+
+export type TCreateSecretReminderDTO = {
+  oldSecret: TPartialSecret;
+  newSecret: TPartialSecret;
+  projectId: string;
+};
+
+export type TRemoveSecretReminderDTO = {
+  secretId: string;
+  repeatDays: number;
+};
+
+export type TBackFillSecretReferencesDTO = TProjectPermission;
+
+export type TCreateManySecretsFnFactory = {
+  projectDAL: TProjectDALFactory;
+  secretDAL: TSecretV2BridgeDALFactory;
+  secretVersionDAL: TSecretVersionV2DALFactory;
+  secretTagDAL: TSecretTagDALFactory;
+  secretVersionTagDAL: TSecretVersionV2TagDALFactory;
+  folderDAL: TSecretFolderDALFactory;
+};
+
+export type TCreateManySecretsFn = {
+  projectId: string;
+  environment: string;
+  path: string;
+  secrets: {
+    secretName: string;
+    secretValue: string;
+    type: SecretType;
+    secretComment?: string;
+    skipMultilineEncoding?: boolean;
+    tags?: string[];
+    metadata?: {
+      source?: string;
+    };
+  }[];
+  userId?: string; // only relevant for personal secret(s)
+};
+
+export type TUpdateManySecretsFnFactory = {
+  projectDAL: TProjectDALFactory;
+  secretDAL: TSecretV2BridgeDALFactory;
+  secretVersionDAL: TSecretVersionV2DALFactory;
+  secretTagDAL: TSecretTagDALFactory;
+  secretVersionTagDAL: TSecretVersionV2TagDALFactory;
+  folderDAL: TSecretFolderDALFactory;
+};
+
+export type TUpdateManySecretsFn = {
+  projectId: string;
+  environment: string;
+  path: string;
+  secrets: {
+    secretName: string;
+    newSecretName?: string;
+    secretValue: string;
+    type: SecretType;
+    secretComment?: string;
+    skipMultilineEncoding?: boolean;
+    secretReminderRepeatDays?: number | null;
+    secretReminderNote?: string | null;
+    tags?: string[];
+    metadata?: {
+      source?: string;
+    };
+  }[];
+  userId?: string;
+};
+
+export enum SecretOperations {
+  Create = "create",
+  Update = "update",
+  Delete = "delete"
+}
+
+export type TMoveSecretsDTO = {
+  projectId: string;
+  sourceEnvironment: string;
+  sourceSecretPath: string;
+  destinationEnvironment: string;
+  destinationSecretPath: string;
+  secretIds: string[];
+  shouldOverwrite: boolean;
+} & Omit<TProjectPermission, "projectId">;
+
+export type TAttachSecretTagsDTO = {
+  projectId: string;
+  secretName: string;
+  tagSlugs: string[];
+  environment: string;
+  secretPath: string;
+  type: SecretType;
+} & Omit<TProjectPermission, "projectId">;

backend/src/services/secret-v2-bridge/secret-version-dal.ts

@@ -0,0 +1,124 @@
+import { Knex } from "knex";
+
+import { TDbClient } from "@app/db";
+import { TableName, TSecretVersionsV2, TSecretVersionsV2Update } from "@app/db/schemas";
+import { BadRequestError, DatabaseError } from "@app/lib/errors";
+import { ormify, selectAllTableCols } from "@app/lib/knex";
+
+export type TSecretVersionV2DALFactory = ReturnType<typeof secretVersionV2BridgeDALFactory>;
+
+export const secretVersionV2BridgeDALFactory = (db: TDbClient) => {
+  const secretVersionV2Orm = ormify(db, TableName.SecretVersionV2);
+
+  // This will fetch all latest secret versions from a folder
+  const findLatestVersionByFolderId = async (folderId: string, tx?: Knex) => {
+    try {
+      const docs = await (tx || db.replicaNode())(TableName.SecretVersionV2)
+        .where(`${TableName.SecretVersionV2}.folderId`, folderId)
+        .join(TableName.SecretV2, `${TableName.SecretV2}.id`, `${TableName.SecretVersionV2}.secretId`)
+        .join<TSecretVersionsV2, TSecretVersionsV2 & { secretId: string; max: number }>(
+          (tx || db)(TableName.SecretVersionV2)
+            .groupBy("folderId", "secretId")
+            .max("version")
+            .select("secretId")
+            .as("latestVersion"),
+          (bd) => {
+            bd.on(`${TableName.SecretVersionV2}.secretId`, "latestVersion.secretId").andOn(
+              `${TableName.SecretVersionV2}.version`,
+              "latestVersion.max"
+            );
+          }
+        )
+        .select(selectAllTableCols(TableName.SecretVersionV2));
+      return docs;
+    } catch (error) {
+      throw new DatabaseError({ error, name: "FindLatestVersionByFolderId" });
+    }
+  };
+
+  const bulkUpdate = async (
+    data: Array<{ filter: Partial<TSecretVersionsV2>; data: TSecretVersionsV2Update }>,
+    tx?: Knex
+  ) => {
+    try {
+      const secs = await Promise.all(
+        data.map(async ({ filter, data: updateData }) => {
+          const [doc] = await (tx || db)(TableName.SecretVersionV2)
+            .where(filter)
+            .update(updateData)
+            .increment("version", 1) // TODO: Is this really needed?
+            .returning("*");
+          if (!doc) throw new BadRequestError({ message: "Failed to update document" });
+          return doc;
+        })
+      );
+      return secs;
+    } catch (error) {
+      throw new DatabaseError({ error, name: "bulk update secret" });
+    }
+  };
+
+  const findLatestVersionMany = async (folderId: string, secretIds: string[], tx?: Knex) => {
+    try {
+      if (!secretIds.length) return {};
+      const docs: Array<TSecretVersionsV2 & { max: number }> = await (tx || db.replicaNode())(TableName.SecretVersionV2)
+        .where("folderId", folderId)
+        .whereIn(`${TableName.SecretVersionV2}.secretId`, secretIds)
+        .join(
+          (tx || db)(TableName.SecretVersionV2)
+            .groupBy("secretId")
+            .max("version")
+            .select("secretId")
+            .as("latestVersion"),
+          (bd) => {
+            bd.on(`${TableName.SecretVersionV2}.secretId`, "latestVersion.secretId").andOn(
+              `${TableName.SecretVersionV2}.version`,
+              "latestVersion.max"
+            );
+          }
+        );
+      return docs.reduce<Record<string, TSecretVersionsV2>>(
+        (prev, curr) => ({ ...prev, [curr.secretId || ""]: curr }),
+        {}
+      );
+    } catch (error) {
+      throw new DatabaseError({ error, name: "FindLatestVersinMany" });
+    }
+  };
+
+  const pruneExcessVersions = async () => {
+    try {
+      await db(TableName.SecretVersionV2)
+        .with("version_cte", (qb) => {
+          void qb
+            .from(TableName.SecretVersionV2)
+            .select(
+              "id",
+              "folderId",
+              db.raw(
+                `ROW_NUMBER() OVER (PARTITION BY ${TableName.SecretVersionV2}."secretId" ORDER BY ${TableName.SecretVersionV2}."createdAt" DESC) AS row_num`
+              )
+            );
+        })
+        .join(TableName.SecretFolder, `${TableName.SecretFolder}.id`, `${TableName.SecretVersionV2}.folderId`)
+        .join(TableName.Environment, `${TableName.Environment}.id`, `${TableName.SecretFolder}.envId`)
+        .join(TableName.Project, `${TableName.Project}.id`, `${TableName.Environment}.projectId`)
+        .join("version_cte", "version_cte.id", `${TableName.SecretVersionV2}.id`)
+        .whereRaw(`version_cte.row_num > ${TableName.Project}."pitVersionLimit"`)
+        .delete();
+    } catch (error) {
+      throw new DatabaseError({
+        error,
+        name: "Secret Version Prune"
+      });
+    }
+  };
+
+  return {
+    ...secretVersionV2Orm,
+    pruneExcessVersions,
+    findLatestVersionMany,
+    bulkUpdate,
+    findLatestVersionByFolderId
+  };
+};

backend/src/services/secret-v2-bridge/secret-version-tag-dal.ts

@@ -0,0 +1,10 @@
+import { TDbClient } from "@app/db";
+import { TableName } from "@app/db/schemas";
+import { ormify } from "@app/lib/knex";
+
+export type TSecretVersionV2TagDALFactory = ReturnType<typeof secretVersionV2TagBridgeDALFactory>;
+
+export const secretVersionV2TagBridgeDALFactory = (db: TDbClient) => {
+  const secretVersionTagDAL = ormify(db, TableName.SecretVersionV2Tag);
+  return secretVersionTagDAL;
+};

backend/src/services/secret/secret-dal.ts

@@ -20,8 +20,6 @@ export const secretDALFactory = (db: TDbClient) => {
     }
   };
 
-  // the idea is to use postgres specific function
-  // insert with id this will cause a conflict then merge the data
   const bulkUpdate = async (
     data: Array<{ filter: Partial<TSecrets>; data: TSecretsUpdate }>,
 

backend/src/services/secret/secret-fns.ts

@@ -689,7 +689,7 @@ export const createManySecretsRawFnFactory = ({
     secrets,
     userId
   }: TCreateManySecretsRawFn) => {
-    const botKey = await getBotKeyFn(projectId);
+    const { botKey } = await getBotKeyFn(projectId);
     if (!botKey) throw new BadRequestError({ message: "Project bot not found", name: "bot_not_found_error" });
 
     await projectDAL.checkProjectUpgradeStatus(projectId);
@@ -787,7 +787,7 @@ export const updateManySecretsRawFnFactory = ({
     secrets, // consider accepting instead ciphertext secrets
     userId
   }: TUpdateManySecretsRawFn): Promise<Array<TSecrets & { _id: string }>> => {
-    const botKey = await getBotKeyFn(projectId);
+    const { botKey } = await getBotKeyFn(projectId);
     if (!botKey) throw new BadRequestError({ message: "Project bot not found", name: "bot_not_found_error" });
 
     await projectDAL.checkProjectUpgradeStatus(projectId);

backend/src/services/secret/secret-service.ts

@@ -35,6 +35,7 @@ import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal";
 import { TSecretImportDALFactory } from "../secret-import/secret-import-dal";
 import { fnSecretsFromImports } from "../secret-import/secret-import-fns";
 import { TSecretTagDALFactory } from "../secret-tag/secret-tag-dal";
+import { TSecretV2BridgeServiceFactory } from "../secret-v2-bridge/secret-v2-bridge-service";
 import { TSecretDALFactory } from "./secret-dal";
 import {
   decryptSecretRaw,
@@ -83,6 +84,7 @@ type TSecretServiceFactoryDep = {
     TSecretFolderDALFactory,
     "findBySecretPath" | "updateById" | "findById" | "findByManySecretPath" | "find"
   >;
+  secretV2BridgeService: TSecretV2BridgeServiceFactory;
   secretBlindIndexDAL: TSecretBlindIndexDALFactory;
   permissionService: Pick<TPermissionServiceFactory, "getProjectPermission">;
   snapshotService: Pick<TSecretSnapshotServiceFactory, "performSnapshot">;
@@ -115,19 +117,20 @@ export const secretServiceFactory = ({
   secretVersionTagDAL,
   secretApprovalPolicyService,
   secretApprovalRequestDAL,
-  secretApprovalRequestSecretDAL
+  secretApprovalRequestSecretDAL,
+  secretV2BridgeService
 }: TSecretServiceFactoryDep) => {
   const getSecretReference = async (projectId: string) => {
     // if bot key missing means e2e still exist
-    const botKey = await projectBotService.getBotKey(projectId).catch(() => null);
+    const projectBot = await projectBotService.getBotKey(projectId).catch(() => null);
     return (el: { ciphertext?: string; iv: string; tag: string }) =>
-      botKey
+      projectBot?.botKey
         ? getAllNestedSecretReferences(
             decryptSymmetric128BitHexKeyUTF8({
               ciphertext: el.ciphertext || "",
               iv: el.iv,
               tag: el.tag,
-              key: botKey
+              key: projectBot.botKey
             })
           )
         : undefined;
@@ -951,7 +954,23 @@ export const secretServiceFactory = ({
     expandSecretReferences,
     recursive
   }: TGetSecretsRawDTO) => {
-    const botKey = await projectBotService.getBotKey(projectId);
+    const { botKey, shouldUseSecretV2Bridge } = await projectBotService.getBotKey(projectId);
+    if (shouldUseSecretV2Bridge) {
+      const { secrets, imports } = await secretV2BridgeService.getSecrets({
+        projectId,
+        expandSecretReferences,
+        actorId,
+        actor,
+        actorOrgId,
+        environment,
+        path,
+        recursive,
+        actorAuthMethod,
+        includeImports
+      });
+      return { secrets, imports };
+    }
+
     if (!botKey) throw new BadRequestError({ message: "Project bot not found", name: "bot_not_found_error" });
 
     const { secrets, imports } = await getSecrets({
@@ -1103,9 +1122,23 @@ export const secretServiceFactory = ({
     version
   }: TGetASecretRawDTO) => {
     const projectId = workspaceId || (await projectDAL.findProjectBySlug(projectSlug as string, actorOrgId)).id;
-
+    const { botKey, shouldUseSecretV2Bridge } = await projectBotService.getBotKey(projectId);
-    const botKey = await projectBotService.getBotKey(projectId);
+    if (shouldUseSecretV2Bridge) {
-    if (!botKey) throw new BadRequestError({ message: "Project bot not found", name: "bot_not_found_error" });
+      const secret = await secretV2BridgeService.getSecretByName({
+        environment,
+        projectId,
+        includeImports,
+        actorAuthMethod,
+        path,
+        actorOrgId,
+        actor,
+        actorId,
+        expandSecretReferences,
+        type,
+        secretName
+      });
+      return secret;
+    }
 
     const encryptedSecret = await getSecretByName({
       actorId,
@@ -1121,6 +1154,8 @@ export const secretServiceFactory = ({
       version
     });
 
+    if (!botKey)
+      throw new BadRequestError({ message: "Please upgrade your project first", name: "bot_not_found_error" });
     const decryptedSecret = decryptSecretRaw(encryptedSecret, botKey);
 
     if (expandSecretReferences) {
@@ -1180,9 +1215,29 @@ export const secretServiceFactory = ({
     secretReminderNote,
     secretReminderRepeatDays
   }: TCreateSecretRawDTO) => {
-    const botKey = await projectBotService.getBotKey(projectId);
+    const { botKey, shouldUseSecretV2Bridge } = await projectBotService.getBotKey(projectId);
-    if (!botKey) throw new BadRequestError({ message: "Project bot not found", name: "bot_not_found_error" });
+    if (shouldUseSecretV2Bridge) {
+      const secret = await secretV2BridgeService.createSecret({
+        secretName,
+        type,
+        actorId,
+        actor,
+        actorOrgId,
+        actorAuthMethod,
+        projectId,
+        environment,
+        secretPath,
+        secretComment,
+        secretValue,
+        tagIds,
+        secretReminderNote,
+        skipMultilineEncoding,
+        secretReminderRepeatDays
+      });
+      return secret;
+    }
 
+    if (!botKey) throw new BadRequestError({ message: "Project bot not found", name: "bot_not_found_error" });
     const secretKeyEncrypted = encryptSymmetric128BitHexKeyUTF8(secretName, botKey);
     const secretValueEncrypted = encryptSymmetric128BitHexKeyUTF8(secretValue || "", botKey);
     const secretCommentEncrypted = encryptSymmetric128BitHexKeyUTF8(secretComment || "", botKey);
@@ -1234,7 +1289,30 @@ export const secretServiceFactory = ({
     secretComment,
     newSecretName
   }: TUpdateSecretRawDTO) => {
-    const botKey = await projectBotService.getBotKey(projectId);
+    const { botKey, shouldUseSecretV2Bridge } = await projectBotService.getBotKey(projectId);
+    if (shouldUseSecretV2Bridge) {
+      const secret = await secretV2BridgeService.updateSecret({
+        secretReminderRepeatDays,
+        skipMultilineEncoding,
+        secretReminderNote,
+        tagIds,
+        secretComment,
+        secretPath,
+        environment,
+        projectId,
+        actorAuthMethod,
+        actorOrgId,
+        actor,
+        actorId,
+        type,
+        secretName,
+        newSecretName,
+        metadata,
+        secretValue
+      });
+      return secret;
+    }
+
     if (!botKey) throw new BadRequestError({ message: "Project bot not found", name: "bot_not_found_error" });
 
     const secretValueEncrypted = encryptSymmetric128BitHexKeyUTF8(secretValue || "", botKey);
@@ -1283,7 +1361,21 @@ export const secretServiceFactory = ({
     type,
     secretPath
   }: TDeleteSecretRawDTO) => {
-    const botKey = await projectBotService.getBotKey(projectId);
+    const { botKey, shouldUseSecretV2Bridge } = await projectBotService.getBotKey(projectId);
+    if (shouldUseSecretV2Bridge) {
+      const secret = await secretV2BridgeService.deleteSecret({
+        secretName,
+        type,
+        actorId,
+        actor,
+        actorOrgId,
+        actorAuthMethod,
+        projectId,
+        environment,
+        secretPath
+      });
+      return secret;
+    }
     if (!botKey) throw new BadRequestError({ message: "Project bot not found", name: "bot_not_found_error" });
 
     const secret = await deleteSecret({
@@ -1323,7 +1415,20 @@ export const secretServiceFactory = ({
       projectId = project.id;
     }
 
-    const botKey = await projectBotService.getBotKey(projectId);
+    const { botKey, shouldUseSecretV2Bridge } = await projectBotService.getBotKey(projectId);
+    if (shouldUseSecretV2Bridge) {
+      const secrets = await secretV2BridgeService.createManySecret({
+        secretPath,
+        environment,
+        projectId,
+        actorAuthMethod,
+        actorOrgId,
+        actor,
+        actorId,
+        secrets: inputSecrets
+      });
+      return secrets;
+    }
     if (!botKey) throw new BadRequestError({ message: "Project bot not found", name: "bot_not_found_error" });
 
     const secrets = await createManySecret({
@@ -1384,7 +1489,21 @@ export const secretServiceFactory = ({
       projectId = project.id;
     }
 
-    const botKey = await projectBotService.getBotKey(projectId);
+    const { botKey, shouldUseSecretV2Bridge } = await projectBotService.getBotKey(projectId);
+    if (shouldUseSecretV2Bridge) {
+      const secrets = await secretV2BridgeService.updateManySecret({
+        secretPath,
+        environment,
+        projectId,
+        actorAuthMethod,
+        actorOrgId,
+        actor,
+        actorId,
+        secrets: inputSecrets
+      });
+      return secrets;
+    }
+
     if (!botKey) throw new BadRequestError({ message: "Project bot not found", name: "bot_not_found_error" });
 
     const secrets = await updateManySecret({
@@ -1457,7 +1576,21 @@ export const secretServiceFactory = ({
       projectId = project.id;
     }
 
-    const botKey = await projectBotService.getBotKey(projectId);
+    const { botKey, shouldUseSecretV2Bridge } = await projectBotService.getBotKey(projectId);
+    if (shouldUseSecretV2Bridge) {
+      const secrets = await secretV2BridgeService.deleteManySecret({
+        secretPath,
+        environment,
+        projectId,
+        actorAuthMethod,
+        actorOrgId,
+        actor,
+        actorId,
+        secrets: inputSecrets
+      });
+      return secrets;
+    }
+
     if (!botKey) throw new BadRequestError({ message: "Project bot not found", name: "bot_not_found_error" });
 
     const secrets = await deleteManySecret({
@@ -1517,7 +1650,6 @@ export const secretServiceFactory = ({
     actorId
   }: TAttachSecretTagsDTO) => {
     const project = await projectDAL.findProjectBySlug(projectSlug, actorOrgId);
-
     const { permission } = await permissionService.getProjectPermission(
       actor,
       actorId,
@@ -1620,7 +1752,6 @@ export const secretServiceFactory = ({
     actorId
   }: TAttachSecretTagsDTO) => {
     const project = await projectDAL.findProjectBySlug(projectSlug, actorOrgId);
-
     const { permission } = await permissionService.getProjectPermission(
       actor,
       actorId,
@@ -1735,7 +1866,17 @@ export const secretServiceFactory = ({
     if (!hasRole(ProjectMembershipRole.Admin))
       throw new BadRequestError({ message: "Only admins are allowed to take this action" });
 
-    const botKey = await projectBotService.getBotKey(projectId);
+    const { botKey, shouldUseSecretV2Bridge } = await projectBotService.getBotKey(projectId);
+    if (shouldUseSecretV2Bridge) {
+      return secretV2BridgeService.backfillSecretReferences({
+        projectId,
+        actor,
+        actorId,
+        actorOrgId,
+        actorAuthMethod
+      });
+    }
+
     if (!botKey)
       throw new BadRequestError({ message: "Please upgrade your project first", name: "bot_not_found_error" });
 
@@ -1779,6 +1920,21 @@ export const secretServiceFactory = ({
         message: "Project not found."
       });
     }
+    if (project.version === 3) {
+      return secretV2BridgeService.moveSecrets({
+        sourceEnvironment,
+        sourceSecretPath,
+        destinationEnvironment,
+        destinationSecretPath,
+        secretIds,
+        projectId: project.id,
+        shouldOverwrite,
+        actor,
+        actorId,
+        actorAuthMethod,
+        actorOrgId
+      });
+    }
 
     const { permission } = await permissionService.getProjectPermission(
       actor,
@@ -1803,7 +1959,7 @@ export const secretServiceFactory = ({
       subject(ProjectPermissionSub.Secrets, { environment: destinationEnvironment, secretPath: destinationSecretPath })
     );
 
-    const botKey = await projectBotService.getBotKey(project.id);
+    const { botKey } = await projectBotService.getBotKey(project.id);
     if (!botKey) {
       throw new BadRequestError({ message: "Project bot not found", name: "bot_not_found_error" });
     }