fix: replace keystore lock with postgres lock

backend/src/keystore/keystore.ts

@@ -11,7 +11,8 @@ export const PgSqlLock = {
   OrgGatewayRootCaInit: (orgId: string) => pgAdvisoryLockHashText(`org-gateway-root-ca:${orgId}`),
   OrgGatewayCertExchange: (orgId: string) => pgAdvisoryLockHashText(`org-gateway-cert-exchange:${orgId}`),
   SecretRotationV2Creation: (folderId: string) => pgAdvisoryLockHashText(`secret-rotation-v2-creation:${folderId}`),
-  CreateProject: (orgId: string) => pgAdvisoryLockHashText(`create-project:${orgId}`)
+  CreateProject: (orgId: string) => pgAdvisoryLockHashText(`create-project:${orgId}`),
+  CreateFolder: (envId: string, projectId: string) => pgAdvisoryLockHashText(`create-folder:${envId}-${projectId}`)
 } as const;
 
 // all the key prefixes used must be set here to avoid conflict
@@ -44,11 +45,7 @@ export const KeyStorePrefixes = {
   IdentityAccessTokenStatusUpdate: (identityAccessTokenId: string) =>
     `identity-access-token-status:${identityAccessTokenId}`,
   ServiceTokenStatusUpdate: (serviceTokenId: string) => `service-token-status:${serviceTokenId}`,
-  GatewayIdentityCredential: (identityId: string) => `gateway-credentials:${identityId}`,
+  GatewayIdentityCredential: (identityId: string) => `gateway-credentials:${identityId}`
-
-  CreateFolderLock: (envId: string, projectId: string) => `folder-creation-${envId}-${projectId}` as const,
-  WaitUntilReadyCreateFolder: (envId: string, projectId: string) =>
-    `wait-until-ready-folder-creation-${envId}-${projectId}` as const
 };
 
 export const KeyStoreTtls = {

backend/src/server/routes/index.ts

@@ -1187,8 +1187,7 @@ export const registerRoutes = async (
     projectEnvDAL,
     snapshotService,
     projectDAL,
-    folderCommitService,
+    folderCommitService
-    keyStore
   });
 
   const secretImportService = secretImportServiceFactory({

backend/src/services/secret-folder/secret-folder-service.ts

@@ -6,9 +6,8 @@ import { ActionProjectType, TSecretFoldersInsert } from "@app/db/schemas";
 import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service-types";
 import { ProjectPermissionActions, ProjectPermissionSub } from "@app/ee/services/permission/project-permission";
 import { TSecretSnapshotServiceFactory } from "@app/ee/services/secret-snapshot/secret-snapshot-service";
-import { KeyStorePrefixes, TKeyStoreFactory } from "@app/keystore/keystore";
+import { PgSqlLock } from "@app/keystore/keystore";
 import { BadRequestError, NotFoundError } from "@app/lib/errors";
-import { logger } from "@app/lib/logger";
 import { OrderByDirection, OrgServiceActor } from "@app/lib/types";
 import { buildFolderPath } from "@app/services/secret-folder/secret-folder-fns";
 
@@ -35,7 +34,6 @@ type TSecretFolderServiceFactoryDep = {
   folderVersionDAL: Pick<TSecretFolderVersionDALFactory, "findLatestFolderVersions" | "create" | "insertMany" | "find">;
   folderCommitService: Pick<TFolderCommitServiceFactory, "createCommit">;
   projectDAL: Pick<TProjectDALFactory, "findProjectBySlug">;
-  keyStore: Pick<TKeyStoreFactory, "acquireLock" | "setItemWithExpiry" | "getItem" | "waitTillReady">;
 };
 
 export type TSecretFolderServiceFactory = ReturnType<typeof secretFolderServiceFactory>;
@@ -47,8 +45,7 @@ export const secretFolderServiceFactory = ({
   projectEnvDAL,
   folderVersionDAL,
   folderCommitService,
-  projectDAL,
+  projectDAL
-  keyStore
 }: TSecretFolderServiceFactoryDep) => {
   const createFolder = async ({
     projectId,
@@ -82,165 +79,148 @@ export const secretFolderServiceFactory = ({
       });
     }
 
-    const lock = await keyStore
+    const folder = await folderDAL.transaction(async (tx) => {
-      .acquireLock([KeyStorePrefixes.CreateFolderLock(env.id, projectId)], 5000)
+      await tx.raw("SELECT pg_advisory_xact_lock(?)", [PgSqlLock.CreateFolder(env.id, env.projectId)]);
-      .catch(() => null);
 
-    try {
+      const pathWithFolder = path.join(secretPath, name);
-      if (!lock) {
+      const parentFolder = await folderDAL.findClosestFolder(projectId, environment, pathWithFolder, tx);
-        await keyStore.waitTillReady({
+
-          key: KeyStorePrefixes.WaitUntilReadyCreateFolder(env.id, projectId),
+      if (!parentFolder) {
-          keyCheckCb: (val) => val === "true",
+        throw new NotFoundError({
-          waitingCb: () => logger.debug("CreateFolder: Waiting for key store lock."),
+          message: `Parent folder for path '${pathWithFolder}' not found`
-          delay: 500
         });
       }
 
-      const folder = await folderDAL.transaction(async (tx) => {
+      // check if the exact folder already exists
-        const pathWithFolder = path.join(secretPath, name);
+      const existingFolder = await folderDAL.findOne(
-        const parentFolder = await folderDAL.findClosestFolder(projectId, environment, pathWithFolder, tx);
+        {
+          envId: env.id,
+          parentId: parentFolder.id,
+          name,
+          isReserved: false
+        },
+        tx
+      );
 
-        if (!parentFolder) {
+      if (existingFolder) {
-          throw new NotFoundError({
+        return existingFolder;
-            message: `Parent folder for path '${pathWithFolder}' not found`
+      }
-          });
-        }
 
-        // check if the exact folder already exists
+      // exact folder case
-        const existingFolder = await folderDAL.findOne(
+      if (parentFolder.path === pathWithFolder) {
-          {
+        return parentFolder;
-            envId: env.id,
+      }
-            parentId: parentFolder.id,
-            name,
-            isReserved: false
-          },
-          tx
-        );
 
-        if (existingFolder) {
+      let currentParentId = parentFolder.id;
-          return existingFolder;
-        }
 
-        // exact folder case
+      // build the full path we need by processing each segment
-        if (parentFolder.path === pathWithFolder) {
+      if (parentFolder.path !== secretPath) {
-          return parentFolder;
+        const missingSegments = secretPath.substring(parentFolder.path.length).split("/").filter(Boolean);
-        }
 
-        let currentParentId = parentFolder.id;
+        const newFolders: TSecretFoldersInsert[] = [];
 
-        // build the full path we need by processing each segment
+        // process each segment sequentially
-        if (parentFolder.path !== secretPath) {
+        for await (const segment of missingSegments) {
-          const missingSegments = secretPath.substring(parentFolder.path.length).split("/").filter(Boolean);
+          const existingSegment = await folderDAL.findOne(
-
+            {
-          const newFolders: TSecretFoldersInsert[] = [];
+              name: segment,
-
+              parentId: currentParentId,
-          // process each segment sequentially
+              envId: env.id,
-          for await (const segment of missingSegments) {
+              isReserved: false
-            const existingSegment = await folderDAL.findOne(
-              {
-                name: segment,
-                parentId: currentParentId,
-                envId: env.id,
-                isReserved: false
-              },
-              tx
-            );
-
-            if (existingSegment) {
-              // use existing folder and update the path / parent
-              currentParentId = existingSegment.id;
-            } else {
-              const newFolder = {
-                name: segment,
-                parentId: currentParentId,
-                id: uuidv4(),
-                envId: env.id,
-                version: 1
-              };
-
-              currentParentId = newFolder.id;
-              newFolders.push(newFolder);
-            }
-          }
-
-          if (newFolders.length) {
-            const docs = await folderDAL.insertMany(newFolders, tx);
-            const folderVersions = await folderVersionDAL.insertMany(
-              docs.map((doc) => ({
-                name: doc.name,
-                envId: doc.envId,
-                version: doc.version,
-                folderId: doc.id,
-                description: doc.description
-              })),
-              tx
-            );
-            await folderCommitService.createCommit(
-              {
-                actor: {
-                  type: actor,
-                  metadata: {
-                    id: actorId
-                  }
-                },
-                message: "Folder created",
-                folderId: currentParentId,
-                changes: folderVersions.map((fv) => ({
-                  type: CommitType.ADD,
-                  folderVersionId: fv.id
-                }))
-              },
-              tx
-            );
-          }
-        }
-
-        const doc = await folderDAL.create(
-          { name, envId: env.id, version: 1, parentId: currentParentId, description },
-          tx
-        );
-
-        const folderVersion = await folderVersionDAL.create(
-          {
-            name: doc.name,
-            envId: doc.envId,
-            version: doc.version,
-            folderId: doc.id,
-            description: doc.description
-          },
-          tx
-        );
-
-        await folderCommitService.createCommit(
-          {
-            actor: {
-              type: actor,
-              metadata: {
-                id: actorId
-              }
             },
-            message: "Folder created",
+            tx
-            folderId: doc.id,
+          );
-            changes: [
+
-              {
+          if (existingSegment) {
+            // use existing folder and update the path / parent
+            currentParentId = existingSegment.id;
+          } else {
+            const newFolder = {
+              name: segment,
+              parentId: currentParentId,
+              id: uuidv4(),
+              envId: env.id,
+              version: 1
+            };
+
+            currentParentId = newFolder.id;
+            newFolders.push(newFolder);
+          }
+        }
+
+        if (newFolders.length) {
+          const docs = await folderDAL.insertMany(newFolders, tx);
+          const folderVersions = await folderVersionDAL.insertMany(
+            docs.map((doc) => ({
+              name: doc.name,
+              envId: doc.envId,
+              version: doc.version,
+              folderId: doc.id,
+              description: doc.description
+            })),
+            tx
+          );
+          await folderCommitService.createCommit(
+            {
+              actor: {
+                type: actor,
+                metadata: {
+                  id: actorId
+                }
+              },
+              message: "Folder created",
+              folderId: currentParentId,
+              changes: folderVersions.map((fv) => ({
                 type: CommitType.ADD,
-                folderVersionId: folderVersion.id
+                folderVersionId: fv.id
-              }
+              }))
-            ]
+            },
+            tx
+          );
+        }
+      }
+
+      const doc = await folderDAL.create(
+        { name, envId: env.id, version: 1, parentId: currentParentId, description },
+        tx
+      );
+
+      const folderVersion = await folderVersionDAL.create(
+        {
+          name: doc.name,
+          envId: doc.envId,
+          version: doc.version,
+          folderId: doc.id,
+          description: doc.description
+        },
+        tx
+      );
+
+      await folderCommitService.createCommit(
+        {
+          actor: {
+            type: actor,
+            metadata: {
+              id: actorId
+            }
           },
-          tx
+          message: "Folder created",
-        );
+          folderId: doc.id,
+          changes: [
+            {
+              type: CommitType.ADD,
+              folderVersionId: folderVersion.id
+            }
+          ]
+        },
+        tx
+      );
 
-        return doc;
+      return doc;
-      });
+    });
 
-      await keyStore.setItemWithExpiry(KeyStorePrefixes.WaitUntilReadyCreateFolder(env.id, projectId), 10, "true");
+    await snapshotService.performSnapshot(folder.parentId as string);
-
+    return folder;
-      await snapshotService.performSnapshot(folder.parentId as string);
-      return folder;
-    } finally {
-      await lock?.release();
-    }
   };
 
   const updateManyFolders = async ({