add lock mechanism for backfillPermission script

backend/package-lock.json

@@ -37,6 +37,7 @@
         "handlebars": "^4.7.7",
         "helmet": "^5.1.1",
         "infisical-node": "^1.2.1",
+        "ioredis": "^5.3.2",
         "js-yaml": "^4.1.0",
         "jsonwebtoken": "^9.0.0",
         "jsrp": "^0.2.4",
@@ -3508,8 +3509,7 @@
     "node_modules/@ioredis/commands": {
       "version": "1.2.0",
       "resolved": "https://registry.npmjs.org/@ioredis/commands/-/commands-1.2.0.tgz",
-      "integrity": "sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg==",
+      "integrity": "sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg=="
-      "dev": true
     },
     "node_modules/@istanbuljs/load-nyc-config": {
       "version": "1.1.0",
@@ -7031,39 +7031,6 @@
         "node": ">=12"
       }
     },
-    "node_modules/bull/node_modules/denque": {
-      "version": "2.1.0",
-      "resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz",
-      "integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==",
-      "dev": true,
-      "engines": {
-        "node": ">=0.10"
-      }
-    },
-    "node_modules/bull/node_modules/ioredis": {
-      "version": "5.3.2",
-      "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.3.2.tgz",
-      "integrity": "sha512-1DKMMzlIHM02eBBVOFQ1+AolGjs6+xEcM4PDL7NqOS6szq7H9jSaEkIUH6/a5Hl241LzW6JLSiAbNvTQjUupUA==",
-      "dev": true,
-      "dependencies": {
-        "@ioredis/commands": "^1.1.1",
-        "cluster-key-slot": "^1.1.0",
-        "debug": "^4.3.4",
-        "denque": "^2.1.0",
-        "lodash.defaults": "^4.2.0",
-        "lodash.isarguments": "^3.1.0",
-        "redis-errors": "^1.2.0",
-        "redis-parser": "^3.0.0",
-        "standard-as-callback": "^2.1.0"
-      },
-      "engines": {
-        "node": ">=12.22.0"
-      },
-      "funding": {
-        "type": "opencollective",
-        "url": "https://opencollective.com/ioredis"
-      }
-    },
     "node_modules/bytes": {
       "version": "3.1.2",
       "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz",
@@ -9071,30 +9038,36 @@
       }
     },
     "node_modules/ioredis": {
-      "version": "4.28.5",
+      "version": "5.3.2",
-      "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-4.28.5.tgz",
+      "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.3.2.tgz",
-      "integrity": "sha512-3GYo0GJtLqgNXj4YhrisLaNNvWSNwSS2wS4OELGfGxH8I69+XfNdnmV1AyN+ZqMh0i7eX+SWjrwFKDBDgfBC1A==",
+      "integrity": "sha512-1DKMMzlIHM02eBBVOFQ1+AolGjs6+xEcM4PDL7NqOS6szq7H9jSaEkIUH6/a5Hl241LzW6JLSiAbNvTQjUupUA==",
       "dependencies": {
+        "@ioredis/commands": "^1.1.1",
         "cluster-key-slot": "^1.1.0",
-        "debug": "^4.3.1",
+        "debug": "^4.3.4",
-        "denque": "^1.1.0",
+        "denque": "^2.1.0",
         "lodash.defaults": "^4.2.0",
-        "lodash.flatten": "^4.4.0",
         "lodash.isarguments": "^3.1.0",
-        "p-map": "^2.1.0",
-        "redis-commands": "1.7.0",
         "redis-errors": "^1.2.0",
         "redis-parser": "^3.0.0",
         "standard-as-callback": "^2.1.0"
       },
       "engines": {
-        "node": ">=6"
+        "node": ">=12.22.0"
       },
       "funding": {
         "type": "opencollective",
         "url": "https://opencollective.com/ioredis"
       }
     },
+    "node_modules/ioredis/node_modules/denque": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz",
+      "integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==",
+      "engines": {
+        "node": ">=0.10"
+      }
+    },
     "node_modules/ip": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/ip/-/ip-2.0.0.tgz",
@@ -14445,6 +14418,31 @@
         "node": ">=10"
       }
     },
+    "node_modules/probot/node_modules/ioredis": {
+      "version": "4.28.5",
+      "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-4.28.5.tgz",
+      "integrity": "sha512-3GYo0GJtLqgNXj4YhrisLaNNvWSNwSS2wS4OELGfGxH8I69+XfNdnmV1AyN+ZqMh0i7eX+SWjrwFKDBDgfBC1A==",
+      "dependencies": {
+        "cluster-key-slot": "^1.1.0",
+        "debug": "^4.3.1",
+        "denque": "^1.1.0",
+        "lodash.defaults": "^4.2.0",
+        "lodash.flatten": "^4.4.0",
+        "lodash.isarguments": "^3.1.0",
+        "p-map": "^2.1.0",
+        "redis-commands": "1.7.0",
+        "redis-errors": "^1.2.0",
+        "redis-parser": "^3.0.0",
+        "standard-as-callback": "^2.1.0"
+      },
+      "engines": {
+        "node": ">=6"
+      },
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/ioredis"
+      }
+    },
     "node_modules/probot/node_modules/js-yaml": {
       "version": "3.14.1",
       "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.1.tgz",
@@ -19517,8 +19515,7 @@
     "@ioredis/commands": {
       "version": "1.2.0",
       "resolved": "https://registry.npmjs.org/@ioredis/commands/-/commands-1.2.0.tgz",
-      "integrity": "sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg==",
+      "integrity": "sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg=="
-      "dev": true
     },
     "@istanbuljs/load-nyc-config": {
       "version": "1.1.0",
@@ -22302,31 +22299,6 @@
         "msgpackr": "^1.5.2",
         "semver": "^7.3.2",
         "uuid": "^8.3.0"
-      },
-      "dependencies": {
-        "denque": {
-          "version": "2.1.0",
-          "resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz",
-          "integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==",
-          "dev": true
-        },
-        "ioredis": {
-          "version": "5.3.2",
-          "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.3.2.tgz",
-          "integrity": "sha512-1DKMMzlIHM02eBBVOFQ1+AolGjs6+xEcM4PDL7NqOS6szq7H9jSaEkIUH6/a5Hl241LzW6JLSiAbNvTQjUupUA==",
-          "dev": true,
-          "requires": {
-            "@ioredis/commands": "^1.1.1",
-            "cluster-key-slot": "^1.1.0",
-            "debug": "^4.3.4",
-            "denque": "^2.1.0",
-            "lodash.defaults": "^4.2.0",
-            "lodash.isarguments": "^3.1.0",
-            "redis-errors": "^1.2.0",
-            "redis-parser": "^3.0.0",
-            "standard-as-callback": "^2.1.0"
-          }
-        }
       }
     },
     "bytes": {
@@ -23809,21 +23781,26 @@
       "dev": true
     },
     "ioredis": {
-      "version": "4.28.5",
+      "version": "5.3.2",
-      "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-4.28.5.tgz",
+      "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.3.2.tgz",
-      "integrity": "sha512-3GYo0GJtLqgNXj4YhrisLaNNvWSNwSS2wS4OELGfGxH8I69+XfNdnmV1AyN+ZqMh0i7eX+SWjrwFKDBDgfBC1A==",
+      "integrity": "sha512-1DKMMzlIHM02eBBVOFQ1+AolGjs6+xEcM4PDL7NqOS6szq7H9jSaEkIUH6/a5Hl241LzW6JLSiAbNvTQjUupUA==",
       "requires": {
+        "@ioredis/commands": "^1.1.1",
         "cluster-key-slot": "^1.1.0",
-        "debug": "^4.3.1",
+        "debug": "^4.3.4",
-        "denque": "^1.1.0",
+        "denque": "^2.1.0",
         "lodash.defaults": "^4.2.0",
-        "lodash.flatten": "^4.4.0",
         "lodash.isarguments": "^3.1.0",
-        "p-map": "^2.1.0",
-        "redis-commands": "1.7.0",
         "redis-errors": "^1.2.0",
         "redis-parser": "^3.0.0",
         "standard-as-callback": "^2.1.0"
+      },
+      "dependencies": {
+        "denque": {
+          "version": "2.1.0",
+          "resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz",
+          "integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw=="
+        }
       }
     },
     "ip": {
@@ -27791,6 +27768,24 @@
           "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-8.6.0.tgz",
           "integrity": "sha512-IrPdXQsk2BbzvCBGBOTmmSH5SodmqZNt4ERAZDmW4CT+tL8VtvinqywuANaFu4bOMWki16nqf0e4oC0QIaDr/g=="
         },
+        "ioredis": {
+          "version": "4.28.5",
+          "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-4.28.5.tgz",
+          "integrity": "sha512-3GYo0GJtLqgNXj4YhrisLaNNvWSNwSS2wS4OELGfGxH8I69+XfNdnmV1AyN+ZqMh0i7eX+SWjrwFKDBDgfBC1A==",
+          "requires": {
+            "cluster-key-slot": "^1.1.0",
+            "debug": "^4.3.1",
+            "denque": "^1.1.0",
+            "lodash.defaults": "^4.2.0",
+            "lodash.flatten": "^4.4.0",
+            "lodash.isarguments": "^3.1.0",
+            "p-map": "^2.1.0",
+            "redis-commands": "1.7.0",
+            "redis-errors": "^1.2.0",
+            "redis-parser": "^3.0.0",
+            "standard-as-callback": "^2.1.0"
+          }
+        },
         "js-yaml": {
           "version": "3.14.1",
           "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.1.tgz",

backend/package.json

@@ -28,6 +28,7 @@
     "handlebars": "^4.7.7",
     "helmet": "^5.1.1",
     "infisical-node": "^1.2.1",
+    "ioredis": "^5.3.2",
     "js-yaml": "^4.1.0",
     "jsonwebtoken": "^9.0.0",
     "jsrp": "^0.2.4",

backend/src/services/RedisService.ts

@@ -0,0 +1,3 @@
+import { Redis } from "ioredis"
+
+export const redisClient = new Redis(process.env.REDIS_URL as string);

backend/src/utils/setup/backfillData.ts

@@ -3,6 +3,7 @@ import crypto from "crypto";
 import { Types } from "mongoose";
 import { encryptSymmetric128BitHexKeyUTF8 } from "../crypto";
 import { EESecretService } from "../../ee/services";
+import { redisClient } from "../../services/RedisService"
 import { IPType, ISecretVersion, SecretSnapshot, SecretVersion, TrustedIP } from "../../ee/models";
 import {
   AuthMethod,
@@ -686,6 +687,14 @@ export const backfillUserAuthMethods = async () => {
 };
 
 export const backfillPermission = async () => {
+  const lockKey = "backfill_permission_lock";
+  const timeout = 5000; // Lock timeout in milliseconds
+  const lock = await redisClient.set(lockKey, 1, "PX", timeout, "NX");
+
+  if (lock) {
+    try {
+      console.info("Lock acquired for script [backfillPermission]");
+
       const memberships = await Membership.find({
         deniedPermissions: {
           $exists: true,
@@ -780,7 +789,7 @@ export const backfillPermission = async () => {
         }
       }
 
-  console.log("Backfill: Finished converting old denied permission in workspace to viewers");
+      console.info("Backfill: Finished converting old denied permission in workspace to viewers");
 
       await MembershipOrg.updateMany(
         {
@@ -793,5 +802,14 @@ export const backfillPermission = async () => {
         }
       );
 
-  console.log("Backfill: Finished converting owner role to member");
+      console.info("Backfill: Finished converting owner role to member");
+
+    } catch (error) {
+      console.error("An error occurred when running script [backfillPermission]:", error);
+    } finally {
+      await redisClient.del(lockKey);
+    }
+  } else {
+    console.info("Could not acquire lock for script [backfillPermission], skipping");
+  }
 };