add lock mechanism for backfillPermission script

2026-10-15 19:16:22 +00:00 · 2023-09-15 16:39:59 -04:00
parent 336b5897f0
commit b88a319582
4 changed files with 193 additions and 176 deletions
@@ -0,0 +1,3 @@
+import { Redis } from "ioredis"
+
+export const redisClient = new Redis(process.env.REDIS_URL as string);
@@ -3,6 +3,7 @@ import crypto from "crypto";
 import { Types } from "mongoose";
 import { encryptSymmetric128BitHexKeyUTF8 } from "../crypto";
 import { EESecretService } from "../../ee/services";
+import { redisClient } from "../../services/RedisService"
 import { IPType, ISecretVersion, SecretSnapshot, SecretVersion, TrustedIP } from "../../ee/models";
 import {
  AuthMethod,
@@ -686,112 +687,129 @@ export const backfillUserAuthMethods = async () => {
 };

 export const backfillPermission = async () => {
-  const memberships = await Membership.find({
-    deniedPermissions: {
-      $exists: true,
-      $ne: []
-    },
-    role: MEMBER,
-  })
-    .populate<{ workspace: IWorkspace }>("workspace")
-    .lean();
+  const lockKey = "backfill_permission_lock";
+  const timeout = 5000; // Lock timeout in milliseconds
+  const lock = await redisClient.set(lockKey, 1, "PX", timeout, "NX");

-  // group memberships that need the same permission set
-  const roleMap = new Map<string, { membershipIds: string[], permissions: any[], organizationId: string, workspaceId: string }>();
+  if (lock) {
+    try {
+      console.info("Lock acquired for script [backfillPermission]");

-  for (const membership of memberships) {
-    // get permissions of members except secret permission
-    const customPermissions = memberProjectPermissions.rules.filter(
-      ({ subject }) => subject !== ProjectPermissionSub.Secrets
-    );
-    const secretAccessRule: Record<string, { read: boolean; write: boolean }> = {};
+      const memberships = await Membership.find({
+        deniedPermissions: {
+          $exists: true,
+          $ne: []
+        },
+        role: MEMBER,
+      })
+        .populate<{ workspace: IWorkspace }>("workspace")
+        .lean();

-    // iterate and record true and false ones
-    membership.deniedPermissions.forEach(({ ability, environmentSlug }) => {
-      if (!secretAccessRule?.[environmentSlug])
-        secretAccessRule[environmentSlug] = { read: true, write: true };
-      if (ability === "write") secretAccessRule[environmentSlug].write = false;
-      if (ability === "read") secretAccessRule[environmentSlug].read = false;
-    });
+      // group memberships that need the same permission set
+      const roleMap = new Map<string, { membershipIds: string[], permissions: any[], organizationId: string, workspaceId: string }>();

-    const secretPermissions: any = [];
-    Object.entries(secretAccessRule).forEach(([envSlug, { read, write }]) => {
-      if (read) {
-        secretPermissions.push({
-          subject: ProjectPermissionSub.Secrets,
-          action: ProjectPermissionActions.Read,
-          conditions: { environment: envSlug }
-        });
-      }
-      if (write) {
-        secretPermissions.push(
-          {
-            subject: ProjectPermissionSub.Secrets,
-            action: ProjectPermissionActions.Edit,
-            conditions: { environment: envSlug }
-          },
-          {
-            subject: ProjectPermissionSub.Secrets,
-            action: ProjectPermissionActions.Delete,
-            conditions: { environment: envSlug }
-          },
-          {
-            subject: ProjectPermissionSub.Secrets,
-            action: ProjectPermissionActions.Create,
-            conditions: { environment: envSlug }
-          }
+      for (const membership of memberships) {
+        // get permissions of members except secret permission
+        const customPermissions = memberProjectPermissions.rules.filter(
+          ({ subject }) => subject !== ProjectPermissionSub.Secrets
        );
-      }
-    });
+        const secretAccessRule: Record<string, { read: boolean; write: boolean }> = {};

-    const key = `${JSON.stringify(secretPermissions)}-${membership.workspace.organization.toString()}-${membership.workspace._id.toString()}`; // group roles that have same permission with in the same org and workspace
-    const value = roleMap.get(key);
-    if (value) {
-      value.membershipIds.push(membership._id.toString());
-      value.organizationId = membership.workspace.organization.toString()
-      value.workspaceId = membership.workspace._id.toString()
-    } else {
-      roleMap.set(key, { membershipIds: [membership._id.toString()], permissions: [...customPermissions, ...secretPermissions], organizationId: membership.workspace.organization.toString(), workspaceId: membership.workspace._id.toString() });
-    }
-  }
+        // iterate and record true and false ones
+        membership.deniedPermissions.forEach(({ ability, environmentSlug }) => {
+          if (!secretAccessRule?.[environmentSlug])
+            secretAccessRule[environmentSlug] = { read: true, write: true };
+          if (ability === "write") secretAccessRule[environmentSlug].write = false;
+          if (ability === "read") secretAccessRule[environmentSlug].read = false;
+        });

-  for (const [key, value] of roleMap.entries()) {
-    const { membershipIds, permissions, workspaceId, organizationId } = value
+        const secretPermissions: any = [];
+        Object.entries(secretAccessRule).forEach(([envSlug, { read, write }]) => {
+          if (read) {
+            secretPermissions.push({
+              subject: ProjectPermissionSub.Secrets,
+              action: ProjectPermissionActions.Read,
+              conditions: { environment: envSlug }
+            });
+          }
+          if (write) {
+            secretPermissions.push(
+              {
+                subject: ProjectPermissionSub.Secrets,
+                action: ProjectPermissionActions.Edit,
+                conditions: { environment: envSlug }
+              },
+              {
+                subject: ProjectPermissionSub.Secrets,
+                action: ProjectPermissionActions.Delete,
+                conditions: { environment: envSlug }
+              },
+              {
+                subject: ProjectPermissionSub.Secrets,
+                action: ProjectPermissionActions.Create,
+                conditions: { environment: envSlug }
+              }
+            );
+          }
+        });

-    const role = new Role({
-      name: "Migrated Role",
-      organization: organizationId,
-      workspace: workspaceId,
-      description: "This role was auto generated by Infisical in effort to migrate your project members to our new permission system",
-      isOrgRole: false,
-      slug: `custom-role-${crypto.randomBytes(3).toString("hex")}`,
-      permissions: permissions
-    });
-
-    await role.save();
-
-    for (const id of membershipIds) {
-      await Membership.findByIdAndUpdate(id, { // document db doesn't support update many so we must loop
-        $set: {
-          role: CUSTOM,
-          customRole: role
+        const key = `${JSON.stringify(secretPermissions)}-${membership.workspace.organization.toString()}-${membership.workspace._id.toString()}`; // group roles that have same permission with in the same org and workspace
+        const value = roleMap.get(key);
+        if (value) {
+          value.membershipIds.push(membership._id.toString());
+          value.organizationId = membership.workspace.organization.toString()
+          value.workspaceId = membership.workspace._id.toString()
+        } else {
+          roleMap.set(key, { membershipIds: [membership._id.toString()], permissions: [...customPermissions, ...secretPermissions], organizationId: membership.workspace.organization.toString(), workspaceId: membership.workspace._id.toString() });
        }
-      });
-    }
-  }
-
-  console.log("Backfill: Finished converting old denied permission in workspace to viewers");
-
-  await MembershipOrg.updateMany(
-    {
-      role: OWNER
-    },
-    {
-      $set: {
-        role: ADMIN
      }
-    }
-  );

-  console.log("Backfill: Finished converting owner role to member");
+      for (const [key, value] of roleMap.entries()) {
+        const { membershipIds, permissions, workspaceId, organizationId } = value
+
+        const role = new Role({
+          name: "Migrated Role",
+          organization: organizationId,
+          workspace: workspaceId,
+          description: "This role was auto generated by Infisical in effort to migrate your project members to our new permission system",
+          isOrgRole: false,
+          slug: `custom-role-${crypto.randomBytes(3).toString("hex")}`,
+          permissions: permissions
+        });
+
+        await role.save();
+
+        for (const id of membershipIds) {
+          await Membership.findByIdAndUpdate(id, { // document db doesn't support update many so we must loop
+            $set: {
+              role: CUSTOM,
+              customRole: role
+            }
+          });
+        }
+      }
+
+      console.info("Backfill: Finished converting old denied permission in workspace to viewers");
+
+      await MembershipOrg.updateMany(
+        {
+          role: OWNER
+        },
+        {
+          $set: {
+            role: ADMIN
+          }
+        }
+      );
+
+      console.info("Backfill: Finished converting owner role to member");
+
+    } catch (error) {
+      console.error("An error occurred when running script [backfillPermission]:", error);
+    } finally {
+      await redisClient.del(lockKey);
+    }
+  } else {
+    console.info("Could not acquire lock for script [backfillPermission], skipping");
+  }
 };