diff --git a/backend/package-lock.json b/backend/package-lock.json index 76b10ce2e..66aeceee2 100644 --- a/backend/package-lock.json +++ b/backend/package-lock.json @@ -37,6 +37,7 @@ "handlebars": "^4.7.7", "helmet": "^5.1.1", "infisical-node": "^1.2.1", + "ioredis": "^5.3.2", "js-yaml": "^4.1.0", "jsonwebtoken": "^9.0.0", "jsrp": "^0.2.4", @@ -3508,8 +3509,7 @@ "node_modules/@ioredis/commands": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/@ioredis/commands/-/commands-1.2.0.tgz", - "integrity": "sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg==", - "dev": true + "integrity": "sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg==" }, "node_modules/@istanbuljs/load-nyc-config": { "version": "1.1.0", @@ -7031,39 +7031,6 @@ "node": ">=12" } }, - "node_modules/bull/node_modules/denque": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz", - "integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==", - "dev": true, - "engines": { - "node": ">=0.10" - } - }, - "node_modules/bull/node_modules/ioredis": { - "version": "5.3.2", - "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.3.2.tgz", - "integrity": "sha512-1DKMMzlIHM02eBBVOFQ1+AolGjs6+xEcM4PDL7NqOS6szq7H9jSaEkIUH6/a5Hl241LzW6JLSiAbNvTQjUupUA==", - "dev": true, - "dependencies": { - "@ioredis/commands": "^1.1.1", - "cluster-key-slot": "^1.1.0", - "debug": "^4.3.4", - "denque": "^2.1.0", - "lodash.defaults": "^4.2.0", - "lodash.isarguments": "^3.1.0", - "redis-errors": "^1.2.0", - "redis-parser": "^3.0.0", - "standard-as-callback": "^2.1.0" - }, - "engines": { - "node": ">=12.22.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/ioredis" - } - }, "node_modules/bytes": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", @@ -9071,30 +9038,36 @@ } }, "node_modules/ioredis": { - "version": "4.28.5", - "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-4.28.5.tgz", - "integrity": "sha512-3GYo0GJtLqgNXj4YhrisLaNNvWSNwSS2wS4OELGfGxH8I69+XfNdnmV1AyN+ZqMh0i7eX+SWjrwFKDBDgfBC1A==", + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.3.2.tgz", + "integrity": "sha512-1DKMMzlIHM02eBBVOFQ1+AolGjs6+xEcM4PDL7NqOS6szq7H9jSaEkIUH6/a5Hl241LzW6JLSiAbNvTQjUupUA==", "dependencies": { + "@ioredis/commands": "^1.1.1", "cluster-key-slot": "^1.1.0", - "debug": "^4.3.1", - "denque": "^1.1.0", + "debug": "^4.3.4", + "denque": "^2.1.0", "lodash.defaults": "^4.2.0", - "lodash.flatten": "^4.4.0", "lodash.isarguments": "^3.1.0", - "p-map": "^2.1.0", - "redis-commands": "1.7.0", "redis-errors": "^1.2.0", "redis-parser": "^3.0.0", "standard-as-callback": "^2.1.0" }, "engines": { - "node": ">=6" + "node": ">=12.22.0" }, "funding": { "type": "opencollective", "url": "https://opencollective.com/ioredis" } }, + "node_modules/ioredis/node_modules/denque": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz", + "integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==", + "engines": { + "node": ">=0.10" + } + }, "node_modules/ip": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ip/-/ip-2.0.0.tgz", @@ -14445,6 +14418,31 @@ "node": ">=10" } }, + "node_modules/probot/node_modules/ioredis": { + "version": "4.28.5", + "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-4.28.5.tgz", + "integrity": "sha512-3GYo0GJtLqgNXj4YhrisLaNNvWSNwSS2wS4OELGfGxH8I69+XfNdnmV1AyN+ZqMh0i7eX+SWjrwFKDBDgfBC1A==", + "dependencies": { + "cluster-key-slot": "^1.1.0", + "debug": "^4.3.1", + "denque": "^1.1.0", + "lodash.defaults": "^4.2.0", + "lodash.flatten": "^4.4.0", + "lodash.isarguments": "^3.1.0", + "p-map": "^2.1.0", + "redis-commands": "1.7.0", + "redis-errors": "^1.2.0", + "redis-parser": "^3.0.0", + "standard-as-callback": "^2.1.0" + }, + "engines": { + "node": ">=6" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/ioredis" + } + }, "node_modules/probot/node_modules/js-yaml": { "version": "3.14.1", "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.1.tgz", @@ -19517,8 +19515,7 @@ "@ioredis/commands": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/@ioredis/commands/-/commands-1.2.0.tgz", - "integrity": "sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg==", - "dev": true + "integrity": "sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg==" }, "@istanbuljs/load-nyc-config": { "version": "1.1.0", @@ -22302,31 +22299,6 @@ "msgpackr": "^1.5.2", "semver": "^7.3.2", "uuid": "^8.3.0" - }, - "dependencies": { - "denque": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz", - "integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==", - "dev": true - }, - "ioredis": { - "version": "5.3.2", - "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.3.2.tgz", - "integrity": "sha512-1DKMMzlIHM02eBBVOFQ1+AolGjs6+xEcM4PDL7NqOS6szq7H9jSaEkIUH6/a5Hl241LzW6JLSiAbNvTQjUupUA==", - "dev": true, - "requires": { - "@ioredis/commands": "^1.1.1", - "cluster-key-slot": "^1.1.0", - "debug": "^4.3.4", - "denque": "^2.1.0", - "lodash.defaults": "^4.2.0", - "lodash.isarguments": "^3.1.0", - "redis-errors": "^1.2.0", - "redis-parser": "^3.0.0", - "standard-as-callback": "^2.1.0" - } - } } }, "bytes": { @@ -23809,21 +23781,26 @@ "dev": true }, "ioredis": { - "version": "4.28.5", - "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-4.28.5.tgz", - "integrity": "sha512-3GYo0GJtLqgNXj4YhrisLaNNvWSNwSS2wS4OELGfGxH8I69+XfNdnmV1AyN+ZqMh0i7eX+SWjrwFKDBDgfBC1A==", + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.3.2.tgz", + "integrity": "sha512-1DKMMzlIHM02eBBVOFQ1+AolGjs6+xEcM4PDL7NqOS6szq7H9jSaEkIUH6/a5Hl241LzW6JLSiAbNvTQjUupUA==", "requires": { + "@ioredis/commands": "^1.1.1", "cluster-key-slot": "^1.1.0", - "debug": "^4.3.1", - "denque": "^1.1.0", + "debug": "^4.3.4", + "denque": "^2.1.0", "lodash.defaults": "^4.2.0", - "lodash.flatten": "^4.4.0", "lodash.isarguments": "^3.1.0", - "p-map": "^2.1.0", - "redis-commands": "1.7.0", "redis-errors": "^1.2.0", "redis-parser": "^3.0.0", "standard-as-callback": "^2.1.0" + }, + "dependencies": { + "denque": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz", + "integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==" + } } }, "ip": { @@ -27791,6 +27768,24 @@ "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-8.6.0.tgz", "integrity": "sha512-IrPdXQsk2BbzvCBGBOTmmSH5SodmqZNt4ERAZDmW4CT+tL8VtvinqywuANaFu4bOMWki16nqf0e4oC0QIaDr/g==" }, + "ioredis": { + "version": "4.28.5", + "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-4.28.5.tgz", + "integrity": "sha512-3GYo0GJtLqgNXj4YhrisLaNNvWSNwSS2wS4OELGfGxH8I69+XfNdnmV1AyN+ZqMh0i7eX+SWjrwFKDBDgfBC1A==", + "requires": { + "cluster-key-slot": "^1.1.0", + "debug": "^4.3.1", + "denque": "^1.1.0", + "lodash.defaults": "^4.2.0", + "lodash.flatten": "^4.4.0", + "lodash.isarguments": "^3.1.0", + "p-map": "^2.1.0", + "redis-commands": "1.7.0", + "redis-errors": "^1.2.0", + "redis-parser": "^3.0.0", + "standard-as-callback": "^2.1.0" + } + }, "js-yaml": { "version": "3.14.1", "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.1.tgz", diff --git a/backend/package.json b/backend/package.json index e9846de54..e8ecb384f 100644 --- a/backend/package.json +++ b/backend/package.json @@ -28,6 +28,7 @@ "handlebars": "^4.7.7", "helmet": "^5.1.1", "infisical-node": "^1.2.1", + "ioredis": "^5.3.2", "js-yaml": "^4.1.0", "jsonwebtoken": "^9.0.0", "jsrp": "^0.2.4", diff --git a/backend/src/services/RedisService.ts b/backend/src/services/RedisService.ts new file mode 100644 index 000000000..1b7646c06 --- /dev/null +++ b/backend/src/services/RedisService.ts @@ -0,0 +1,3 @@ +import { Redis } from "ioredis" + +export const redisClient = new Redis(process.env.REDIS_URL as string); \ No newline at end of file diff --git a/backend/src/utils/setup/backfillData.ts b/backend/src/utils/setup/backfillData.ts index 3e9678f01..e65d29b16 100644 --- a/backend/src/utils/setup/backfillData.ts +++ b/backend/src/utils/setup/backfillData.ts @@ -3,6 +3,7 @@ import crypto from "crypto"; import { Types } from "mongoose"; import { encryptSymmetric128BitHexKeyUTF8 } from "../crypto"; import { EESecretService } from "../../ee/services"; +import { redisClient } from "../../services/RedisService" import { IPType, ISecretVersion, SecretSnapshot, SecretVersion, TrustedIP } from "../../ee/models"; import { AuthMethod, @@ -686,112 +687,129 @@ export const backfillUserAuthMethods = async () => { }; export const backfillPermission = async () => { - const memberships = await Membership.find({ - deniedPermissions: { - $exists: true, - $ne: [] - }, - role: MEMBER, - }) - .populate<{ workspace: IWorkspace }>("workspace") - .lean(); + const lockKey = "backfill_permission_lock"; + const timeout = 5000; // Lock timeout in milliseconds + const lock = await redisClient.set(lockKey, 1, "PX", timeout, "NX"); - // group memberships that need the same permission set - const roleMap = new Map(); + if (lock) { + try { + console.info("Lock acquired for script [backfillPermission]"); - for (const membership of memberships) { - // get permissions of members except secret permission - const customPermissions = memberProjectPermissions.rules.filter( - ({ subject }) => subject !== ProjectPermissionSub.Secrets - ); - const secretAccessRule: Record = {}; + const memberships = await Membership.find({ + deniedPermissions: { + $exists: true, + $ne: [] + }, + role: MEMBER, + }) + .populate<{ workspace: IWorkspace }>("workspace") + .lean(); - // iterate and record true and false ones - membership.deniedPermissions.forEach(({ ability, environmentSlug }) => { - if (!secretAccessRule?.[environmentSlug]) - secretAccessRule[environmentSlug] = { read: true, write: true }; - if (ability === "write") secretAccessRule[environmentSlug].write = false; - if (ability === "read") secretAccessRule[environmentSlug].read = false; - }); + // group memberships that need the same permission set + const roleMap = new Map(); - const secretPermissions: any = []; - Object.entries(secretAccessRule).forEach(([envSlug, { read, write }]) => { - if (read) { - secretPermissions.push({ - subject: ProjectPermissionSub.Secrets, - action: ProjectPermissionActions.Read, - conditions: { environment: envSlug } - }); - } - if (write) { - secretPermissions.push( - { - subject: ProjectPermissionSub.Secrets, - action: ProjectPermissionActions.Edit, - conditions: { environment: envSlug } - }, - { - subject: ProjectPermissionSub.Secrets, - action: ProjectPermissionActions.Delete, - conditions: { environment: envSlug } - }, - { - subject: ProjectPermissionSub.Secrets, - action: ProjectPermissionActions.Create, - conditions: { environment: envSlug } - } + for (const membership of memberships) { + // get permissions of members except secret permission + const customPermissions = memberProjectPermissions.rules.filter( + ({ subject }) => subject !== ProjectPermissionSub.Secrets ); - } - }); + const secretAccessRule: Record = {}; - const key = `${JSON.stringify(secretPermissions)}-${membership.workspace.organization.toString()}-${membership.workspace._id.toString()}`; // group roles that have same permission with in the same org and workspace - const value = roleMap.get(key); - if (value) { - value.membershipIds.push(membership._id.toString()); - value.organizationId = membership.workspace.organization.toString() - value.workspaceId = membership.workspace._id.toString() - } else { - roleMap.set(key, { membershipIds: [membership._id.toString()], permissions: [...customPermissions, ...secretPermissions], organizationId: membership.workspace.organization.toString(), workspaceId: membership.workspace._id.toString() }); - } - } + // iterate and record true and false ones + membership.deniedPermissions.forEach(({ ability, environmentSlug }) => { + if (!secretAccessRule?.[environmentSlug]) + secretAccessRule[environmentSlug] = { read: true, write: true }; + if (ability === "write") secretAccessRule[environmentSlug].write = false; + if (ability === "read") secretAccessRule[environmentSlug].read = false; + }); - for (const [key, value] of roleMap.entries()) { - const { membershipIds, permissions, workspaceId, organizationId } = value + const secretPermissions: any = []; + Object.entries(secretAccessRule).forEach(([envSlug, { read, write }]) => { + if (read) { + secretPermissions.push({ + subject: ProjectPermissionSub.Secrets, + action: ProjectPermissionActions.Read, + conditions: { environment: envSlug } + }); + } + if (write) { + secretPermissions.push( + { + subject: ProjectPermissionSub.Secrets, + action: ProjectPermissionActions.Edit, + conditions: { environment: envSlug } + }, + { + subject: ProjectPermissionSub.Secrets, + action: ProjectPermissionActions.Delete, + conditions: { environment: envSlug } + }, + { + subject: ProjectPermissionSub.Secrets, + action: ProjectPermissionActions.Create, + conditions: { environment: envSlug } + } + ); + } + }); - const role = new Role({ - name: "Migrated Role", - organization: organizationId, - workspace: workspaceId, - description: "This role was auto generated by Infisical in effort to migrate your project members to our new permission system", - isOrgRole: false, - slug: `custom-role-${crypto.randomBytes(3).toString("hex")}`, - permissions: permissions - }); - - await role.save(); - - for (const id of membershipIds) { - await Membership.findByIdAndUpdate(id, { // document db doesn't support update many so we must loop - $set: { - role: CUSTOM, - customRole: role + const key = `${JSON.stringify(secretPermissions)}-${membership.workspace.organization.toString()}-${membership.workspace._id.toString()}`; // group roles that have same permission with in the same org and workspace + const value = roleMap.get(key); + if (value) { + value.membershipIds.push(membership._id.toString()); + value.organizationId = membership.workspace.organization.toString() + value.workspaceId = membership.workspace._id.toString() + } else { + roleMap.set(key, { membershipIds: [membership._id.toString()], permissions: [...customPermissions, ...secretPermissions], organizationId: membership.workspace.organization.toString(), workspaceId: membership.workspace._id.toString() }); } - }); - } - } - - console.log("Backfill: Finished converting old denied permission in workspace to viewers"); - - await MembershipOrg.updateMany( - { - role: OWNER - }, - { - $set: { - role: ADMIN } - } - ); - console.log("Backfill: Finished converting owner role to member"); + for (const [key, value] of roleMap.entries()) { + const { membershipIds, permissions, workspaceId, organizationId } = value + + const role = new Role({ + name: "Migrated Role", + organization: organizationId, + workspace: workspaceId, + description: "This role was auto generated by Infisical in effort to migrate your project members to our new permission system", + isOrgRole: false, + slug: `custom-role-${crypto.randomBytes(3).toString("hex")}`, + permissions: permissions + }); + + await role.save(); + + for (const id of membershipIds) { + await Membership.findByIdAndUpdate(id, { // document db doesn't support update many so we must loop + $set: { + role: CUSTOM, + customRole: role + } + }); + } + } + + console.info("Backfill: Finished converting old denied permission in workspace to viewers"); + + await MembershipOrg.updateMany( + { + role: OWNER + }, + { + $set: { + role: ADMIN + } + } + ); + + console.info("Backfill: Finished converting owner role to member"); + + } catch (error) { + console.error("An error occurred when running script [backfillPermission]:", error); + } finally { + await redisClient.del(lockKey); + } + } else { + console.info("Could not acquire lock for script [backfillPermission], skipping"); + } };