add lock mechanism for backfillPermission script

This commit is contained in:
Maidul Islam
2023-09-15 16:39:59 -04:00
parent 336b5897f0
commit b88a319582
4 changed files with 193 additions and 176 deletions

View File

@@ -37,6 +37,7 @@
"handlebars": "^4.7.7",
"helmet": "^5.1.1",
"infisical-node": "^1.2.1",
"ioredis": "^5.3.2",
"js-yaml": "^4.1.0",
"jsonwebtoken": "^9.0.0",
"jsrp": "^0.2.4",
@@ -3508,8 +3509,7 @@
"node_modules/@ioredis/commands": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/@ioredis/commands/-/commands-1.2.0.tgz",
"integrity": "sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg==",
"dev": true
"integrity": "sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg=="
},
"node_modules/@istanbuljs/load-nyc-config": {
"version": "1.1.0",
@@ -7031,39 +7031,6 @@
"node": ">=12"
}
},
"node_modules/bull/node_modules/denque": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz",
"integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==",
"dev": true,
"engines": {
"node": ">=0.10"
}
},
"node_modules/bull/node_modules/ioredis": {
"version": "5.3.2",
"resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.3.2.tgz",
"integrity": "sha512-1DKMMzlIHM02eBBVOFQ1+AolGjs6+xEcM4PDL7NqOS6szq7H9jSaEkIUH6/a5Hl241LzW6JLSiAbNvTQjUupUA==",
"dev": true,
"dependencies": {
"@ioredis/commands": "^1.1.1",
"cluster-key-slot": "^1.1.0",
"debug": "^4.3.4",
"denque": "^2.1.0",
"lodash.defaults": "^4.2.0",
"lodash.isarguments": "^3.1.0",
"redis-errors": "^1.2.0",
"redis-parser": "^3.0.0",
"standard-as-callback": "^2.1.0"
},
"engines": {
"node": ">=12.22.0"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/ioredis"
}
},
"node_modules/bytes": {
"version": "3.1.2",
"resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz",
@@ -9071,30 +9038,36 @@
}
},
"node_modules/ioredis": {
"version": "4.28.5",
"resolved": "https://registry.npmjs.org/ioredis/-/ioredis-4.28.5.tgz",
"integrity": "sha512-3GYo0GJtLqgNXj4YhrisLaNNvWSNwSS2wS4OELGfGxH8I69+XfNdnmV1AyN+ZqMh0i7eX+SWjrwFKDBDgfBC1A==",
"version": "5.3.2",
"resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.3.2.tgz",
"integrity": "sha512-1DKMMzlIHM02eBBVOFQ1+AolGjs6+xEcM4PDL7NqOS6szq7H9jSaEkIUH6/a5Hl241LzW6JLSiAbNvTQjUupUA==",
"dependencies": {
"@ioredis/commands": "^1.1.1",
"cluster-key-slot": "^1.1.0",
"debug": "^4.3.1",
"denque": "^1.1.0",
"debug": "^4.3.4",
"denque": "^2.1.0",
"lodash.defaults": "^4.2.0",
"lodash.flatten": "^4.4.0",
"lodash.isarguments": "^3.1.0",
"p-map": "^2.1.0",
"redis-commands": "1.7.0",
"redis-errors": "^1.2.0",
"redis-parser": "^3.0.0",
"standard-as-callback": "^2.1.0"
},
"engines": {
"node": ">=6"
"node": ">=12.22.0"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/ioredis"
}
},
"node_modules/ioredis/node_modules/denque": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz",
"integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==",
"engines": {
"node": ">=0.10"
}
},
"node_modules/ip": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/ip/-/ip-2.0.0.tgz",
@@ -14445,6 +14418,31 @@
"node": ">=10"
}
},
"node_modules/probot/node_modules/ioredis": {
"version": "4.28.5",
"resolved": "https://registry.npmjs.org/ioredis/-/ioredis-4.28.5.tgz",
"integrity": "sha512-3GYo0GJtLqgNXj4YhrisLaNNvWSNwSS2wS4OELGfGxH8I69+XfNdnmV1AyN+ZqMh0i7eX+SWjrwFKDBDgfBC1A==",
"dependencies": {
"cluster-key-slot": "^1.1.0",
"debug": "^4.3.1",
"denque": "^1.1.0",
"lodash.defaults": "^4.2.0",
"lodash.flatten": "^4.4.0",
"lodash.isarguments": "^3.1.0",
"p-map": "^2.1.0",
"redis-commands": "1.7.0",
"redis-errors": "^1.2.0",
"redis-parser": "^3.0.0",
"standard-as-callback": "^2.1.0"
},
"engines": {
"node": ">=6"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/ioredis"
}
},
"node_modules/probot/node_modules/js-yaml": {
"version": "3.14.1",
"resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.1.tgz",
@@ -19517,8 +19515,7 @@
"@ioredis/commands": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/@ioredis/commands/-/commands-1.2.0.tgz",
"integrity": "sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg==",
"dev": true
"integrity": "sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg=="
},
"@istanbuljs/load-nyc-config": {
"version": "1.1.0",
@@ -22302,31 +22299,6 @@
"msgpackr": "^1.5.2",
"semver": "^7.3.2",
"uuid": "^8.3.0"
},
"dependencies": {
"denque": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz",
"integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==",
"dev": true
},
"ioredis": {
"version": "5.3.2",
"resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.3.2.tgz",
"integrity": "sha512-1DKMMzlIHM02eBBVOFQ1+AolGjs6+xEcM4PDL7NqOS6szq7H9jSaEkIUH6/a5Hl241LzW6JLSiAbNvTQjUupUA==",
"dev": true,
"requires": {
"@ioredis/commands": "^1.1.1",
"cluster-key-slot": "^1.1.0",
"debug": "^4.3.4",
"denque": "^2.1.0",
"lodash.defaults": "^4.2.0",
"lodash.isarguments": "^3.1.0",
"redis-errors": "^1.2.0",
"redis-parser": "^3.0.0",
"standard-as-callback": "^2.1.0"
}
}
}
},
"bytes": {
@@ -23809,21 +23781,26 @@
"dev": true
},
"ioredis": {
"version": "4.28.5",
"resolved": "https://registry.npmjs.org/ioredis/-/ioredis-4.28.5.tgz",
"integrity": "sha512-3GYo0GJtLqgNXj4YhrisLaNNvWSNwSS2wS4OELGfGxH8I69+XfNdnmV1AyN+ZqMh0i7eX+SWjrwFKDBDgfBC1A==",
"version": "5.3.2",
"resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.3.2.tgz",
"integrity": "sha512-1DKMMzlIHM02eBBVOFQ1+AolGjs6+xEcM4PDL7NqOS6szq7H9jSaEkIUH6/a5Hl241LzW6JLSiAbNvTQjUupUA==",
"requires": {
"@ioredis/commands": "^1.1.1",
"cluster-key-slot": "^1.1.0",
"debug": "^4.3.1",
"denque": "^1.1.0",
"debug": "^4.3.4",
"denque": "^2.1.0",
"lodash.defaults": "^4.2.0",
"lodash.flatten": "^4.4.0",
"lodash.isarguments": "^3.1.0",
"p-map": "^2.1.0",
"redis-commands": "1.7.0",
"redis-errors": "^1.2.0",
"redis-parser": "^3.0.0",
"standard-as-callback": "^2.1.0"
},
"dependencies": {
"denque": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz",
"integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw=="
}
}
},
"ip": {
@@ -27791,6 +27768,24 @@
"resolved": "https://registry.npmjs.org/dotenv/-/dotenv-8.6.0.tgz",
"integrity": "sha512-IrPdXQsk2BbzvCBGBOTmmSH5SodmqZNt4ERAZDmW4CT+tL8VtvinqywuANaFu4bOMWki16nqf0e4oC0QIaDr/g=="
},
"ioredis": {
"version": "4.28.5",
"resolved": "https://registry.npmjs.org/ioredis/-/ioredis-4.28.5.tgz",
"integrity": "sha512-3GYo0GJtLqgNXj4YhrisLaNNvWSNwSS2wS4OELGfGxH8I69+XfNdnmV1AyN+ZqMh0i7eX+SWjrwFKDBDgfBC1A==",
"requires": {
"cluster-key-slot": "^1.1.0",
"debug": "^4.3.1",
"denque": "^1.1.0",
"lodash.defaults": "^4.2.0",
"lodash.flatten": "^4.4.0",
"lodash.isarguments": "^3.1.0",
"p-map": "^2.1.0",
"redis-commands": "1.7.0",
"redis-errors": "^1.2.0",
"redis-parser": "^3.0.0",
"standard-as-callback": "^2.1.0"
}
},
"js-yaml": {
"version": "3.14.1",
"resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.1.tgz",

View File

@@ -28,6 +28,7 @@
"handlebars": "^4.7.7",
"helmet": "^5.1.1",
"infisical-node": "^1.2.1",
"ioredis": "^5.3.2",
"js-yaml": "^4.1.0",
"jsonwebtoken": "^9.0.0",
"jsrp": "^0.2.4",

View File

@@ -0,0 +1,3 @@
import { Redis } from "ioredis"
export const redisClient = new Redis(process.env.REDIS_URL as string);

View File

@@ -3,6 +3,7 @@ import crypto from "crypto";
import { Types } from "mongoose";
import { encryptSymmetric128BitHexKeyUTF8 } from "../crypto";
import { EESecretService } from "../../ee/services";
import { redisClient } from "../../services/RedisService"
import { IPType, ISecretVersion, SecretSnapshot, SecretVersion, TrustedIP } from "../../ee/models";
import {
AuthMethod,
@@ -686,112 +687,129 @@ export const backfillUserAuthMethods = async () => {
};
export const backfillPermission = async () => {
const memberships = await Membership.find({
deniedPermissions: {
$exists: true,
$ne: []
},
role: MEMBER,
})
.populate<{ workspace: IWorkspace }>("workspace")
.lean();
const lockKey = "backfill_permission_lock";
const timeout = 5000; // Lock timeout in milliseconds
const lock = await redisClient.set(lockKey, 1, "PX", timeout, "NX");
// group memberships that need the same permission set
const roleMap = new Map<string, { membershipIds: string[], permissions: any[], organizationId: string, workspaceId: string }>();
if (lock) {
try {
console.info("Lock acquired for script [backfillPermission]");
for (const membership of memberships) {
// get permissions of members except secret permission
const customPermissions = memberProjectPermissions.rules.filter(
({ subject }) => subject !== ProjectPermissionSub.Secrets
);
const secretAccessRule: Record<string, { read: boolean; write: boolean }> = {};
const memberships = await Membership.find({
deniedPermissions: {
$exists: true,
$ne: []
},
role: MEMBER,
})
.populate<{ workspace: IWorkspace }>("workspace")
.lean();
// iterate and record true and false ones
membership.deniedPermissions.forEach(({ ability, environmentSlug }) => {
if (!secretAccessRule?.[environmentSlug])
secretAccessRule[environmentSlug] = { read: true, write: true };
if (ability === "write") secretAccessRule[environmentSlug].write = false;
if (ability === "read") secretAccessRule[environmentSlug].read = false;
});
// group memberships that need the same permission set
const roleMap = new Map<string, { membershipIds: string[], permissions: any[], organizationId: string, workspaceId: string }>();
const secretPermissions: any = [];
Object.entries(secretAccessRule).forEach(([envSlug, { read, write }]) => {
if (read) {
secretPermissions.push({
subject: ProjectPermissionSub.Secrets,
action: ProjectPermissionActions.Read,
conditions: { environment: envSlug }
});
}
if (write) {
secretPermissions.push(
{
subject: ProjectPermissionSub.Secrets,
action: ProjectPermissionActions.Edit,
conditions: { environment: envSlug }
},
{
subject: ProjectPermissionSub.Secrets,
action: ProjectPermissionActions.Delete,
conditions: { environment: envSlug }
},
{
subject: ProjectPermissionSub.Secrets,
action: ProjectPermissionActions.Create,
conditions: { environment: envSlug }
}
for (const membership of memberships) {
// get permissions of members except secret permission
const customPermissions = memberProjectPermissions.rules.filter(
({ subject }) => subject !== ProjectPermissionSub.Secrets
);
}
});
const secretAccessRule: Record<string, { read: boolean; write: boolean }> = {};
const key = `${JSON.stringify(secretPermissions)}-${membership.workspace.organization.toString()}-${membership.workspace._id.toString()}`; // group roles that have same permission with in the same org and workspace
const value = roleMap.get(key);
if (value) {
value.membershipIds.push(membership._id.toString());
value.organizationId = membership.workspace.organization.toString()
value.workspaceId = membership.workspace._id.toString()
} else {
roleMap.set(key, { membershipIds: [membership._id.toString()], permissions: [...customPermissions, ...secretPermissions], organizationId: membership.workspace.organization.toString(), workspaceId: membership.workspace._id.toString() });
}
}
// iterate and record true and false ones
membership.deniedPermissions.forEach(({ ability, environmentSlug }) => {
if (!secretAccessRule?.[environmentSlug])
secretAccessRule[environmentSlug] = { read: true, write: true };
if (ability === "write") secretAccessRule[environmentSlug].write = false;
if (ability === "read") secretAccessRule[environmentSlug].read = false;
});
for (const [key, value] of roleMap.entries()) {
const { membershipIds, permissions, workspaceId, organizationId } = value
const secretPermissions: any = [];
Object.entries(secretAccessRule).forEach(([envSlug, { read, write }]) => {
if (read) {
secretPermissions.push({
subject: ProjectPermissionSub.Secrets,
action: ProjectPermissionActions.Read,
conditions: { environment: envSlug }
});
}
if (write) {
secretPermissions.push(
{
subject: ProjectPermissionSub.Secrets,
action: ProjectPermissionActions.Edit,
conditions: { environment: envSlug }
},
{
subject: ProjectPermissionSub.Secrets,
action: ProjectPermissionActions.Delete,
conditions: { environment: envSlug }
},
{
subject: ProjectPermissionSub.Secrets,
action: ProjectPermissionActions.Create,
conditions: { environment: envSlug }
}
);
}
});
const role = new Role({
name: "Migrated Role",
organization: organizationId,
workspace: workspaceId,
description: "This role was auto generated by Infisical in effort to migrate your project members to our new permission system",
isOrgRole: false,
slug: `custom-role-${crypto.randomBytes(3).toString("hex")}`,
permissions: permissions
});
await role.save();
for (const id of membershipIds) {
await Membership.findByIdAndUpdate(id, { // document db doesn't support update many so we must loop
$set: {
role: CUSTOM,
customRole: role
const key = `${JSON.stringify(secretPermissions)}-${membership.workspace.organization.toString()}-${membership.workspace._id.toString()}`; // group roles that have same permission with in the same org and workspace
const value = roleMap.get(key);
if (value) {
value.membershipIds.push(membership._id.toString());
value.organizationId = membership.workspace.organization.toString()
value.workspaceId = membership.workspace._id.toString()
} else {
roleMap.set(key, { membershipIds: [membership._id.toString()], permissions: [...customPermissions, ...secretPermissions], organizationId: membership.workspace.organization.toString(), workspaceId: membership.workspace._id.toString() });
}
});
}
}
console.log("Backfill: Finished converting old denied permission in workspace to viewers");
await MembershipOrg.updateMany(
{
role: OWNER
},
{
$set: {
role: ADMIN
}
}
);
console.log("Backfill: Finished converting owner role to member");
for (const [key, value] of roleMap.entries()) {
const { membershipIds, permissions, workspaceId, organizationId } = value
const role = new Role({
name: "Migrated Role",
organization: organizationId,
workspace: workspaceId,
description: "This role was auto generated by Infisical in effort to migrate your project members to our new permission system",
isOrgRole: false,
slug: `custom-role-${crypto.randomBytes(3).toString("hex")}`,
permissions: permissions
});
await role.save();
for (const id of membershipIds) {
await Membership.findByIdAndUpdate(id, { // document db doesn't support update many so we must loop
$set: {
role: CUSTOM,
customRole: role
}
});
}
}
console.info("Backfill: Finished converting old denied permission in workspace to viewers");
await MembershipOrg.updateMany(
{
role: OWNER
},
{
$set: {
role: ADMIN
}
}
);
console.info("Backfill: Finished converting owner role to member");
} catch (error) {
console.error("An error occurred when running script [backfillPermission]:", error);
} finally {
await redisClient.del(lockKey);
}
} else {
console.info("Could not acquire lock for script [backfillPermission], skipping");
}
};