diff --git a/.github/workflows/build-staging-img.yml b/.github/workflows/build-staging-img.yml index 3ff23ba23..806e89b87 100644 --- a/.github/workflows/build-staging-img.yml +++ b/.github/workflows/build-staging-img.yml @@ -11,9 +11,9 @@ jobs: - name: 📦 Install dependencies to test all dependencies run: npm ci --only-production working-directory: backend - - name: 🧪 Run tests - run: npm run test:ci - working-directory: backend + # - name: 🧪 Run tests + # run: npm run test:ci + # working-directory: backend - name: Save commit hashes for tag id: commit uses: pr-mpt/actions-commit-hash@v2 diff --git a/backend/package-lock.json b/backend/package-lock.json index 76b10ce2e..66aeceee2 100644 --- a/backend/package-lock.json +++ b/backend/package-lock.json @@ -37,6 +37,7 @@ "handlebars": "^4.7.7", "helmet": "^5.1.1", "infisical-node": "^1.2.1", + "ioredis": "^5.3.2", "js-yaml": "^4.1.0", "jsonwebtoken": "^9.0.0", "jsrp": "^0.2.4", @@ -3508,8 +3509,7 @@ "node_modules/@ioredis/commands": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/@ioredis/commands/-/commands-1.2.0.tgz", - "integrity": "sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg==", - "dev": true + "integrity": "sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg==" }, "node_modules/@istanbuljs/load-nyc-config": { "version": "1.1.0", @@ -7031,39 +7031,6 @@ "node": ">=12" } }, - "node_modules/bull/node_modules/denque": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz", - "integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==", - "dev": true, - "engines": { - "node": ">=0.10" - } - }, - "node_modules/bull/node_modules/ioredis": { - "version": "5.3.2", - "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.3.2.tgz", - "integrity": "sha512-1DKMMzlIHM02eBBVOFQ1+AolGjs6+xEcM4PDL7NqOS6szq7H9jSaEkIUH6/a5Hl241LzW6JLSiAbNvTQjUupUA==", - "dev": true, - "dependencies": { - "@ioredis/commands": "^1.1.1", - "cluster-key-slot": "^1.1.0", - "debug": "^4.3.4", - "denque": "^2.1.0", - "lodash.defaults": "^4.2.0", - "lodash.isarguments": "^3.1.0", - "redis-errors": "^1.2.0", - "redis-parser": "^3.0.0", - "standard-as-callback": "^2.1.0" - }, - "engines": { - "node": ">=12.22.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/ioredis" - } - }, "node_modules/bytes": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", @@ -9071,30 +9038,36 @@ } }, "node_modules/ioredis": { - "version": "4.28.5", - "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-4.28.5.tgz", - "integrity": "sha512-3GYo0GJtLqgNXj4YhrisLaNNvWSNwSS2wS4OELGfGxH8I69+XfNdnmV1AyN+ZqMh0i7eX+SWjrwFKDBDgfBC1A==", + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.3.2.tgz", + "integrity": "sha512-1DKMMzlIHM02eBBVOFQ1+AolGjs6+xEcM4PDL7NqOS6szq7H9jSaEkIUH6/a5Hl241LzW6JLSiAbNvTQjUupUA==", "dependencies": { + "@ioredis/commands": "^1.1.1", "cluster-key-slot": "^1.1.0", - "debug": "^4.3.1", - "denque": "^1.1.0", + "debug": "^4.3.4", + "denque": "^2.1.0", "lodash.defaults": "^4.2.0", - "lodash.flatten": "^4.4.0", "lodash.isarguments": "^3.1.0", - "p-map": "^2.1.0", - "redis-commands": "1.7.0", "redis-errors": "^1.2.0", "redis-parser": "^3.0.0", "standard-as-callback": "^2.1.0" }, "engines": { - "node": ">=6" + "node": ">=12.22.0" }, "funding": { "type": "opencollective", "url": "https://opencollective.com/ioredis" } }, + "node_modules/ioredis/node_modules/denque": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz", + "integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==", + "engines": { + "node": ">=0.10" + } + }, "node_modules/ip": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ip/-/ip-2.0.0.tgz", @@ -14445,6 +14418,31 @@ "node": ">=10" } }, + "node_modules/probot/node_modules/ioredis": { + "version": "4.28.5", + "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-4.28.5.tgz", + "integrity": "sha512-3GYo0GJtLqgNXj4YhrisLaNNvWSNwSS2wS4OELGfGxH8I69+XfNdnmV1AyN+ZqMh0i7eX+SWjrwFKDBDgfBC1A==", + "dependencies": { + "cluster-key-slot": "^1.1.0", + "debug": "^4.3.1", + "denque": "^1.1.0", + "lodash.defaults": "^4.2.0", + "lodash.flatten": "^4.4.0", + "lodash.isarguments": "^3.1.0", + "p-map": "^2.1.0", + "redis-commands": "1.7.0", + "redis-errors": "^1.2.0", + "redis-parser": "^3.0.0", + "standard-as-callback": "^2.1.0" + }, + "engines": { + "node": ">=6" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/ioredis" + } + }, "node_modules/probot/node_modules/js-yaml": { "version": "3.14.1", "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.1.tgz", @@ -19517,8 +19515,7 @@ "@ioredis/commands": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/@ioredis/commands/-/commands-1.2.0.tgz", - "integrity": "sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg==", - "dev": true + "integrity": "sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg==" }, "@istanbuljs/load-nyc-config": { "version": "1.1.0", @@ -22302,31 +22299,6 @@ "msgpackr": "^1.5.2", "semver": "^7.3.2", "uuid": "^8.3.0" - }, - "dependencies": { - "denque": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz", - "integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==", - "dev": true - }, - "ioredis": { - "version": "5.3.2", - "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.3.2.tgz", - "integrity": "sha512-1DKMMzlIHM02eBBVOFQ1+AolGjs6+xEcM4PDL7NqOS6szq7H9jSaEkIUH6/a5Hl241LzW6JLSiAbNvTQjUupUA==", - "dev": true, - "requires": { - "@ioredis/commands": "^1.1.1", - "cluster-key-slot": "^1.1.0", - "debug": "^4.3.4", - "denque": "^2.1.0", - "lodash.defaults": "^4.2.0", - "lodash.isarguments": "^3.1.0", - "redis-errors": "^1.2.0", - "redis-parser": "^3.0.0", - "standard-as-callback": "^2.1.0" - } - } } }, "bytes": { @@ -23809,21 +23781,26 @@ "dev": true }, "ioredis": { - "version": "4.28.5", - "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-4.28.5.tgz", - "integrity": "sha512-3GYo0GJtLqgNXj4YhrisLaNNvWSNwSS2wS4OELGfGxH8I69+XfNdnmV1AyN+ZqMh0i7eX+SWjrwFKDBDgfBC1A==", + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.3.2.tgz", + "integrity": "sha512-1DKMMzlIHM02eBBVOFQ1+AolGjs6+xEcM4PDL7NqOS6szq7H9jSaEkIUH6/a5Hl241LzW6JLSiAbNvTQjUupUA==", "requires": { + "@ioredis/commands": "^1.1.1", "cluster-key-slot": "^1.1.0", - "debug": "^4.3.1", - "denque": "^1.1.0", + "debug": "^4.3.4", + "denque": "^2.1.0", "lodash.defaults": "^4.2.0", - "lodash.flatten": "^4.4.0", "lodash.isarguments": "^3.1.0", - "p-map": "^2.1.0", - "redis-commands": "1.7.0", "redis-errors": "^1.2.0", "redis-parser": "^3.0.0", "standard-as-callback": "^2.1.0" + }, + "dependencies": { + "denque": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz", + "integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==" + } } }, "ip": { @@ -27791,6 +27768,24 @@ "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-8.6.0.tgz", "integrity": "sha512-IrPdXQsk2BbzvCBGBOTmmSH5SodmqZNt4ERAZDmW4CT+tL8VtvinqywuANaFu4bOMWki16nqf0e4oC0QIaDr/g==" }, + "ioredis": { + "version": "4.28.5", + "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-4.28.5.tgz", + "integrity": "sha512-3GYo0GJtLqgNXj4YhrisLaNNvWSNwSS2wS4OELGfGxH8I69+XfNdnmV1AyN+ZqMh0i7eX+SWjrwFKDBDgfBC1A==", + "requires": { + "cluster-key-slot": "^1.1.0", + "debug": "^4.3.1", + "denque": "^1.1.0", + "lodash.defaults": "^4.2.0", + "lodash.flatten": "^4.4.0", + "lodash.isarguments": "^3.1.0", + "p-map": "^2.1.0", + "redis-commands": "1.7.0", + "redis-errors": "^1.2.0", + "redis-parser": "^3.0.0", + "standard-as-callback": "^2.1.0" + } + }, "js-yaml": { "version": "3.14.1", "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.1.tgz", diff --git a/backend/package.json b/backend/package.json index e9846de54..e8ecb384f 100644 --- a/backend/package.json +++ b/backend/package.json @@ -28,6 +28,7 @@ "handlebars": "^4.7.7", "helmet": "^5.1.1", "infisical-node": "^1.2.1", + "ioredis": "^5.3.2", "js-yaml": "^4.1.0", "jsonwebtoken": "^9.0.0", "jsrp": "^0.2.4", diff --git a/backend/src/controllers/v1/membershipController.ts b/backend/src/controllers/v1/membershipController.ts index 7ad570885..b022ac5de 100644 --- a/backend/src/controllers/v1/membershipController.ts +++ b/backend/src/controllers/v1/membershipController.ts @@ -148,7 +148,12 @@ export const changeMembershipRole = async (req: Request, res: Response) => { const membership = await Membership.findByIdAndUpdate( membershipId, { - role + $set: { + role + }, + $unset: { + customRole: 1 + } }, { new: true diff --git a/backend/src/controllers/v2/organizationsController.ts b/backend/src/controllers/v2/organizationsController.ts index 277a29af8..ad776864c 100644 --- a/backend/src/controllers/v2/organizationsController.ts +++ b/backend/src/controllers/v2/organizationsController.ts @@ -159,7 +159,12 @@ export const updateOrganizationMembership = async (req: Request, res: Response) const membership = await MembershipOrg.findByIdAndUpdate( membershipId, { - role + $set: { + role + }, + $unset: { + customRole: 1 + } }, { new: true diff --git a/backend/src/integrations/sync.ts b/backend/src/integrations/sync.ts index 6605b231f..a25a93c80 100644 --- a/backend/src/integrations/sync.ts +++ b/backend/src/integrations/sync.ts @@ -328,15 +328,19 @@ const syncSecretsGCPSecretManager = async ({ const pageSize = 100; let pageToken: string | undefined; let hasMorePages = true; + + const filterParam = integration.metadata.secretGCPLabel + ? `?filter=labels.${integration.metadata.secretGCPLabel.labelName}=${integration.metadata.secretGCPLabel.labelValue}` + : ""; while (hasMorePages) { const params = new URLSearchParams({ pageSize: String(pageSize), ...(pageToken ? { pageToken } : {}) }); - + const res: GCPSMListSecretsRes = (await standardRequest.get( - `${INTEGRATION_GCP_SECRET_MANAGER_URL}/v1/projects/${integration.appId}/secrets?filter=labels.managed-by=infisical`, + `${INTEGRATION_GCP_SECRET_MANAGER_URL}/v1/projects/${integration.appId}/secrets${filterParam}`, { params, headers: { @@ -347,7 +351,24 @@ const syncSecretsGCPSecretManager = async ({ )).data; if (res.secrets) { - gcpSecrets = gcpSecrets.concat(res.secrets); + const filteredSecrets = res.secrets?.filter((gcpSecret) => { + const arr = gcpSecret.name.split("/"); + const key = arr[arr.length - 1]; + + let isValid = true; + + if (integration.metadata.secretPrefix && !key.startsWith(integration.metadata.secretPrefix)) { + isValid = false; + } + + if (integration.metadata.secretSuffix && !key.endsWith(integration.metadata.secretSuffix)) { + isValid = false; + } + + return isValid; + }); + + gcpSecrets = gcpSecrets.concat(filteredSecrets); } if (!res.nextPageToken) { @@ -371,7 +392,7 @@ const syncSecretsGCPSecretManager = async ({ const key = arr[arr.length - 1]; const secretLatest: GCPLatestSecretVersionAccess = (await standardRequest.get( - `${INTEGRATION_GCP_SECRET_MANAGER_URL}/v1beta1/projects/${integration.appId}/secrets/${key}/versions/latest:access`, + `${INTEGRATION_GCP_SECRET_MANAGER_URL}/v1/projects/${integration.appId}/secrets/${key}/versions/latest:access`, { headers: { Authorization: `Bearer ${accessToken}`, @@ -379,6 +400,7 @@ const syncSecretsGCPSecretManager = async ({ } } )).data; + res[key] = Buffer.from(secretLatest.payload.data, "base64").toString("utf-8"); } @@ -387,14 +409,16 @@ const syncSecretsGCPSecretManager = async ({ if (!(key in res)) { // case: create secret await standardRequest.post( - `${INTEGRATION_GCP_SECRET_MANAGER_URL}/v1beta1/projects/${integration.appId}/secrets`, + `${INTEGRATION_GCP_SECRET_MANAGER_URL}/v1/projects/${integration.appId}/secrets`, { replication: { automatic: {} }, - labels: { - "managed-by": "infisical" - } + ...(integration.metadata.secretGCPLabel ? { + labels: { + [integration.metadata.secretGCPLabel.labelName]: integration.metadata.secretGCPLabel.labelValue + } + } : {}) }, { params: { @@ -408,7 +432,7 @@ const syncSecretsGCPSecretManager = async ({ ); await standardRequest.post( - `${INTEGRATION_GCP_SECRET_MANAGER_URL}/v1beta1/projects/${integration.appId}/secrets/${key}:addVersion`, + `${INTEGRATION_GCP_SECRET_MANAGER_URL}/v1/projects/${integration.appId}/secrets/${key}:addVersion`, { payload: { data: Buffer.from(secrets[key].value).toString("base64") @@ -428,7 +452,7 @@ const syncSecretsGCPSecretManager = async ({ if (!(key in secrets)) { // case: delete secret await standardRequest.delete( - `${INTEGRATION_GCP_SECRET_MANAGER_URL}/v1beta1/projects/${integration.appId}/secrets/${key}`, + `${INTEGRATION_GCP_SECRET_MANAGER_URL}/v1/projects/${integration.appId}/secrets/${key}`, { headers: { Authorization: `Bearer ${accessToken}`, @@ -440,7 +464,7 @@ const syncSecretsGCPSecretManager = async ({ // case: update secret if (secrets[key].value !== res[key]) { await standardRequest.post( - `${INTEGRATION_GCP_SECRET_MANAGER_URL}/v1beta1/projects/${integration.appId}/secrets/${key}:addVersion`, + `${INTEGRATION_GCP_SECRET_MANAGER_URL}/v1/projects/${integration.appId}/secrets/${key}:addVersion`, { payload: { data: Buffer.from(secrets[key].value).toString("base64") @@ -1863,10 +1887,24 @@ const syncSecretsGitLab = async ({ }; const allEnvVariables = await getAllEnvVariables(integration?.appId, accessToken); - const getSecretsRes: GitLabSecret[] = allEnvVariables.filter( - (secret: GitLabSecret) => secret.environment_scope === integration.targetEnvironment - ); + const getSecretsRes: GitLabSecret[] = allEnvVariables + .filter( + (secret: GitLabSecret) => secret.environment_scope === integration.targetEnvironment + ) + .filter((gitLabSecret) => { + let isValid = true; + if (integration.metadata.secretPrefix && !gitLabSecret.key.startsWith(integration.metadata.secretPrefix)) { + isValid = false; + } + + if (integration.metadata.secretSuffix && !gitLabSecret.key.endsWith(integration.metadata.secretSuffix)) { + isValid = false; + } + + return isValid; + }); + for await (const key of Object.keys(secrets)) { const existingSecret = getSecretsRes.find((s: any) => s.key == key); if (!existingSecret) { diff --git a/backend/src/models/integration/types.ts b/backend/src/models/integration/types.ts index 0415a9556..0c0b998e6 100644 --- a/backend/src/models/integration/types.ts +++ b/backend/src/models/integration/types.ts @@ -1,3 +1,11 @@ + +// TODO: in the future separate metadata +// into distinct types by integration export type Metadata = { + secretPrefix?: string; secretSuffix?: string; + secretGCPLabel?: { + labelName: string; + labelValue: string; + } } \ No newline at end of file diff --git a/backend/src/queues/integrations/syncSecretsToThirdPartyServices.ts b/backend/src/queues/integrations/syncSecretsToThirdPartyServices.ts index d3ffef6e6..7b6819b8c 100644 --- a/backend/src/queues/integrations/syncSecretsToThirdPartyServices.ts +++ b/backend/src/queues/integrations/syncSecretsToThirdPartyServices.ts @@ -35,9 +35,12 @@ syncSecretsToThirdPartyServices.process(async (job: Job) => { }); const suffixedSecrets: any = {}; - if (integration.metadata?.secretSuffix) { + if (integration.metadata) { for (const key in secrets) { - const newKey = key + integration.metadata?.secretSuffix; + const prefix = (integration.metadata?.secretPrefix || ""); + const suffix = (integration.metadata?.secretSuffix || ""); + const newKey = prefix + key + suffix; + suffixedSecrets[newKey] = secrets[key]; } } diff --git a/backend/src/services/RedisService.ts b/backend/src/services/RedisService.ts new file mode 100644 index 000000000..a6ae5b026 --- /dev/null +++ b/backend/src/services/RedisService.ts @@ -0,0 +1,12 @@ +import { Redis } from "ioredis" + +let redisClient: Redis | null; + +if (process.env.REDIS_URL) { + redisClient = new Redis(process.env.REDIS_URL as string); +} else { + console.warn("Redis URL not set, skipping Redis initialization."); + redisClient = null +} + +export { redisClient } \ No newline at end of file diff --git a/backend/src/utils/setup/backfillData.ts b/backend/src/utils/setup/backfillData.ts index ccd38edcf..6a0c58216 100644 --- a/backend/src/utils/setup/backfillData.ts +++ b/backend/src/utils/setup/backfillData.ts @@ -3,6 +3,7 @@ import crypto from "crypto"; import { Types } from "mongoose"; import { encryptSymmetric128BitHexKeyUTF8 } from "../crypto"; import { EESecretService } from "../../ee/services"; +import { redisClient } from "../../services/RedisService" import { IPType, ISecretVersion, SecretSnapshot, SecretVersion, TrustedIP } from "../../ee/models"; import { AuthMethod, @@ -10,6 +11,7 @@ import { Bot, BotOrg, ISecret, + IWorkspace, Integration, IntegrationAuth, Membership, @@ -26,14 +28,20 @@ import { client, getEncryptionKey, getRootEncryptionKey } from "../../config"; import { ADMIN, ALGORITHM_AES_256_GCM, + CUSTOM, ENCODING_SCHEME_BASE64, ENCODING_SCHEME_UTF8, MEMBER, - OWNER, - VIEWER + OWNER } from "../../variables"; import { InternalServerError } from "../errors"; +import { + ProjectPermissionActions, + ProjectPermissionSub, + memberProjectPermissions +} from "../../ee/services/ProjectRoleService"; +import Role from "../../ee/models/role"; /** * Backfill secrets to ensure that they're all versioned and have @@ -679,32 +687,128 @@ export const backfillUserAuthMethods = async () => { }; export const backfillPermission = async () => { - await Membership.updateMany( - { - deniedPermissions: { - $exists: true, - $ne: [] - }, - role: MEMBER - }, - [ - { - $set: { - role: VIEWER + const lockKey = "backfill_permission_lock"; + const timeout = 900000; // 15 min lock timeout in milliseconds + const lock = await redisClient?.set(lockKey, 1, "PX", timeout, "NX"); + + if (lock) { + try { + console.info("Lock acquired for script [backfillPermission]"); + + const memberships = await Membership.find({ + deniedPermissions: { + $exists: true, + $ne: [] + }, + role: MEMBER, + }) + .populate<{ workspace: IWorkspace }>("workspace") + .lean(); + + // group memberships that need the same permission set + const roleMap = new Map(); + + for (const membership of memberships) { + // get permissions of members except secret permission + const customPermissions = memberProjectPermissions.rules.filter( + ({ subject }) => subject !== ProjectPermissionSub.Secrets + ); + const secretAccessRule: Record = {}; + + // iterate and record true and false ones + membership.deniedPermissions.forEach(({ ability, environmentSlug }) => { + if (!secretAccessRule?.[environmentSlug]) + secretAccessRule[environmentSlug] = { read: true, write: true }; + if (ability === "write") secretAccessRule[environmentSlug].write = false; + if (ability === "read") secretAccessRule[environmentSlug].read = false; + }); + + const secretPermissions: any = []; + Object.entries(secretAccessRule).forEach(([envSlug, { read, write }]) => { + if (read) { + secretPermissions.push({ + subject: ProjectPermissionSub.Secrets, + action: ProjectPermissionActions.Read, + conditions: { environment: envSlug } + }); + } + if (write) { + secretPermissions.push( + { + subject: ProjectPermissionSub.Secrets, + action: ProjectPermissionActions.Edit, + conditions: { environment: envSlug } + }, + { + subject: ProjectPermissionSub.Secrets, + action: ProjectPermissionActions.Delete, + conditions: { environment: envSlug } + }, + { + subject: ProjectPermissionSub.Secrets, + action: ProjectPermissionActions.Create, + conditions: { environment: envSlug } + } + ); + } + }); + + const key = `${JSON.stringify(secretPermissions)}-${membership.workspace._id.toString()}`; // group roles that have same permission with in the same workspace + const value = roleMap.get(key); + if (value) { + value.membershipIds.push(membership._id.toString()); + value.organizationId = membership.workspace.organization.toString() + value.workspaceId = membership.workspace._id.toString() + } else { + roleMap.set(key, { membershipIds: [membership._id.toString()], permissions: [...customPermissions, ...secretPermissions], organizationId: membership.workspace.organization.toString(), workspaceId: membership.workspace._id.toString() }); } } - ] - ); - await MembershipOrg.updateMany( - { - role: OWNER - }, - { - $set: { - role: ADMIN + for (const [key, value] of roleMap.entries()) { + const { membershipIds, permissions, workspaceId, organizationId } = value + const membership_identity = crypto.randomBytes(3).toString("hex") + const role = new Role({ + name: `Migrated Role [${membership_identity.toUpperCase()}]`, + organization: organizationId, + workspace: workspaceId, + description: "This role was auto generated by Infisical in effort to migrate your project members to our new permission system", + isOrgRole: false, + slug: `custom-role-${membership_identity}`, + permissions: permissions + }); + + await role.save(); + + for (const id of membershipIds) { + await Membership.findByIdAndUpdate(id, { // document db doesn't support update many so we must loop + $set: { + role: CUSTOM, + customRole: role + } + }); + } } + + console.info("Backfill: Finished converting old denied permission in workspace to viewers"); + + await MembershipOrg.updateMany( + { + role: OWNER + }, + { + $set: { + role: ADMIN + } + } + ); + + console.info("Backfill: Finished converting owner role to member"); + + } catch (error) { + console.error("An error occurred when running script [backfillPermission]:", error); } - ); - console.log("Backfill: Finishing converting old denied permission in workspace to viewers"); + + } else { + console.info("Could not acquire lock for script [backfillPermission], skipping"); + } }; diff --git a/backend/src/validation/integration.ts b/backend/src/validation/integration.ts index dfe1ed505..7795b0084 100644 --- a/backend/src/validation/integration.ts +++ b/backend/src/validation/integration.ts @@ -77,7 +77,12 @@ export const CreateIntegrationV1 = z.object({ path: z.string().trim().optional(), region: z.string().trim().optional(), metadata: z.object({ - secretSuffix: z.string().optional() + secretPrefix: z.string().optional(), + secretSuffix: z.string().optional(), + secretGCPLabel: z.object({ + labelName: z.string(), + labelValue: z.string() + }).optional() }).optional() }) }); diff --git a/docs/images/integrations/gcp-secret-manager/integrations-gcp-secret-manager-create-options.png b/docs/images/integrations/gcp-secret-manager/integrations-gcp-secret-manager-create-options.png new file mode 100644 index 000000000..d084ebb6a Binary files /dev/null and b/docs/images/integrations/gcp-secret-manager/integrations-gcp-secret-manager-create-options.png differ diff --git a/docs/images/integrations/gcp-secret-manager/integrations-gcp-secret-manager-create.png b/docs/images/integrations/gcp-secret-manager/integrations-gcp-secret-manager-create.png index 9e1719a59..afc4724cb 100644 Binary files a/docs/images/integrations/gcp-secret-manager/integrations-gcp-secret-manager-create.png and b/docs/images/integrations/gcp-secret-manager/integrations-gcp-secret-manager-create.png differ diff --git a/docs/images/integrations/gitlab/integrations-gitlab-create-options.png b/docs/images/integrations/gitlab/integrations-gitlab-create-options.png new file mode 100644 index 000000000..abb507997 Binary files /dev/null and b/docs/images/integrations/gitlab/integrations-gitlab-create-options.png differ diff --git a/docs/images/integrations/gitlab/integrations-gitlab-create.png b/docs/images/integrations/gitlab/integrations-gitlab-create.png index 4d13658c7..ae3ca7141 100644 Binary files a/docs/images/integrations/gitlab/integrations-gitlab-create.png and b/docs/images/integrations/gitlab/integrations-gitlab-create.png differ diff --git a/docs/integrations/cicd/gitlab.mdx b/docs/integrations/cicd/gitlab.mdx index d7b2637c6..9c3df9720 100644 --- a/docs/integrations/cicd/gitlab.mdx +++ b/docs/integrations/cicd/gitlab.mdx @@ -32,6 +32,16 @@ Press on the GitLab tile and grant Infisical access to your GitLab account. Select which Infisical environment secrets you want to sync to which GitLab repository and press create integration to start syncing secrets to GitLab. ![integrations gitlab](../../images/integrations/gitlab/integrations-gitlab-create.png) + +Note that the GitLab integration supports a few options in the **Options** tab: + +- Secret Prefix: If inputted, the prefix is appended to the front of every secret name prior to being synced. +- Secret Suffix: If inputted, the suffix to appended to the back of every name of every secret prior to being synced. + +Setting a secret prefix or suffix ensures that existing secrets in GCP Secret Manager are not overwritten during the sync. As part of this process, Infisical abstains from mutating any secrets in GitLab without the specified prefix or suffix. + +![integrations gitlab options](../../images/integrations/gitlab/integrations-gitlab-create-options.png) + ![integrations gitlab](../../images/integrations/gitlab/integrations-gitlab.png) diff --git a/docs/integrations/cloud/gcp-secret-manager.mdx b/docs/integrations/cloud/gcp-secret-manager.mdx index 3c1863006..505475009 100644 --- a/docs/integrations/cloud/gcp-secret-manager.mdx +++ b/docs/integrations/cloud/gcp-secret-manager.mdx @@ -35,14 +35,21 @@ Grant Infisical access to GCP. ## Start integration -Select which Infisical environment secrets you want to sync to which GCP secret manager project. Lastly, press create integration to start syncing secrets to GCP secret manager. +In the **Connection** tab, select which Infisical environment secrets you want to sync to which GCP secret manager project. Lastly, press create integration to start syncing secrets to GCP secret manager. ![integrations GCP secret manager](../../images/integrations/gcp-secret-manager/integrations-gcp-secret-manager-create.png) -![integrations GCP secret manager](../../images/integrations/gcp-secret-manager/integrations-gcp-secret-manager.png) - - Secrets synced from Infisical to GCP Secret Manager are automatically labeled `managed-by:infisical` to avoid overwriting existing values in GCP Secret Manager. - +Note that the GCP Secret Manager integration supports a few options in the **Options** tab: + +- Secret Prefix: If inputted, the prefix is appended to the front of every secret name prior to being synced. +- Secret Suffix: If inputted, the suffix to appended to the back of every name of every secret prior to being synced. +- Label in GCP Secret Manager: If selected, every secret will be labeled in GCP Secret Manager (e.g. as `managed-by:infisical`); labels can be customized. + +Setting a secret prefix, suffix, or enabling the labeling option ensures that existing secrets in GCP Secret Manager are not overwritten during the sync. As part of this process, Infisical abstains from mutating any secrets in GCP Secret Manager without the specified prefix, suffix, or attached label. + +![integrations GCP secret manager options](../../images/integrations/gcp-secret-manager/integrations-gcp-secret-manager-create-options.png) + +![integrations GCP secret manager](../../images/integrations/gcp-secret-manager/integrations-gcp-secret-manager.png) Using Infisical to sync secrets to GCP Secret Manager requires that you enable @@ -89,14 +96,21 @@ service account in IAM & Admin > Service Accounts > Service Account > Keys). ## Start integration -Select which Infisical environment secrets you want to sync to the GCP secret manager project. Lastly, press create integration to start syncing secrets to GCP secret manager. +In the **Connection** tab, select which Infisical environment secrets you want to sync to the GCP secret manager project. Lastly, press create integration to start syncing secrets to GCP secret manager. ![integrations GCP secret manager](../../images/integrations/gcp-secret-manager/integrations-gcp-secret-manager-create.png) -![integrations GCP secret manager](../../images/integrations/gcp-secret-manager/integrations-gcp-secret-manager.png) - - Secrets synced from Infisical to GCP Secret Manager are automatically labeled `managed-by:infisical` to avoid overwriting existing values in GCP Secret Manager. - +Note that the GCP Secret Manager integration supports a few options in the **Options** tab: + +- Secret Prefix: If inputted, the prefix is appended to the front of every secret name prior to being synced. +- Secret Suffix: If inputted, the suffix to appended to the back of every name of every secret prior to being synced. +- Label in GCP Secret Manager: If selected, every secret will be labeled in GCP Secret Manager (e.g. as `managed-by:infisical`); labels can be customized. + +Setting a secret prefix, suffix, or enabling the labeling option ensures that existing secrets in GCP Secret Manager are not overwritten during the sync. As part of this process, Infisical abstains from mutating any secrets in GCP Secret Manager without the specified prefix, suffix, or attached label. + +![integrations GCP secret manager options](../../images/integrations/gcp-secret-manager/integrations-gcp-secret-manager-create-options.png) + +![integrations GCP secret manager](../../images/integrations/gcp-secret-manager/integrations-gcp-secret-manager.png) Using Infisical to sync secrets to GCP Secret Manager requires that you enable diff --git a/frontend/src/components/utilities/parseDotEnv.ts b/frontend/src/components/utilities/parseDotEnv.ts index 683670b33..bec3a506b 100644 --- a/frontend/src/components/utilities/parseDotEnv.ts +++ b/frontend/src/components/utilities/parseDotEnv.ts @@ -1,5 +1,5 @@ const LINE = - /(?:^|^)\s*(?:export\s+)?([\w.-:]+)(?:\s*=\s*?|:\s+?)(\s*'(?:\\'|[^'])*'|\s*"(?:\\"|[^"])*"|\s*`(?:\\`|[^`])*`|[^#\r\n]+)?\s*(?:#.*)?(?:$|$)/gm; + /(?:^|^)\s*(?:export\s+)?([\w.:-]+)(?:\s*=\s*?|:\s+?)(\s*'(?:\\'|[^'])*'|\s*"(?:\\"|[^"])*"|\s*`(?:\\`|[^`])*`|[^#\r\n]+)?\s*(?:#.*)?(?:$|$)/gm; /** * Return text that is the buffer parsed diff --git a/frontend/src/pages/integrations/gcp-secret-manager/authorize.tsx b/frontend/src/pages/integrations/gcp-secret-manager/authorize.tsx index 0e0b1bd2f..d38c09887 100644 --- a/frontend/src/pages/integrations/gcp-secret-manager/authorize.tsx +++ b/frontend/src/pages/integrations/gcp-secret-manager/authorize.tsx @@ -43,7 +43,7 @@ export default function GCPSecretManagerAuthorizeIntegrationPage() { setIsLoading(false); - router.push(`/integrations/gcp-secret-manager/pat/create?integrationAuthId=${integrationAuth._id}`); + router.push(`/integrations/gcp-secret-manager/create?integrationAuthId=${integrationAuth._id}`); } catch (err) { console.error(err); } diff --git a/frontend/src/pages/integrations/gcp-secret-manager/pat/create.tsx b/frontend/src/pages/integrations/gcp-secret-manager/pat/create.tsx deleted file mode 100644 index 3bc98b615..000000000 --- a/frontend/src/pages/integrations/gcp-secret-manager/pat/create.tsx +++ /dev/null @@ -1,146 +0,0 @@ -import { useEffect, useState } from "react"; -import { useRouter } from "next/router"; -import queryString from "query-string"; - -import { - Button, - Card, - CardTitle, - FormControl, - Input, - Select, - SelectItem -} from "@app/components/v2"; -import { - useCreateIntegration -} from "@app/hooks/api"; -import { useGetIntegrationAuthApps,useGetIntegrationAuthById } from "@app/hooks/api/integrationAuth"; -import { useGetWorkspaceById } from "@app/hooks/api/workspace"; - -export default function GCPSecretManagerCreateIntegrationPage() { - const router = useRouter(); - const { mutateAsync } = useCreateIntegration(); - - const { integrationAuthId } = queryString.parse(router.asPath.split("?")[1]); - - const { data: workspace } = useGetWorkspaceById(localStorage.getItem("projectData.id") ?? ""); - const { data: integrationAuth } = useGetIntegrationAuthById((integrationAuthId as string) ?? ""); - const { data: integrationAuthApps } = useGetIntegrationAuthApps({ - integrationAuthId: (integrationAuthId as string) ?? "" - }); - - const [selectedSourceEnvironment, setSelectedSourceEnvironment] = useState(""); - const [targetAppId, setTargetAppId] = useState(""); - const [secretPath, setSecretPath] = useState("/"); - - const [isLoading, setIsLoading] = useState(false); - - useEffect(() => { - if (workspace) { - setSelectedSourceEnvironment(workspace.environments[0].slug); - } - }, [workspace]); - - useEffect(() => { - if (integrationAuthApps) { - if (integrationAuthApps.length > 0) { - setTargetAppId(integrationAuthApps[0].appId as string); - } else { - setTargetAppId("none"); - } - } - }, [integrationAuthApps]); - - const handleButtonClick = async () => { - try { - setIsLoading(true); - - if (!integrationAuth?._id) return; - - await mutateAsync({ - integrationAuthId: integrationAuth?._id, - isActive: true, - app: integrationAuthApps?.find((integrationAuthApp) => integrationAuthApp.appId === targetAppId)?.name, - appId: targetAppId, - sourceEnvironment: selectedSourceEnvironment, - secretPath - }); - - setIsLoading(false); - router.push(`/integrations/${localStorage.getItem("projectData.id")}`); - } catch (err) { - console.error(err); - } - }; - - return integrationAuth && - workspace && - selectedSourceEnvironment && - integrationAuthApps - ? ( -
- - GCP Secret Manager Integration - - - - - setSecretPath(evt.target.value)} - placeholder="Provide a path, default is /" - /> - - - - - - -
- ) : ( -
- ); -} - -GCPSecretManagerCreateIntegrationPage.requireAuth = true; diff --git a/frontend/src/pages/integrations/gitlab/create.tsx b/frontend/src/pages/integrations/gitlab/create.tsx index 2c6d27e5a..270df3e0d 100644 --- a/frontend/src/pages/integrations/gitlab/create.tsx +++ b/frontend/src/pages/integrations/gitlab/create.tsx @@ -85,6 +85,7 @@ export default function GitLabCreateIntegrationPage() { const { data: integrationAuth } = useGetIntegrationAuthById((integrationAuthId as string) ?? ""); const { data: integrationAuthApps, isLoading: isintegrationAuthAppsLoading } = useGetIntegrationAuthApps({ + integrationAuthId: (integrationAuthId as string) ?? "", ...(targetTeamId ? { teamId: targetTeamId } : {}) });