Merge branch 'main' into integrations-uiux-updates

.github/workflows/build-staging-img.yml

@@ -11,9 +11,9 @@ jobs:
       - name: 📦 Install dependencies to test all dependencies
         run: npm ci --only-production
         working-directory: backend
-      - name: 🧪 Run tests
+      # - name: 🧪 Run tests
-        run: npm run test:ci
+      #   run: npm run test:ci
-        working-directory: backend
+      #   working-directory: backend
       - name: Save commit hashes for tag
         id: commit
         uses: pr-mpt/actions-commit-hash@v2

backend/package-lock.json

@@ -37,6 +37,7 @@
         "handlebars": "^4.7.7",
         "helmet": "^5.1.1",
         "infisical-node": "^1.2.1",
+        "ioredis": "^5.3.2",
         "js-yaml": "^4.1.0",
         "jsonwebtoken": "^9.0.0",
         "jsrp": "^0.2.4",
@@ -3508,8 +3509,7 @@
     "node_modules/@ioredis/commands": {
       "version": "1.2.0",
       "resolved": "https://registry.npmjs.org/@ioredis/commands/-/commands-1.2.0.tgz",
-      "integrity": "sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg==",
+      "integrity": "sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg=="
-      "dev": true
     },
     "node_modules/@istanbuljs/load-nyc-config": {
       "version": "1.1.0",
@@ -7031,39 +7031,6 @@
         "node": ">=12"
       }
     },
-    "node_modules/bull/node_modules/denque": {
-      "version": "2.1.0",
-      "resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz",
-      "integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==",
-      "dev": true,
-      "engines": {
-        "node": ">=0.10"
-      }
-    },
-    "node_modules/bull/node_modules/ioredis": {
-      "version": "5.3.2",
-      "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.3.2.tgz",
-      "integrity": "sha512-1DKMMzlIHM02eBBVOFQ1+AolGjs6+xEcM4PDL7NqOS6szq7H9jSaEkIUH6/a5Hl241LzW6JLSiAbNvTQjUupUA==",
-      "dev": true,
-      "dependencies": {
-        "@ioredis/commands": "^1.1.1",
-        "cluster-key-slot": "^1.1.0",
-        "debug": "^4.3.4",
-        "denque": "^2.1.0",
-        "lodash.defaults": "^4.2.0",
-        "lodash.isarguments": "^3.1.0",
-        "redis-errors": "^1.2.0",
-        "redis-parser": "^3.0.0",
-        "standard-as-callback": "^2.1.0"
-      },
-      "engines": {
-        "node": ">=12.22.0"
-      },
-      "funding": {
-        "type": "opencollective",
-        "url": "https://opencollective.com/ioredis"
-      }
-    },
     "node_modules/bytes": {
       "version": "3.1.2",
       "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz",
@@ -9071,30 +9038,36 @@
       }
     },
     "node_modules/ioredis": {
-      "version": "4.28.5",
+      "version": "5.3.2",
-      "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-4.28.5.tgz",
+      "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.3.2.tgz",
-      "integrity": "sha512-3GYo0GJtLqgNXj4YhrisLaNNvWSNwSS2wS4OELGfGxH8I69+XfNdnmV1AyN+ZqMh0i7eX+SWjrwFKDBDgfBC1A==",
+      "integrity": "sha512-1DKMMzlIHM02eBBVOFQ1+AolGjs6+xEcM4PDL7NqOS6szq7H9jSaEkIUH6/a5Hl241LzW6JLSiAbNvTQjUupUA==",
       "dependencies": {
+        "@ioredis/commands": "^1.1.1",
         "cluster-key-slot": "^1.1.0",
-        "debug": "^4.3.1",
+        "debug": "^4.3.4",
-        "denque": "^1.1.0",
+        "denque": "^2.1.0",
         "lodash.defaults": "^4.2.0",
-        "lodash.flatten": "^4.4.0",
         "lodash.isarguments": "^3.1.0",
-        "p-map": "^2.1.0",
-        "redis-commands": "1.7.0",
         "redis-errors": "^1.2.0",
         "redis-parser": "^3.0.0",
         "standard-as-callback": "^2.1.0"
       },
       "engines": {
-        "node": ">=6"
+        "node": ">=12.22.0"
       },
       "funding": {
         "type": "opencollective",
         "url": "https://opencollective.com/ioredis"
       }
     },
+    "node_modules/ioredis/node_modules/denque": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz",
+      "integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==",
+      "engines": {
+        "node": ">=0.10"
+      }
+    },
     "node_modules/ip": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/ip/-/ip-2.0.0.tgz",
@@ -14445,6 +14418,31 @@
         "node": ">=10"
       }
     },
+    "node_modules/probot/node_modules/ioredis": {
+      "version": "4.28.5",
+      "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-4.28.5.tgz",
+      "integrity": "sha512-3GYo0GJtLqgNXj4YhrisLaNNvWSNwSS2wS4OELGfGxH8I69+XfNdnmV1AyN+ZqMh0i7eX+SWjrwFKDBDgfBC1A==",
+      "dependencies": {
+        "cluster-key-slot": "^1.1.0",
+        "debug": "^4.3.1",
+        "denque": "^1.1.0",
+        "lodash.defaults": "^4.2.0",
+        "lodash.flatten": "^4.4.0",
+        "lodash.isarguments": "^3.1.0",
+        "p-map": "^2.1.0",
+        "redis-commands": "1.7.0",
+        "redis-errors": "^1.2.0",
+        "redis-parser": "^3.0.0",
+        "standard-as-callback": "^2.1.0"
+      },
+      "engines": {
+        "node": ">=6"
+      },
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/ioredis"
+      }
+    },
     "node_modules/probot/node_modules/js-yaml": {
       "version": "3.14.1",
       "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.1.tgz",
@@ -19517,8 +19515,7 @@
     "@ioredis/commands": {
       "version": "1.2.0",
       "resolved": "https://registry.npmjs.org/@ioredis/commands/-/commands-1.2.0.tgz",
-      "integrity": "sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg==",
+      "integrity": "sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg=="
-      "dev": true
     },
     "@istanbuljs/load-nyc-config": {
       "version": "1.1.0",
@@ -22302,31 +22299,6 @@
         "msgpackr": "^1.5.2",
         "semver": "^7.3.2",
         "uuid": "^8.3.0"
-      },
-      "dependencies": {
-        "denque": {
-          "version": "2.1.0",
-          "resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz",
-          "integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==",
-          "dev": true
-        },
-        "ioredis": {
-          "version": "5.3.2",
-          "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.3.2.tgz",
-          "integrity": "sha512-1DKMMzlIHM02eBBVOFQ1+AolGjs6+xEcM4PDL7NqOS6szq7H9jSaEkIUH6/a5Hl241LzW6JLSiAbNvTQjUupUA==",
-          "dev": true,
-          "requires": {
-            "@ioredis/commands": "^1.1.1",
-            "cluster-key-slot": "^1.1.0",
-            "debug": "^4.3.4",
-            "denque": "^2.1.0",
-            "lodash.defaults": "^4.2.0",
-            "lodash.isarguments": "^3.1.0",
-            "redis-errors": "^1.2.0",
-            "redis-parser": "^3.0.0",
-            "standard-as-callback": "^2.1.0"
-          }
-        }
       }
     },
     "bytes": {
@@ -23809,21 +23781,26 @@
       "dev": true
     },
     "ioredis": {
-      "version": "4.28.5",
+      "version": "5.3.2",
-      "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-4.28.5.tgz",
+      "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.3.2.tgz",
-      "integrity": "sha512-3GYo0GJtLqgNXj4YhrisLaNNvWSNwSS2wS4OELGfGxH8I69+XfNdnmV1AyN+ZqMh0i7eX+SWjrwFKDBDgfBC1A==",
+      "integrity": "sha512-1DKMMzlIHM02eBBVOFQ1+AolGjs6+xEcM4PDL7NqOS6szq7H9jSaEkIUH6/a5Hl241LzW6JLSiAbNvTQjUupUA==",
       "requires": {
+        "@ioredis/commands": "^1.1.1",
         "cluster-key-slot": "^1.1.0",
-        "debug": "^4.3.1",
+        "debug": "^4.3.4",
-        "denque": "^1.1.0",
+        "denque": "^2.1.0",
         "lodash.defaults": "^4.2.0",
-        "lodash.flatten": "^4.4.0",
         "lodash.isarguments": "^3.1.0",
-        "p-map": "^2.1.0",
-        "redis-commands": "1.7.0",
         "redis-errors": "^1.2.0",
         "redis-parser": "^3.0.0",
         "standard-as-callback": "^2.1.0"
+      },
+      "dependencies": {
+        "denque": {
+          "version": "2.1.0",
+          "resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz",
+          "integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw=="
+        }
       }
     },
     "ip": {
@@ -27791,6 +27768,24 @@
           "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-8.6.0.tgz",
           "integrity": "sha512-IrPdXQsk2BbzvCBGBOTmmSH5SodmqZNt4ERAZDmW4CT+tL8VtvinqywuANaFu4bOMWki16nqf0e4oC0QIaDr/g=="
         },
+        "ioredis": {
+          "version": "4.28.5",
+          "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-4.28.5.tgz",
+          "integrity": "sha512-3GYo0GJtLqgNXj4YhrisLaNNvWSNwSS2wS4OELGfGxH8I69+XfNdnmV1AyN+ZqMh0i7eX+SWjrwFKDBDgfBC1A==",
+          "requires": {
+            "cluster-key-slot": "^1.1.0",
+            "debug": "^4.3.1",
+            "denque": "^1.1.0",
+            "lodash.defaults": "^4.2.0",
+            "lodash.flatten": "^4.4.0",
+            "lodash.isarguments": "^3.1.0",
+            "p-map": "^2.1.0",
+            "redis-commands": "1.7.0",
+            "redis-errors": "^1.2.0",
+            "redis-parser": "^3.0.0",
+            "standard-as-callback": "^2.1.0"
+          }
+        },
         "js-yaml": {
           "version": "3.14.1",
           "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.1.tgz",

backend/package.json

@@ -28,6 +28,7 @@
     "handlebars": "^4.7.7",
     "helmet": "^5.1.1",
     "infisical-node": "^1.2.1",
+    "ioredis": "^5.3.2",
     "js-yaml": "^4.1.0",
     "jsonwebtoken": "^9.0.0",
     "jsrp": "^0.2.4",

backend/src/controllers/v1/membershipController.ts

@@ -148,7 +148,12 @@ export const changeMembershipRole = async (req: Request, res: Response) => {
   const membership = await Membership.findByIdAndUpdate(
     membershipId,
     {
-      role
+      $set: {
+        role
+      },
+      $unset: {
+        customRole: 1
+      }
     },
     {
       new: true

backend/src/controllers/v2/organizationsController.ts

@@ -159,7 +159,12 @@ export const updateOrganizationMembership = async (req: Request, res: Response)
   const membership = await MembershipOrg.findByIdAndUpdate(
     membershipId,
     {
-      role
+      $set: {
+        role
+      },
+      $unset: {
+        customRole: 1
+      }
     },
     {
       new: true

backend/src/integrations/sync.ts

@@ -328,15 +328,19 @@ const syncSecretsGCPSecretManager = async ({
   const pageSize = 100;
   let pageToken: string | undefined;
   let hasMorePages = true;
+
+  const filterParam = integration.metadata.secretGCPLabel 
+    ? `?filter=labels.${integration.metadata.secretGCPLabel.labelName}=${integration.metadata.secretGCPLabel.labelValue}` 
+    : "";
   
   while (hasMorePages) {
     const params = new URLSearchParams({
       pageSize: String(pageSize),
       ...(pageToken ? { pageToken } : {})
     });
-    
+
     const res: GCPSMListSecretsRes = (await standardRequest.get(
-      `${INTEGRATION_GCP_SECRET_MANAGER_URL}/v1/projects/${integration.appId}/secrets?filter=labels.managed-by=infisical`,
+      `${INTEGRATION_GCP_SECRET_MANAGER_URL}/v1/projects/${integration.appId}/secrets${filterParam}`,
       {
         params,
         headers: {
@@ -347,7 +351,24 @@ const syncSecretsGCPSecretManager = async ({
     )).data;
     
     if (res.secrets) {
-      gcpSecrets = gcpSecrets.concat(res.secrets);
+      const filteredSecrets = res.secrets?.filter((gcpSecret) => {
+        const arr = gcpSecret.name.split("/");
+        const key = arr[arr.length - 1];
+
+        let isValid = true;
+
+        if (integration.metadata.secretPrefix && !key.startsWith(integration.metadata.secretPrefix)) {
+          isValid = false;
+        }
+
+        if (integration.metadata.secretSuffix && !key.endsWith(integration.metadata.secretSuffix)) {
+          isValid = false;
+        }
+        
+        return isValid;
+      });
+
+      gcpSecrets = gcpSecrets.concat(filteredSecrets);
     }
     
     if (!res.nextPageToken) {
@@ -371,7 +392,7 @@ const syncSecretsGCPSecretManager = async ({
     const key = arr[arr.length - 1];
 
     const secretLatest: GCPLatestSecretVersionAccess = (await standardRequest.get(
-      `${INTEGRATION_GCP_SECRET_MANAGER_URL}/v1beta1/projects/${integration.appId}/secrets/${key}/versions/latest:access`,
+      `${INTEGRATION_GCP_SECRET_MANAGER_URL}/v1/projects/${integration.appId}/secrets/${key}/versions/latest:access`,
       {
         headers: {
           Authorization: `Bearer ${accessToken}`,
@@ -379,6 +400,7 @@ const syncSecretsGCPSecretManager = async ({
         }
       }
     )).data;
+
     
     res[key] = Buffer.from(secretLatest.payload.data, "base64").toString("utf-8");
   }
@@ -387,14 +409,16 @@ const syncSecretsGCPSecretManager = async ({
     if (!(key in res)) {
       // case: create secret
       await standardRequest.post(
-        `${INTEGRATION_GCP_SECRET_MANAGER_URL}/v1beta1/projects/${integration.appId}/secrets`,
+        `${INTEGRATION_GCP_SECRET_MANAGER_URL}/v1/projects/${integration.appId}/secrets`,
         {
           replication: {
             automatic: {}
           },
-          labels: {
+          ...(integration.metadata.secretGCPLabel ? {
-            "managed-by": "infisical"
+            labels: {
-          }
+              [integration.metadata.secretGCPLabel.labelName]: integration.metadata.secretGCPLabel.labelValue
+            }
+          } : {})
         },
         {
           params: {
@@ -408,7 +432,7 @@ const syncSecretsGCPSecretManager = async ({
       );
       
       await standardRequest.post(
-        `${INTEGRATION_GCP_SECRET_MANAGER_URL}/v1beta1/projects/${integration.appId}/secrets/${key}:addVersion`,
+        `${INTEGRATION_GCP_SECRET_MANAGER_URL}/v1/projects/${integration.appId}/secrets/${key}:addVersion`,
         {
           payload: {
             data: Buffer.from(secrets[key].value).toString("base64")
@@ -428,7 +452,7 @@ const syncSecretsGCPSecretManager = async ({
     if (!(key in secrets)) {
       // case: delete secret
       await standardRequest.delete(
-        `${INTEGRATION_GCP_SECRET_MANAGER_URL}/v1beta1/projects/${integration.appId}/secrets/${key}`,
+        `${INTEGRATION_GCP_SECRET_MANAGER_URL}/v1/projects/${integration.appId}/secrets/${key}`,
         {
           headers: {
             Authorization: `Bearer ${accessToken}`,
@@ -440,7 +464,7 @@ const syncSecretsGCPSecretManager = async ({
       // case: update secret
       if (secrets[key].value !== res[key]) {
         await standardRequest.post(
-          `${INTEGRATION_GCP_SECRET_MANAGER_URL}/v1beta1/projects/${integration.appId}/secrets/${key}:addVersion`,
+          `${INTEGRATION_GCP_SECRET_MANAGER_URL}/v1/projects/${integration.appId}/secrets/${key}:addVersion`,
           {
             payload: {
               data: Buffer.from(secrets[key].value).toString("base64")
@@ -1863,10 +1887,24 @@ const syncSecretsGitLab = async ({
   };
 
   const allEnvVariables = await getAllEnvVariables(integration?.appId, accessToken);
-  const getSecretsRes: GitLabSecret[] = allEnvVariables.filter(
+  const getSecretsRes: GitLabSecret[] = allEnvVariables
-    (secret: GitLabSecret) => secret.environment_scope === integration.targetEnvironment
+    .filter(
-  );
+      (secret: GitLabSecret) => secret.environment_scope === integration.targetEnvironment
+    )
+    .filter((gitLabSecret) => {
+      let isValid = true;
 
+      if (integration.metadata.secretPrefix && !gitLabSecret.key.startsWith(integration.metadata.secretPrefix)) {
+        isValid = false;
+      }
+
+      if (integration.metadata.secretSuffix && !gitLabSecret.key.endsWith(integration.metadata.secretSuffix)) {
+        isValid = false;
+      }
+      
+      return isValid;
+    });
+  
   for await (const key of Object.keys(secrets)) {
     const existingSecret = getSecretsRes.find((s: any) => s.key == key);
     if (!existingSecret) {

backend/src/models/integration/types.ts

@@ -1,3 +1,11 @@
+
+// TODO: in the future separate metadata 
+// into distinct types by integration
 export type Metadata = {
+    secretPrefix?: string;
     secretSuffix?: string;
+    secretGCPLabel?: {
+        labelName: string;
+        labelValue: string;
+    }
 }

backend/src/queues/integrations/syncSecretsToThirdPartyServices.ts

@@ -35,9 +35,12 @@ syncSecretsToThirdPartyServices.process(async (job: Job) => {
     });
 
     const suffixedSecrets: any = {};
-    if (integration.metadata?.secretSuffix) {
+    if (integration.metadata) {
       for (const key in secrets) {
-        const newKey = key + integration.metadata?.secretSuffix;
+        const prefix = (integration.metadata?.secretPrefix || "");
+        const suffix = (integration.metadata?.secretSuffix || "");
+        const newKey = prefix + key + suffix;
+        
         suffixedSecrets[newKey] = secrets[key];
       }      
     }

backend/src/services/RedisService.ts

@@ -0,0 +1,12 @@
+import { Redis } from "ioredis"
+
+let redisClient: Redis | null;
+
+if (process.env.REDIS_URL) {
+  redisClient = new Redis(process.env.REDIS_URL as string);
+} else {
+  console.warn("Redis URL not set, skipping Redis initialization.");
+  redisClient = null
+}
+
+export { redisClient }

backend/src/utils/setup/backfillData.ts

@@ -3,6 +3,7 @@ import crypto from "crypto";
 import { Types } from "mongoose";
 import { encryptSymmetric128BitHexKeyUTF8 } from "../crypto";
 import { EESecretService } from "../../ee/services";
+import { redisClient } from "../../services/RedisService"
 import { IPType, ISecretVersion, SecretSnapshot, SecretVersion, TrustedIP } from "../../ee/models";
 import {
   AuthMethod,
@@ -10,6 +11,7 @@ import {
   Bot,
   BotOrg,
   ISecret,
+  IWorkspace,
   Integration,
   IntegrationAuth,
   Membership,
@@ -26,14 +28,20 @@ import { client, getEncryptionKey, getRootEncryptionKey } from "../../config";
 import {
   ADMIN,
   ALGORITHM_AES_256_GCM,
+  CUSTOM,
   ENCODING_SCHEME_BASE64,
   ENCODING_SCHEME_UTF8,
   MEMBER,
-  OWNER,
+  OWNER
-  VIEWER
 } from "../../variables";
 
 import { InternalServerError } from "../errors";
+import {
+  ProjectPermissionActions,
+  ProjectPermissionSub,
+  memberProjectPermissions
+} from "../../ee/services/ProjectRoleService";
+import Role from "../../ee/models/role";
 
 /**
  * Backfill secrets to ensure that they're all versioned and have
@@ -679,32 +687,128 @@ export const backfillUserAuthMethods = async () => {
 };
 
 export const backfillPermission = async () => {
-  await Membership.updateMany(
+  const lockKey = "backfill_permission_lock";
-    {
+  const timeout = 900000; // 15 min lock timeout in milliseconds 
-      deniedPermissions: {
+  const lock = await redisClient?.set(lockKey, 1, "PX", timeout, "NX");
-        $exists: true,
+
-        $ne: []
+  if (lock) {
-      },
+    try {
-      role: MEMBER
+      console.info("Lock acquired for script [backfillPermission]");
-    },
+
-    [
+      const memberships = await Membership.find({
-      {
+        deniedPermissions: {
-        $set: {
+          $exists: true,
-          role: VIEWER
+          $ne: []
+        },
+        role: MEMBER,
+      })
+        .populate<{ workspace: IWorkspace }>("workspace")
+        .lean();
+
+      // group memberships that need the same permission set
+      const roleMap = new Map<string, { membershipIds: string[], permissions: any[], organizationId: string, workspaceId: string }>();
+
+      for (const membership of memberships) {
+        // get permissions of members except secret permission
+        const customPermissions = memberProjectPermissions.rules.filter(
+          ({ subject }) => subject !== ProjectPermissionSub.Secrets
+        );
+        const secretAccessRule: Record<string, { read: boolean; write: boolean }> = {};
+
+        // iterate and record true and false ones
+        membership.deniedPermissions.forEach(({ ability, environmentSlug }) => {
+          if (!secretAccessRule?.[environmentSlug])
+            secretAccessRule[environmentSlug] = { read: true, write: true };
+          if (ability === "write") secretAccessRule[environmentSlug].write = false;
+          if (ability === "read") secretAccessRule[environmentSlug].read = false;
+        });
+
+        const secretPermissions: any = [];
+        Object.entries(secretAccessRule).forEach(([envSlug, { read, write }]) => {
+          if (read) {
+            secretPermissions.push({
+              subject: ProjectPermissionSub.Secrets,
+              action: ProjectPermissionActions.Read,
+              conditions: { environment: envSlug }
+            });
+          }
+          if (write) {
+            secretPermissions.push(
+              {
+                subject: ProjectPermissionSub.Secrets,
+                action: ProjectPermissionActions.Edit,
+                conditions: { environment: envSlug }
+              },
+              {
+                subject: ProjectPermissionSub.Secrets,
+                action: ProjectPermissionActions.Delete,
+                conditions: { environment: envSlug }
+              },
+              {
+                subject: ProjectPermissionSub.Secrets,
+                action: ProjectPermissionActions.Create,
+                conditions: { environment: envSlug }
+              }
+            );
+          }
+        });
+
+        const key = `${JSON.stringify(secretPermissions)}-${membership.workspace._id.toString()}`; // group roles that have same permission with in the same workspace
+        const value = roleMap.get(key);
+        if (value) {
+          value.membershipIds.push(membership._id.toString());
+          value.organizationId = membership.workspace.organization.toString()
+          value.workspaceId = membership.workspace._id.toString()
+        } else {
+          roleMap.set(key, { membershipIds: [membership._id.toString()], permissions: [...customPermissions, ...secretPermissions], organizationId: membership.workspace.organization.toString(), workspaceId: membership.workspace._id.toString() });
         }
       }
-    ]
-  );
 
-  await MembershipOrg.updateMany(
+      for (const [key, value] of roleMap.entries()) {
-    {
+        const { membershipIds, permissions, workspaceId, organizationId } = value
-      role: OWNER
+        const membership_identity = crypto.randomBytes(3).toString("hex")
-    },
+        const role = new Role({
-    {
+          name: `Migrated Role [${membership_identity.toUpperCase()}]`,
-      $set: {
+          organization: organizationId,
-        role: ADMIN
+          workspace: workspaceId,
+          description: "This role was auto generated by Infisical in effort to migrate your project members to our new permission system",
+          isOrgRole: false,
+          slug: `custom-role-${membership_identity}`,
+          permissions: permissions
+        });
+
+        await role.save();
+
+        for (const id of membershipIds) {
+          await Membership.findByIdAndUpdate(id, { // document db doesn't support update many so we must loop
+            $set: {
+              role: CUSTOM,
+              customRole: role
+            }
+          });
+        }
       }
+
+      console.info("Backfill: Finished converting old denied permission in workspace to viewers");
+
+      await MembershipOrg.updateMany(
+        {
+          role: OWNER
+        },
+        {
+          $set: {
+            role: ADMIN
+          }
+        }
+      );
+
+      console.info("Backfill: Finished converting owner role to member");
+
+    } catch (error) {
+      console.error("An error occurred when running script [backfillPermission]:", error);
     }
-  );
+
-  console.log("Backfill: Finishing converting old denied permission in workspace to viewers");
+  } else {
+    console.info("Could not acquire lock for script [backfillPermission], skipping");
+  }
 };

backend/src/validation/integration.ts

@@ -77,7 +77,12 @@ export const CreateIntegrationV1 = z.object({
     path: z.string().trim().optional(),
     region: z.string().trim().optional(),
     metadata: z.object({
-      secretSuffix: z.string().optional()
+      secretPrefix: z.string().optional(),
+      secretSuffix: z.string().optional(),
+      secretGCPLabel: z.object({
+        labelName: z.string(),
+        labelValue: z.string()
+      }).optional()
     }).optional()
   })
 });

docs/integrations/cicd/gitlab.mdx

@@ -32,6 +32,16 @@ Press on the GitLab tile and grant Infisical access to your GitLab account.
 Select which Infisical environment secrets you want to sync to which GitLab repository and press create integration to start syncing secrets to GitLab.
 
 ![integrations gitlab](../../images/integrations/gitlab/integrations-gitlab-create.png)
+
+Note that the GitLab integration supports a few options in the **Options** tab:
+
+- Secret Prefix: If inputted, the prefix is appended to the front of every secret name prior to being synced.
+- Secret Suffix: If inputted, the suffix to appended to the back of every name of every secret prior to being synced.
+
+Setting a secret prefix or suffix ensures that existing secrets in GCP Secret Manager are not overwritten during the sync. As part of this process, Infisical abstains from mutating any secrets in GitLab without the specified prefix or suffix.
+
+![integrations gitlab options](../../images/integrations/gitlab/integrations-gitlab-create-options.png)
+
 ![integrations gitlab](../../images/integrations/gitlab/integrations-gitlab.png)
   </Accordion>
   <Accordion title="Pipeline">

docs/integrations/cloud/gcp-secret-manager.mdx

@@ -35,14 +35,21 @@ Grant Infisical access to GCP.
 
 ## Start integration
 
-Select which Infisical environment secrets you want to sync to which GCP secret manager project. Lastly, press create integration to start syncing secrets to GCP secret manager.
+In the **Connection** tab, select which Infisical environment secrets you want to sync to which GCP secret manager project. Lastly, press create integration to start syncing secrets to GCP secret manager.
 
 ![integrations GCP secret manager](../../images/integrations/gcp-secret-manager/integrations-gcp-secret-manager-create.png)
-![integrations GCP secret manager](../../images/integrations/gcp-secret-manager/integrations-gcp-secret-manager.png)
 
-<Note>
+Note that the GCP Secret Manager integration supports a few options in the **Options** tab:
-  Secrets synced from Infisical to GCP Secret Manager are automatically labeled `managed-by:infisical` to avoid overwriting existing values in GCP Secret Manager.
+
-</Note>
+- Secret Prefix: If inputted, the prefix is appended to the front of every secret name prior to being synced.
+- Secret Suffix: If inputted, the suffix to appended to the back of every name of every secret prior to being synced.
+- Label in GCP Secret Manager: If selected, every secret will be labeled in GCP Secret Manager (e.g. as `managed-by:infisical`); labels can be customized.
+
+Setting a secret prefix, suffix, or enabling the labeling option ensures that existing secrets in GCP Secret Manager are not overwritten during the sync. As part of this process, Infisical abstains from mutating any secrets in GCP Secret Manager without the specified prefix, suffix, or attached label.
+
+![integrations GCP secret manager options](../../images/integrations/gcp-secret-manager/integrations-gcp-secret-manager-create-options.png)
+
+![integrations GCP secret manager](../../images/integrations/gcp-secret-manager/integrations-gcp-secret-manager.png)
 
 <Warning>
     Using Infisical to sync secrets to GCP Secret Manager requires that you enable
@@ -89,14 +96,21 @@ service account in IAM & Admin > Service Accounts > Service Account > Keys).
 
 ## Start integration
 
-Select which Infisical environment secrets you want to sync to the GCP secret manager project. Lastly, press create integration to start syncing secrets to GCP secret manager.
+In the **Connection** tab, select which Infisical environment secrets you want to sync to the GCP secret manager project. Lastly, press create integration to start syncing secrets to GCP secret manager.
 
 ![integrations GCP secret manager](../../images/integrations/gcp-secret-manager/integrations-gcp-secret-manager-create.png)
-![integrations GCP secret manager](../../images/integrations/gcp-secret-manager/integrations-gcp-secret-manager.png)
 
-<Note>
+Note that the GCP Secret Manager integration supports a few options in the **Options** tab:
-  Secrets synced from Infisical to GCP Secret Manager are automatically labeled `managed-by:infisical` to avoid overwriting existing values in GCP Secret Manager.
+
-</Note>
+- Secret Prefix: If inputted, the prefix is appended to the front of every secret name prior to being synced.
+- Secret Suffix: If inputted, the suffix to appended to the back of every name of every secret prior to being synced.
+- Label in GCP Secret Manager: If selected, every secret will be labeled in GCP Secret Manager (e.g. as `managed-by:infisical`); labels can be customized.
+
+Setting a secret prefix, suffix, or enabling the labeling option ensures that existing secrets in GCP Secret Manager are not overwritten during the sync. As part of this process, Infisical abstains from mutating any secrets in GCP Secret Manager without the specified prefix, suffix, or attached label.
+
+![integrations GCP secret manager options](../../images/integrations/gcp-secret-manager/integrations-gcp-secret-manager-create-options.png)
+
+![integrations GCP secret manager](../../images/integrations/gcp-secret-manager/integrations-gcp-secret-manager.png)
 
 <Warning>
     Using Infisical to sync secrets to GCP Secret Manager requires that you enable

frontend/src/components/utilities/parseDotEnv.ts

@@ -1,5 +1,5 @@
 const LINE =
-  /(?:^|^)\s*(?:export\s+)?([\w.-:]+)(?:\s*=\s*?|:\s+?)(\s*'(?:\\'|[^'])*'|\s*"(?:\\"|[^"])*"|\s*`(?:\\`|[^`])*`|[^#\r\n]+)?\s*(?:#.*)?(?:$|$)/gm;
+  /(?:^|^)\s*(?:export\s+)?([\w.:-]+)(?:\s*=\s*?|:\s+?)(\s*'(?:\\'|[^'])*'|\s*"(?:\\"|[^"])*"|\s*`(?:\\`|[^`])*`|[^#\r\n]+)?\s*(?:#.*)?(?:$|$)/gm;
 
 /**
  * Return text that is the buffer parsed

frontend/src/pages/integrations/gcp-secret-manager/authorize.tsx

@@ -43,7 +43,7 @@ export default function GCPSecretManagerAuthorizeIntegrationPage() {
 
       setIsLoading(false);
 
-      router.push(`/integrations/gcp-secret-manager/pat/create?integrationAuthId=${integrationAuth._id}`);
+      router.push(`/integrations/gcp-secret-manager/create?integrationAuthId=${integrationAuth._id}`);
     } catch (err) {
       console.error(err);
     }

frontend/src/pages/integrations/gcp-secret-manager/pat/create.tsx

@@ -1,146 +0,0 @@
-import { useEffect, useState } from "react";
-import { useRouter } from "next/router";
-import queryString from "query-string";
-
-import {
-  Button,
-  Card,
-  CardTitle,
-  FormControl,
-  Input,
-  Select,
-  SelectItem
-} from "@app/components/v2";
-import {
-  useCreateIntegration
-} from "@app/hooks/api";
-import { useGetIntegrationAuthApps,useGetIntegrationAuthById } from "@app/hooks/api/integrationAuth";
-import { useGetWorkspaceById } from "@app/hooks/api/workspace";
-
-export default function GCPSecretManagerCreateIntegrationPage() {
-  const router = useRouter();
-  const { mutateAsync } = useCreateIntegration();
-
-  const { integrationAuthId } = queryString.parse(router.asPath.split("?")[1]);
-
-  const { data: workspace } = useGetWorkspaceById(localStorage.getItem("projectData.id") ?? "");
-  const { data: integrationAuth } = useGetIntegrationAuthById((integrationAuthId as string) ?? "");
-  const { data: integrationAuthApps } = useGetIntegrationAuthApps({
-    integrationAuthId: (integrationAuthId as string) ?? ""
-  });
-
-  const [selectedSourceEnvironment, setSelectedSourceEnvironment] = useState("");
-  const [targetAppId, setTargetAppId] = useState("");
-  const [secretPath, setSecretPath] = useState("/");
-
-  const [isLoading, setIsLoading] = useState(false);
-
-  useEffect(() => {
-    if (workspace) {
-      setSelectedSourceEnvironment(workspace.environments[0].slug);
-    }
-  }, [workspace]);
-  
-  useEffect(() => {
-    if (integrationAuthApps) {
-      if (integrationAuthApps.length > 0) {
-        setTargetAppId(integrationAuthApps[0].appId as string);
-      } else {
-        setTargetAppId("none");
-      }
-    }
-  }, [integrationAuthApps]);
-
-  const handleButtonClick = async () => {
-    try {
-      setIsLoading(true);
-
-      if (!integrationAuth?._id) return;
-      
-      await mutateAsync({
-        integrationAuthId: integrationAuth?._id,
-        isActive: true,
-        app: integrationAuthApps?.find((integrationAuthApp) => integrationAuthApp.appId === targetAppId)?.name,
-        appId: targetAppId,
-        sourceEnvironment: selectedSourceEnvironment,
-        secretPath
-      });
-
-      setIsLoading(false);
-      router.push(`/integrations/${localStorage.getItem("projectData.id")}`);
-    } catch (err) {
-      console.error(err);
-    }
-  };
-
-  return integrationAuth &&
-    workspace &&
-    selectedSourceEnvironment &&
-    integrationAuthApps
-    ? (
-    <div className="flex h-full w-full items-center justify-center">
-      <Card className="max-w-md rounded-md p-8">
-        <CardTitle className="text-center">GCP Secret Manager Integration</CardTitle>
-        <FormControl label="Project Environment" className="mt-4">
-          <Select
-            value={selectedSourceEnvironment}
-            onValueChange={(val) => setSelectedSourceEnvironment(val)}
-            className="w-full border border-mineshaft-500"
-          >
-            {workspace?.environments.map((sourceEnvironment) => (
-              <SelectItem
-                value={sourceEnvironment.slug}
-                key={`source-environment-${sourceEnvironment.slug}`}
-              >
-                {sourceEnvironment.name}
-              </SelectItem>
-            ))}
-          </Select>
-        </FormControl>
-        <FormControl label="Secrets Path">
-          <Input
-            value={secretPath}
-            onChange={(evt) => setSecretPath(evt.target.value)}
-            placeholder="Provide a path, default is /"
-          />
-        </FormControl>
-        <FormControl label="GCP Project">
-          <Select
-            value={targetAppId}
-            onValueChange={(val) => setTargetAppId(val)}
-            className="w-full border border-mineshaft-500"
-            isDisabled={integrationAuthApps.length === 0}
-          >
-            {integrationAuthApps.length > 0 ? (
-              integrationAuthApps.map((integrationAuthApp) => (
-                <SelectItem
-                  value={integrationAuthApp.appId as string}
-                  key={`target-app-${integrationAuthApp.appId}`}
-                >
-                  {integrationAuthApp.name}
-                </SelectItem>
-              ))
-            ) : (
-              <SelectItem value="none" key="target-app-none">
-                No projects found
-              </SelectItem>
-            )}
-          </Select>
-        </FormControl>
-        <Button
-          onClick={handleButtonClick}
-          color="mineshaft"
-          className="mt-4"
-          isLoading={isLoading}
-        //   isDisabled={integrationAuthApps.length === 0}
-        >
-          Create Integration
-        </Button>
-      </Card>
-    </div>
-  ) : (
-    <div />
-  );
-}
-
-GCPSecretManagerCreateIntegrationPage.requireAuth = true;

frontend/src/pages/integrations/gitlab/create.tsx

@@ -85,6 +85,7 @@ export default function GitLabCreateIntegrationPage() {
   const { data: integrationAuth } = useGetIntegrationAuthById((integrationAuthId as string) ?? "");
 
   const { data: integrationAuthApps, isLoading: isintegrationAuthAppsLoading } = useGetIntegrationAuthApps({
+
     integrationAuthId: (integrationAuthId as string) ?? "",
     ...(targetTeamId ? { teamId: targetTeamId } : {})
   });