Merge remote-tracking branch 'origin' into k8s-auth

.github/workflows/build-staging-and-deploy-aws.yml

@@ -122,13 +122,13 @@ jobs:
         uses: pr-mpt/actions-commit-hash@v2
       - name: Download task definition
         run: |
-          aws ecs describe-task-definition --task-definition infisical-prod-platform --query taskDefinition > task-definition.json
+          aws ecs describe-task-definition --task-definition infisical-core-platform --query taskDefinition > task-definition.json
       - name: Render Amazon ECS task definition
         id: render-web-container
         uses: aws-actions/amazon-ecs-render-task-definition@v1
         with:
           task-definition: task-definition.json
-          container-name: infisical-prod-platform
+          container-name: infisical-core-platform
           image: infisical/staging_infisical:${{ steps.commit.outputs.short }}
           environment-variables: "LOG_LEVEL=info"
       - name: Deploy to Amazon ECS service

backend/src/@types/knex.d.ts

@@ -234,6 +234,7 @@ import {
   TWebhooksInsert,
   TWebhooksUpdate
 } from "@app/db/schemas";
+import { TSecretReferences, TSecretReferencesInsert, TSecretReferencesUpdate } from "@app/db/schemas/secret-references";
 
 declare module "knex/types/tables" {
   interface Tables {
@@ -307,6 +308,11 @@ declare module "knex/types/tables" {
     >;
     [TableName.ProjectKeys]: Knex.CompositeTableType<TProjectKeys, TProjectKeysInsert, TProjectKeysUpdate>;
     [TableName.Secret]: Knex.CompositeTableType<TSecrets, TSecretsInsert, TSecretsUpdate>;
+    [TableName.SecretReference]: Knex.CompositeTableType<
+      TSecretReferences,
+      TSecretReferencesInsert,
+      TSecretReferencesUpdate
+    >;
     [TableName.SecretBlindIndex]: Knex.CompositeTableType<
       TSecretBlindIndexes,
       TSecretBlindIndexesInsert,

backend/src/db/migrations/20240514141809_inline-secret-reference-sync.ts

@@ -0,0 +1,24 @@
+import { Knex } from "knex";
+
+import { TableName } from "../schemas";
+import { createOnUpdateTrigger, dropOnUpdateTrigger } from "../utils";
+
+export async function up(knex: Knex): Promise<void> {
+  if (!(await knex.schema.hasTable(TableName.SecretReference))) {
+    await knex.schema.createTable(TableName.SecretReference, (t) => {
+      t.uuid("id", { primaryKey: true }).defaultTo(knex.fn.uuid());
+      t.string("environment").notNullable();
+      t.string("secretPath").notNullable();
+      t.uuid("secretId").notNullable();
+      t.foreign("secretId").references("id").inTable(TableName.Secret).onDelete("CASCADE");
+      t.timestamps(true, true, true);
+    });
+
+    await createOnUpdateTrigger(knex, TableName.SecretReference);
+  }
+}
+
+export async function down(knex: Knex): Promise<void> {
+  await knex.schema.dropTableIfExists(TableName.SecretReference);
+  await dropOnUpdateTrigger(knex, TableName.SecretReference);
+}

backend/src/db/schemas/models.ts

@@ -28,6 +28,7 @@ export enum TableName {
   ProjectUserMembershipRole = "project_user_membership_roles",
   ProjectKeys = "project_keys",
   Secret = "secrets",
+  SecretReference = "secret_references",
   SecretBlindIndex = "secret_blind_indexes",
   SecretVersion = "secret_versions",
   SecretFolder = "secret_folders",

backend/src/db/schemas/secret-references.ts

@@ -0,0 +1,21 @@
+// Code generated by automation script, DO NOT EDIT.
+// Automated by pulling database and generating zod schema
+// To update. Just run npm run generate:schema
+// Written by akhilmhdh.
+
+import { z } from "zod";
+
+import { TImmutableDBKeys } from "./models";
+
+export const SecretReferencesSchema = z.object({
+  id: z.string().uuid(),
+  environment: z.string(),
+  secretPath: z.string(),
+  secretId: z.string().uuid(),
+  createdAt: z.date(),
+  updatedAt: z.date()
+});
+
+export type TSecretReferences = z.infer<typeof SecretReferencesSchema>;
+export type TSecretReferencesInsert = Omit<z.input<typeof SecretReferencesSchema>, TImmutableDBKeys>;
+export type TSecretReferencesUpdate = Partial<Omit<z.input<typeof SecretReferencesSchema>, TImmutableDBKeys>>;

backend/src/ee/routes/v1/identity-project-additional-privilege-router.ts

@@ -8,7 +8,7 @@ import { IDENTITY_ADDITIONAL_PRIVILEGE } from "@app/lib/api-docs";
 import { alphaNumericNanoId } from "@app/lib/nanoid";
 import { readLimit, writeLimit } from "@app/server/config/rateLimiter";
 import { verifyAuth } from "@app/server/plugins/auth/verify-auth";
-import { PermissionSchema, SanitizedIdentityPrivilegeSchema } from "@app/server/routes/sanitizedSchemas";
+import { ProjectPermissionSchema, SanitizedIdentityPrivilegeSchema } from "@app/server/routes/sanitizedSchemas";
 import { AuthMode } from "@app/services/auth/auth-type";
 
 export const registerIdentityProjectAdditionalPrivilegeRouter = async (server: FastifyZodProvider) => {
@@ -39,7 +39,7 @@ export const registerIdentityProjectAdditionalPrivilegeRouter = async (server: F
           })
           .optional()
           .describe(IDENTITY_ADDITIONAL_PRIVILEGE.CREATE.slug),
-        permissions: PermissionSchema.array().describe(IDENTITY_ADDITIONAL_PRIVILEGE.CREATE.permissions)
+        permissions: ProjectPermissionSchema.array().describe(IDENTITY_ADDITIONAL_PRIVILEGE.CREATE.permissions)
       }),
       response: {
         200: z.object({
@@ -90,7 +90,7 @@ export const registerIdentityProjectAdditionalPrivilegeRouter = async (server: F
           })
           .optional()
           .describe(IDENTITY_ADDITIONAL_PRIVILEGE.CREATE.slug),
-        permissions: PermissionSchema.array().describe(IDENTITY_ADDITIONAL_PRIVILEGE.CREATE.permissions),
+        permissions: ProjectPermissionSchema.array().describe(IDENTITY_ADDITIONAL_PRIVILEGE.CREATE.permissions),
         temporaryMode: z
           .nativeEnum(IdentityProjectAdditionalPrivilegeTemporaryMode)
           .describe(IDENTITY_ADDITIONAL_PRIVILEGE.CREATE.temporaryMode),
@@ -155,7 +155,7 @@ export const registerIdentityProjectAdditionalPrivilegeRouter = async (server: F
                 message: "Slug must be a valid slug"
               })
               .describe(IDENTITY_ADDITIONAL_PRIVILEGE.UPDATE.newSlug),
-            permissions: PermissionSchema.array().describe(IDENTITY_ADDITIONAL_PRIVILEGE.UPDATE.permissions),
+            permissions: ProjectPermissionSchema.array().describe(IDENTITY_ADDITIONAL_PRIVILEGE.UPDATE.permissions),
             isTemporary: z.boolean().describe(IDENTITY_ADDITIONAL_PRIVILEGE.UPDATE.isTemporary),
             temporaryMode: z
               .nativeEnum(IdentityProjectAdditionalPrivilegeTemporaryMode)

backend/src/ee/services/secret-approval-request/secret-approval-request-service.ts

@@ -7,12 +7,15 @@ import {
   SecretType,
   TSecretApprovalRequestsSecretsInsert
 } from "@app/db/schemas";
+import { decryptSymmetric128BitHexKeyUTF8 } from "@app/lib/crypto";
 import { BadRequestError, UnauthorizedError } from "@app/lib/errors";
 import { groupBy, pick, unique } from "@app/lib/fn";
 import { alphaNumericNanoId } from "@app/lib/nanoid";
 import { ActorType } from "@app/services/auth/auth-type";
 import { TProjectDALFactory } from "@app/services/project/project-dal";
+import { TProjectBotServiceFactory } from "@app/services/project-bot/project-bot-service";
 import { TSecretDALFactory } from "@app/services/secret/secret-dal";
+import { getAllNestedSecretReferences } from "@app/services/secret/secret-fns";
 import { TSecretQueueFactory } from "@app/services/secret/secret-queue";
 import { TSecretServiceFactory } from "@app/services/secret/secret-service";
 import { TSecretVersionDALFactory } from "@app/services/secret/secret-version-dal";
@@ -53,6 +56,7 @@ type TSecretApprovalRequestServiceFactoryDep = {
   secretVersionDAL: Pick<TSecretVersionDALFactory, "findLatestVersionMany" | "insertMany">;
   secretVersionTagDAL: Pick<TSecretVersionTagDALFactory, "insertMany">;
   projectDAL: Pick<TProjectDALFactory, "checkProjectUpgradeStatus">;
+  projectBotService: Pick<TProjectBotServiceFactory, "getBotKey">;
   secretService: Pick<
     TSecretServiceFactory,
     | "fnSecretBulkInsert"
@@ -80,7 +84,8 @@ export const secretApprovalRequestServiceFactory = ({
   snapshotService,
   secretService,
   secretVersionDAL,
-  secretQueueService
+  secretQueueService,
+  projectBotService
 }: TSecretApprovalRequestServiceFactoryDep) => {
   const requestCount = async ({ projectId, actor, actorId, actorOrgId, actorAuthMethod }: TApprovalRequestCountDTO) => {
     if (actor === ActorType.SERVICE) throw new BadRequestError({ message: "Cannot use service token" });
@@ -352,7 +357,7 @@ export const secretApprovalRequestServiceFactory = ({
     }
 
     const secretDeletionCommits = secretApprovalSecrets.filter(({ op }) => op === CommitType.Delete);
-
+    const botKey = await projectBotService.getBotKey(projectId).catch(() => null);
     const mergeStatus = await secretApprovalRequestDAL.transaction(async (tx) => {
       const newSecrets = secretCreationCommits.length
         ? await secretService.fnSecretBulkInsert({
@@ -379,7 +384,17 @@ export const secretApprovalRequestServiceFactory = ({
               ]),
               tags: el?.tags.map(({ id }) => id),
               version: 1,
-              type: SecretType.Shared
+              type: SecretType.Shared,
+              references: botKey
+                ? getAllNestedSecretReferences(
+                    decryptSymmetric128BitHexKeyUTF8({
+                      ciphertext: el.secretValueCiphertext,
+                      iv: el.secretValueIV,
+                      tag: el.secretValueTag,
+                      key: botKey
+                    })
+                  )
+                : undefined
             })),
             secretDAL,
             secretVersionDAL,
@@ -414,7 +429,17 @@ export const secretApprovalRequestServiceFactory = ({
                   "secretReminderNote",
                   "secretReminderRepeatDays",
                   "secretBlindIndex"
-                ])
+                ]),
+                references: botKey
+                  ? getAllNestedSecretReferences(
+                      decryptSymmetric128BitHexKeyUTF8({
+                        ciphertext: el.secretValueCiphertext,
+                        iv: el.secretValueIV,
+                        tag: el.secretValueTag,
+                        key: botKey
+                      })
+                    )
+                  : undefined
               }
             })),
             secretDAL,

backend/src/lib/api-docs/constants.ts

@@ -89,6 +89,9 @@ export const UNIVERSAL_AUTH = {
   },
   RENEW_ACCESS_TOKEN: {
     accessToken: "The access token to renew."
+  },
+  REVOKE_ACCESS_TOKEN: {
+    accessToken: "The access token to revoke."
   }
 } as const;
 

backend/src/queue/queue-service.ts

@@ -65,7 +65,13 @@ export type TQueueJobTypes = {
   };
   [QueueName.IntegrationSync]: {
     name: QueueJobs.IntegrationSync;
-    payload: { projectId: string; environment: string; secretPath: string; depth?: number };
+    payload: {
+      projectId: string;
+      environment: string;
+      secretPath: string;
+      depth?: number;
+      deDupeQueue?: Record<string, boolean>;
+    };
   };
   [QueueName.SecretFullRepoScan]: {
     name: QueueJobs.SecretScan;

backend/src/server/routes/index.ts

@@ -610,6 +610,7 @@ export const registerRoutes = async (
   });
   const sarService = secretApprovalRequestServiceFactory({
     permissionService,
+    projectBotService,
     folderDAL,
     secretDAL,
     secretTagDAL,

backend/src/server/routes/sanitizedSchemas.ts

@@ -8,6 +8,7 @@ import {
   UsersSchema
 } from "@app/db/schemas";
 import { UnpackedPermissionSchema } from "@app/ee/services/identity-project-additional-privilege/identity-project-additional-privilege-service";
+import { ProjectPermissionActions, ProjectPermissionSub } from "@app/ee/services/permission/project-permission";
 
 // sometimes the return data must be santizied to avoid leaking important values
 // always prefer pick over omit in zod
@@ -64,14 +65,12 @@ export const secretRawSchema = z.object({
   secretComment: z.string().optional()
 });
 
-export const PermissionSchema = z.object({
+export const ProjectPermissionSchema = z.object({
   action: z
-    .string()
+    .nativeEnum(ProjectPermissionActions)
-    .min(1)
     .describe("Describe what action an entity can take. Possible actions: create, edit, delete, and read"),
   subject: z
-    .string()
+    .nativeEnum(ProjectPermissionSub)
-    .min(1)
     .describe("The entity this permission pertains to. Possible options: secrets, environments"),
   conditions: z
     .object({

backend/src/server/routes/v1/identity-access-token-router.ts

@@ -36,4 +36,29 @@ export const registerIdentityAccessTokenRouter = async (server: FastifyZodProvid
       };
     }
   });
+
+  server.route({
+    url: "/token/revoke",
+    method: "POST",
+    config: {
+      rateLimit: writeLimit
+    },
+    schema: {
+      description: "Revoke access token",
+      body: z.object({
+        accessToken: z.string().trim().describe(UNIVERSAL_AUTH.REVOKE_ACCESS_TOKEN.accessToken)
+      }),
+      response: {
+        200: z.object({
+          message: z.string()
+        })
+      }
+    },
+    handler: async (req) => {
+      await server.services.identityAccessToken.revokeAccessToken(req.body.accessToken);
+      return {
+        message: "Successfully revoked access token"
+      };
+    }
+  });
 };

backend/src/server/routes/v3/secret-router.ts

@@ -1926,4 +1926,41 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => {
       return { secrets };
     }
   });
+
+  server.route({
+    method: "POST",
+    url: "/backfill-secret-references",
+    config: {
+      rateLimit: secretsLimit
+    },
+    schema: {
+      description: "Backfill secret references",
+      security: [
+        {
+          bearerAuth: []
+        }
+      ],
+      body: z.object({
+        projectId: z.string().trim().min(1)
+      }),
+      response: {
+        200: z.object({
+          message: z.string()
+        })
+      }
+    },
+    onRequest: verifyAuth([AuthMode.JWT]),
+    handler: async (req) => {
+      const { projectId } = req.body;
+      const message = await server.services.secret.backfillSecretReferences({
+        actorId: req.permission.id,
+        actor: req.permission.type,
+        actorAuthMethod: req.permission.authMethod,
+        actorOrgId: req.permission.orgId,
+        projectId
+      });
+
+      return message;
+    }
+  });
 };

backend/src/services/identity-access-token/identity-access-token-dal.ts

@@ -1,7 +1,7 @@
 import { Knex } from "knex";
 
 import { TDbClient } from "@app/db";
-import { TableName, TIdentityAccessTokens } from "@app/db/schemas";
+import { IdentityAuthMethod, TableName, TIdentityAccessTokens } from "@app/db/schemas";
 import { DatabaseError } from "@app/lib/errors";
 import { ormify, selectAllTableCols } from "@app/lib/knex";
 
@@ -15,23 +15,46 @@ export const identityAccessTokenDALFactory = (db: TDbClient) => {
       const doc = await (tx || db)(TableName.IdentityAccessToken)
         .where(filter)
         .join(TableName.Identity, `${TableName.Identity}.id`, `${TableName.IdentityAccessToken}.identityId`)
-        .leftJoin(
+        .leftJoin(TableName.IdentityUaClientSecret, (qb) => {
-          TableName.IdentityUaClientSecret,
+          qb.on(`${TableName.Identity}.authMethod`, db.raw("?", [IdentityAuthMethod.Univeral])).andOn(
-          `${TableName.IdentityAccessToken}.identityUAClientSecretId`,
+            `${TableName.IdentityAccessToken}.identityUAClientSecretId`,
-          `${TableName.IdentityUaClientSecret}.id`
+            `${TableName.IdentityUaClientSecret}.id`
-        )
+          );
-        .leftJoin(
+        })
-          TableName.IdentityUniversalAuth,
+        .leftJoin(TableName.IdentityUniversalAuth, (qb) => {
-          `${TableName.IdentityUaClientSecret}.identityUAId`,
+          qb.on(`${TableName.Identity}.authMethod`, db.raw("?", [IdentityAuthMethod.Univeral])).andOn(
-          `${TableName.IdentityUniversalAuth}.id`
+            `${TableName.IdentityUaClientSecret}.identityUAId`,
-        )
+            `${TableName.IdentityUniversalAuth}.id`
+          );
+        })
+        .leftJoin(TableName.IdentityGcpAuth, (qb) => {
+          qb.on(`${TableName.Identity}.authMethod`, db.raw("?", [IdentityAuthMethod.GCP_AUTH])).andOn(
+            `${TableName.Identity}.id`,
+            `${TableName.IdentityGcpAuth}.identityId`
+          );
+        })
+        .leftJoin(TableName.IdentityAwsAuth, (qb) => {
+          qb.on(`${TableName.Identity}.authMethod`, db.raw("?", [IdentityAuthMethod.AWS_AUTH])).andOn(
+            `${TableName.Identity}.id`,
+            `${TableName.IdentityAwsAuth}.identityId`
+          );
+        })
         .select(selectAllTableCols(TableName.IdentityAccessToken))
         .select(
-          db.ref("accessTokenTrustedIps").withSchema(TableName.IdentityUniversalAuth),
+          db.ref("accessTokenTrustedIps").withSchema(TableName.IdentityUniversalAuth).as("accessTokenTrustedIpsUa"),
+          db.ref("accessTokenTrustedIps").withSchema(TableName.IdentityGcpAuth).as("accessTokenTrustedIpsGcp"),
+          db.ref("accessTokenTrustedIps").withSchema(TableName.IdentityAwsAuth).as("accessTokenTrustedIpsAws"),
           db.ref("name").withSchema(TableName.Identity)
         )
         .first();
-      return doc;
+
+      if (!doc) return;
+
+      return {
+        ...doc,
+        accessTokenTrustedIps:
+          doc.accessTokenTrustedIpsUa || doc.accessTokenTrustedIpsGcp || doc.accessTokenTrustedIpsAws
+      };
     } catch (error) {
       throw new DatabaseError({ error, name: "IdAccessTokenFindOne" });
     }

backend/src/services/identity-access-token/identity-access-token-service.ts

@@ -106,6 +106,24 @@ export const identityAccessTokenServiceFactory = ({
     return { accessToken, identityAccessToken: updatedIdentityAccessToken };
   };
 
+  const revokeAccessToken = async (accessToken: string) => {
+    const appCfg = getConfig();
+
+    const decodedToken = jwt.verify(accessToken, appCfg.AUTH_SECRET) as JwtPayload & {
+      identityAccessTokenId: string;
+    };
+    if (decodedToken.authTokenType !== AuthTokenType.IDENTITY_ACCESS_TOKEN) throw new UnauthorizedError();
+
+    const identityAccessToken = await identityAccessTokenDAL.findOne({
+      [`${TableName.IdentityAccessToken}.id` as "id"]: decodedToken.identityAccessTokenId,
+      isAccessTokenRevoked: false
+    });
+    if (!identityAccessToken) throw new UnauthorizedError();
+
+    const revokedToken = await identityAccessTokenDAL.deleteById(identityAccessToken.id);
+    return { revokedToken };
+  };
+
   const fnValidateIdentityAccessToken = async (token: TIdentityAccessTokenJwtPayload, ipAddress?: string) => {
     const identityAccessToken = await identityAccessTokenDAL.findOne({
       [`${TableName.IdentityAccessToken}.id` as "id"]: token.identityAccessTokenId,
@@ -132,5 +150,5 @@ export const identityAccessTokenServiceFactory = ({
     return { ...identityAccessToken, orgId: identityOrgMembership.orgId };
   };
 
-  return { renewAccessToken, fnValidateIdentityAccessToken };
+  return { renewAccessToken, revokeAccessToken, fnValidateIdentityAccessToken };
 };

backend/src/services/identity-project/identity-project-service.ts

@@ -82,6 +82,7 @@ export const identityProjectServiceFactory = ({
       role,
       project.id
     );
+
     const hasPriviledge = isAtLeastAsPrivileged(permission, rolePermission);
     if (!hasPriviledge)
       throw new ForbiddenRequestError({
@@ -135,16 +136,18 @@ export const identityProjectServiceFactory = ({
         message: `Identity with id ${identityId} doesn't exists in project with id ${projectId}`
       });
 
-    const { permission: identityRolePermission } = await permissionService.getProjectPermission(
+    for await (const { role: requestedRoleChange } of roles) {
-      ActorType.IDENTITY,
+      const { permission: rolePermission } = await permissionService.getProjectPermissionByRole(
-      projectIdentity.identityId,
+        requestedRoleChange,
-      projectIdentity.projectId,
+        projectId
-      actorAuthMethod,
+      );
-      actorOrgId
+
-    );
+      const hasRequiredPriviledges = isAtLeastAsPrivileged(permission, rolePermission);
-    const hasRequiredPriviledges = isAtLeastAsPrivileged(permission, identityRolePermission);
+
-    if (!hasRequiredPriviledges)
+      if (!hasRequiredPriviledges) {
-      throw new ForbiddenRequestError({ message: "Failed to delete more privileged identity" });
+        throw new ForbiddenRequestError({ message: "Failed to change to a more privileged role" });
+      }
+    }
 
     // validate custom roles input
     const customInputRoles = roles.filter(

backend/src/services/integration-auth/integration-sync-secret.ts

@@ -462,27 +462,39 @@ const syncSecretsAWSParameterStore = async ({
   ssm.config.update(config);
 
   const metadata = z.record(z.any()).parse(integration.metadata || {});
+  const awsParameterStoreSecretsObj: Record<string, AWS.SSM.Parameter> = {};
 
-  const params = {
+  // now fetch all aws parameter store secrets
-    Path: integration.path as string,
+  let hasNext = true;
-    Recursive: false,
+  let nextToken: string | undefined;
-    WithDecryption: true
+  while (hasNext) {
-  };
+    const parameters = await ssm
+      .getParametersByPath({
+        Path: integration.path as string,
+        Recursive: false,
+        WithDecryption: true,
+        MaxResults: 10,
+        NextToken: nextToken
+      })
+      .promise();
 
-  const parameterList = (await ssm.getParametersByPath(params).promise()).Parameters;
+    if (parameters.Parameters) {
+      parameters.Parameters.forEach((parameter) => {
+        if (parameter.Name) {
+          const secKey = parameter.Name.substring((integration.path as string).length);
+          awsParameterStoreSecretsObj[secKey] = parameter;
+        }
+      });
+    }
+    hasNext = Boolean(parameters.NextToken);
+    nextToken = parameters.NextToken;
+  }
 
-  const awsParameterStoreSecretsObj = (parameterList || [])
-    .filter(({ Name }) => Boolean(Name))
-    .reduce(
-      (obj, secret) => ({
-        ...obj,
-        [(secret.Name as string).substring((integration.path as string).length)]: secret
-      }),
-      {} as Record<string, AWS.SSM.Parameter>
-    );
   // Identify secrets to create
-  await Promise.all(
+  // don't use Promise.all() and promise map here
-    Object.keys(secrets).map(async (key) => {
+  // it will cause rate limit
+  for (const key in secrets) {
+    if (Object.hasOwn(secrets, key)) {
       if (!(key in awsParameterStoreSecretsObj)) {
         // case: secret does not exist in AWS parameter store
         // -> create secret
@@ -517,13 +529,16 @@ const syncSecretsAWSParameterStore = async ({
           })
           .promise();
       }
-    })
+
-  );
+      await new Promise((resolve) => {
+        setTimeout(resolve, 50);
+      });
+    }
+  }
 
   if (!metadata.shouldDisableDelete) {
-    // Identify secrets to delete
+    for (const key in awsParameterStoreSecretsObj) {
-    await Promise.all(
+      if (Object.hasOwn(awsParameterStoreSecretsObj, key)) {
-      Object.keys(awsParameterStoreSecretsObj).map(async (key) => {
         if (!(key in secrets)) {
           // case:
           // -> delete secret
@@ -533,8 +548,11 @@ const syncSecretsAWSParameterStore = async ({
             })
             .promise();
         }
-      })
+        await new Promise((resolve) => {
-    );
+          setTimeout(resolve, 50);
+        });
+      }
+    }
   }
 };
 

backend/src/services/secret/secret-dal.ts

@@ -243,6 +243,74 @@ export const secretDALFactory = (db: TDbClient) => {
     }
   };
 
+  const upsertSecretReferences = async (
+    data: {
+      secretId: string;
+      references: Array<{ environment: string; secretPath: string }>;
+    }[] = [],
+    tx?: Knex
+  ) => {
+    try {
+      if (!data.length) return;
+
+      await (tx || db)(TableName.SecretReference)
+        .whereIn(
+          "secretId",
+          data.map(({ secretId }) => secretId)
+        )
+        .delete();
+      const newSecretReferences = data
+        .filter(({ references }) => references.length)
+        .flatMap(({ secretId, references }) =>
+          references.map(({ environment, secretPath }) => ({
+            secretPath,
+            secretId,
+            environment
+          }))
+        );
+      if (!newSecretReferences.length) return;
+      const secretReferences = await (tx || db)(TableName.SecretReference).insert(newSecretReferences);
+      return secretReferences;
+    } catch (error) {
+      throw new DatabaseError({ error, name: "UpsertSecretReference" });
+    }
+  };
+
+  const findReferencedSecretReferences = async (projectId: string, envSlug: string, secretPath: string, tx?: Knex) => {
+    try {
+      const docs = await (tx || db)(TableName.SecretReference)
+        .where({
+          secretPath,
+          environment: envSlug
+        })
+        .join(TableName.Secret, `${TableName.Secret}.id`, `${TableName.SecretReference}.secretId`)
+        .join(TableName.SecretFolder, `${TableName.Secret}.folderId`, `${TableName.SecretFolder}.id`)
+        .join(TableName.Environment, `${TableName.SecretFolder}.envId`, `${TableName.Environment}.id`)
+        .where("projectId", projectId)
+        .select(selectAllTableCols(TableName.SecretReference))
+        .select("folderId");
+      return docs;
+    } catch (error) {
+      throw new DatabaseError({ error, name: "FindReferencedSecretReferences" });
+    }
+  };
+
+  // special query to backfill secret value
+  const findAllProjectSecretValues = async (projectId: string, tx?: Knex) => {
+    try {
+      const docs = await (tx || db)(TableName.Secret)
+        .join(TableName.SecretFolder, `${TableName.Secret}.folderId`, `${TableName.SecretFolder}.id`)
+        .join(TableName.Environment, `${TableName.SecretFolder}.envId`, `${TableName.Environment}.id`)
+        .where("projectId", projectId)
+        // not empty
+        .whereNotNull("secretValueCiphertext")
+        .select("secretValueTag", "secretValueCiphertext", "secretValueIV", `${TableName.Secret}.id` as "id");
+      return docs;
+    } catch (error) {
+      throw new DatabaseError({ error, name: "FindAllProjectSecretValues" });
+    }
+  };
+
   return {
     ...secretOrm,
     update,
@@ -252,6 +320,9 @@ export const secretDALFactory = (db: TDbClient) => {
     getSecretTags,
     findByFolderId,
     findByFolderIds,
-    findByBlindIndexes
+    findByBlindIndexes,
+    upsertSecretReferences,
+    findReferencedSecretReferences,
+    findAllProjectSecretValues
   };
 };

backend/src/services/secret/secret-fns.ts

@@ -194,6 +194,7 @@ type TInterpolateSecretArg = {
   folderDAL: Pick<TSecretFolderDALFactory, "findBySecretPath">;
 };
 
+const INTERPOLATION_SYNTAX_REG = /\${([^}]+)}/g;
 export const interpolateSecrets = ({ projectId, secretEncKey, secretDAL, folderDAL }: TInterpolateSecretArg) => {
   const fetchSecretsCrossEnv = () => {
     const fetchCache: Record<string, Record<string, string>> = {};
@@ -235,7 +236,6 @@ export const interpolateSecrets = ({ projectId, secretEncKey, secretDAL, folderD
     };
   };
 
-  const INTERPOLATION_SYNTAX_REG = /\${([^}]+)}/g;
   const recursivelyExpandSecret = async (
     expandedSec: Record<string, string>,
     interpolatedSec: Record<string, string>,
@@ -353,7 +353,7 @@ export const interpolateSecrets = ({ projectId, secretEncKey, secretDAL, folderD
 };
 
 export const decryptSecretRaw = (
-  secret: TSecrets & { workspace: string; environment: string; secretPath?: string },
+  secret: TSecrets & { workspace: string; environment: string; secretPath: string },
   key: string
 ) => {
   const secretKey = decryptSymmetric128BitHexKeyUTF8({
@@ -396,6 +396,37 @@ export const decryptSecretRaw = (
   };
 };
 
+/**
+ * Grabs and processes nested secret references from a string
+ *
+ * This function looks for patterns that match the interpolation syntax in the input string.
+ * It filters out references that include nested paths, splits them into environment and
+ * secret path parts, and then returns an array of objects with the environment and the
+ * joined secret path.
+ *
+ * @param {string} maybeSecretReference - The string that has the potential secret references.
+ * @returns {Array<{ environment: string, secretPath: string }>} - An array of objects
+ * with the environment and joined secret path.
+ *
+ * @example
+ * const value = "Hello ${dev.someFolder.OtherFolder.SECRET_NAME} and ${prod.anotherFolder.SECRET_NAME}";
+ * const result = getAllNestedSecretReferences(value);
+ * // result will be:
+ * // [
+ * //   { environment: 'dev', secretPath: '/someFolder/OtherFolder' },
+ * //   { environment: 'prod', secretPath: '/anotherFolder' }
+ * // ]
+ */
+export const getAllNestedSecretReferences = (maybeSecretReference: string) => {
+  const references = Array.from(maybeSecretReference.matchAll(INTERPOLATION_SYNTAX_REG), (m) => m[1]);
+  return references
+    .filter((el) => el.includes("."))
+    .map((el) => {
+      const [environment, ...secretPathList] = el.split(".");
+      return { environment, secretPath: path.join("/", ...secretPathList.slice(0, -1)) };
+    });
+};
+
 /**
  * Checks and handles secrets using a blind index method.
  * The function generates mappings between secret names and their blind indexes, validates user IDs for personal secrets, and retrieves secrets from the database based on their blind indexes.
@@ -467,7 +498,7 @@ export const fnSecretBulkInsert = async ({
   tx
 }: TFnSecretBulkInsert) => {
   const newSecrets = await secretDAL.insertMany(
-    inputSecrets.map(({ tags, ...el }) => ({ ...el, folderId })),
+    inputSecrets.map(({ tags, references, ...el }) => ({ ...el, folderId })),
     tx
   );
   const newSecretGroupByBlindIndex = groupBy(newSecrets, (item) => item.secretBlindIndex as string);
@@ -478,13 +509,20 @@ export const fnSecretBulkInsert = async ({
     }))
   );
   const secretVersions = await secretVersionDAL.insertMany(
-    inputSecrets.map(({ tags, ...el }) => ({
+    inputSecrets.map(({ tags, references, ...el }) => ({
       ...el,
       folderId,
       secretId: newSecretGroupByBlindIndex[el.secretBlindIndex as string][0].id
     })),
     tx
   );
+  await secretDAL.upsertSecretReferences(
+    inputSecrets.map(({ references = [], secretBlindIndex }) => ({
+      secretId: newSecretGroupByBlindIndex[secretBlindIndex as string][0].id,
+      references
+    })),
+    tx
+  );
   if (newSecretTags.length) {
     const secTags = await secretTagDAL.saveTagsToSecret(newSecretTags, tx);
     const secVersionsGroupBySecId = groupBy(secretVersions, (i) => i.secretId);
@@ -509,7 +547,7 @@ export const fnSecretBulkUpdate = async ({
   secretVersionTagDAL
 }: TFnSecretBulkUpdate) => {
   const newSecrets = await secretDAL.bulkUpdate(
-    inputSecrets.map(({ filter, data: { tags, ...data } }) => ({
+    inputSecrets.map(({ filter, data: { tags, references, ...data } }) => ({
       filter: { ...filter, folderId },
       data
     })),
@@ -522,6 +560,15 @@ export const fnSecretBulkUpdate = async ({
     })),
     tx
   );
+  await secretDAL.upsertSecretReferences(
+    inputSecrets
+      .filter(({ data: { references } }) => Boolean(references))
+      .map(({ data: { references = [] } }, i) => ({
+        secretId: newSecrets[i].id,
+        references
+      })),
+    tx
+  );
   const secsUpdatedTag = inputSecrets.flatMap(({ data: { tags } }, i) =>
     tags !== undefined ? { tags, secretId: newSecrets[i].id } : []
   );
@@ -591,50 +638,39 @@ export const createManySecretsRawFnFactory = ({
       folderId,
       isNew: true,
       blindIndexCfg,
+      userId,
       secretDAL
     });
 
-    const inputSecrets = await Promise.all(
+    const inputSecrets = secrets.map((secret) => {
-      secrets.map(async (secret) => {
+      const secretKeyEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretName, botKey);
-        const secretKeyEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretName, botKey);
+      const secretValueEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretValue || "", botKey);
-        const secretValueEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretValue || "", botKey);
+      const secretReferences = getAllNestedSecretReferences(secret.secretValue || "");
-        const secretCommentEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretComment || "", botKey);
+      const secretCommentEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretComment || "", botKey);
 
-        if (secret.type === SecretType.Personal) {
+      return {
-          if (!userId) throw new BadRequestError({ message: "Missing user id for personal secret" });
+        type: secret.type,
-          const sharedExist = await secretDAL.findOne({
+        userId: secret.type === SecretType.Personal ? userId : null,
-            secretBlindIndex: keyName2BlindIndex[secret.secretName],
+        secretName: secret.secretName,
-            folderId,
+        secretKeyCiphertext: secretKeyEncrypted.ciphertext,
-            type: SecretType.Shared
+        secretKeyIV: secretKeyEncrypted.iv,
-          });
+        secretKeyTag: secretKeyEncrypted.tag,
+        secretValueCiphertext: secretValueEncrypted.ciphertext,
+        secretValueIV: secretValueEncrypted.iv,
+        secretValueTag: secretValueEncrypted.tag,
+        secretCommentCiphertext: secretCommentEncrypted.ciphertext,
+        secretCommentIV: secretCommentEncrypted.iv,
+        secretCommentTag: secretCommentEncrypted.tag,
+        skipMultilineEncoding: secret.skipMultilineEncoding,
+        tags: secret.tags,
+        references: secretReferences
+      };
+    });
 
-          if (!sharedExist)
+    // get all tags
-            throw new BadRequestError({
+    const tagIds = inputSecrets.flatMap(({ tags = [] }) => tags);
-              message: "Failed to create personal secret override for no corresponding shared secret"
+    const tags = tagIds.length ? await secretTagDAL.findManyTagsById(projectId, tagIds) : [];
-            });
+    if (tags.length !== tagIds.length) throw new BadRequestError({ message: "Tag not found" });
-        }
-
-        const tags = secret.tags ? await secretTagDAL.findManyTagsById(projectId, secret.tags) : [];
-        if ((secret.tags || []).length !== tags.length) throw new BadRequestError({ message: "Tag not found" });
-
-        return {
-          type: secret.type,
-          userId: secret.type === SecretType.Personal ? userId : null,
-          secretName: secret.secretName,
-          secretKeyCiphertext: secretKeyEncrypted.ciphertext,
-          secretKeyIV: secretKeyEncrypted.iv,
-          secretKeyTag: secretKeyEncrypted.tag,
-          secretValueCiphertext: secretValueEncrypted.ciphertext,
-          secretValueIV: secretValueEncrypted.iv,
-          secretValueTag: secretValueEncrypted.tag,
-          secretCommentCiphertext: secretCommentEncrypted.ciphertext,
-          secretCommentIV: secretCommentEncrypted.iv,
-          secretCommentTag: secretCommentEncrypted.tag,
-          skipMultilineEncoding: secret.skipMultilineEncoding,
-          tags: secret.tags
-        };
-      })
-    );
 
     const newSecrets = await secretDAL.transaction(async (tx) =>
       fnSecretBulkInsert({
@@ -703,56 +739,35 @@ export const updateManySecretsRawFnFactory = ({
       userId
     });
 
-    const inputSecrets = await Promise.all(
+    const inputSecrets = secrets.map((secret) => {
-      secrets.map(async (secret) => {
+      if (secret.newSecretName === "") {
-        if (secret.newSecretName === "") {
+        throw new BadRequestError({ message: "New secret name cannot be empty" });
-          throw new BadRequestError({ message: "New secret name cannot be empty" });
+      }
-        }
 
-        const secretKeyEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretName, botKey);
+      const secretKeyEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretName, botKey);
-        const secretValueEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretValue || "", botKey);
+      const secretValueEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretValue || "", botKey);
-        const secretCommentEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretComment || "", botKey);
+      const secretReferences = getAllNestedSecretReferences(secret.secretValue || "");
+      const secretCommentEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretComment || "", botKey);
 
-        if (secret.type === SecretType.Personal) {
+      return {
-          if (!userId) throw new BadRequestError({ message: "Missing user id for personal secret" });
+        type: secret.type,
-
+        userId: secret.type === SecretType.Personal ? userId : null,
-          const sharedExist = await secretDAL.findOne({
+        secretName: secret.secretName,
-            secretBlindIndex: keyName2BlindIndex[secret.secretName],
+        newSecretName: secret.newSecretName,
-            folderId,
+        secretKeyCiphertext: secretKeyEncrypted.ciphertext,
-            type: SecretType.Shared
+        secretKeyIV: secretKeyEncrypted.iv,
-          });
+        secretKeyTag: secretKeyEncrypted.tag,
-
+        secretValueCiphertext: secretValueEncrypted.ciphertext,
-          if (!sharedExist)
+        secretValueIV: secretValueEncrypted.iv,
-            throw new BadRequestError({
+        secretValueTag: secretValueEncrypted.tag,
-              message: "Failed to update personal secret override for no corresponding shared secret"
+        secretCommentCiphertext: secretCommentEncrypted.ciphertext,
-            });
+        secretCommentIV: secretCommentEncrypted.iv,
-
+        secretCommentTag: secretCommentEncrypted.tag,
-          if (secret.newSecretName)
+        skipMultilineEncoding: secret.skipMultilineEncoding,
-            throw new BadRequestError({ message: "Personal secret cannot change the key name" });
+        tags: secret.tags,
-        }
+        references: secretReferences
-
+      };
-        const tags = secret.tags ? await secretTagDAL.findManyTagsById(projectId, secret.tags) : [];
+    });
-        if ((secret.tags || []).length !== tags.length) throw new BadRequestError({ message: "Tag not found" });
-
-        return {
-          type: secret.type,
-          userId: secret.type === SecretType.Personal ? userId : null,
-          secretName: secret.secretName,
-          newSecretName: secret.newSecretName,
-          secretKeyCiphertext: secretKeyEncrypted.ciphertext,
-          secretKeyIV: secretKeyEncrypted.iv,
-          secretKeyTag: secretKeyEncrypted.tag,
-          secretValueCiphertext: secretValueEncrypted.ciphertext,
-          secretValueIV: secretValueEncrypted.iv,
-          secretValueTag: secretValueEncrypted.tag,
-          secretCommentCiphertext: secretCommentEncrypted.ciphertext,
-          secretCommentIV: secretCommentEncrypted.iv,
-          secretCommentTag: secretCommentEncrypted.tag,
-          skipMultilineEncoding: secret.skipMultilineEncoding,
-          tags: secret.tags
-        };
-      })
-    );
 
     const tagIds = inputSecrets.flatMap(({ tags = [] }) => tags);
     const tags = tagIds.length ? await secretTagDAL.findManyTagsById(projectId, tagIds) : [];

backend/src/services/secret/secret-queue.ts

@@ -59,6 +59,7 @@ export type TGetSecrets = {
 };
 
 const MAX_SYNC_SECRET_DEPTH = 5;
+const uniqueIntegrationKey = (environment: string, secretPath: string) => `integration-${environment}-${secretPath}`;
 
 export const secretQueueFactory = ({
   queueService,
@@ -102,28 +103,35 @@ export const secretQueueFactory = ({
     folderDAL
   });
 
-  const syncIntegrations = async (dto: TGetSecrets) => {
+  const syncIntegrations = async (dto: TGetSecrets & { deDupeQueue?: Record<string, boolean> }) => {
     await queueService.queue(QueueName.IntegrationSync, QueueJobs.IntegrationSync, dto, {
-      attempts: 5,
+      attempts: 3,
       delay: 1000,
       backoff: {
         type: "exponential",
         delay: 3000
       },
       removeOnComplete: true,
-      removeOnFail: {
+      removeOnFail: true
-        count: 5 // keep the most recent  jobs
-      }
     });
   };
 
-  const syncSecrets = async (dto: TGetSecrets & { depth?: number }) => {
+  const syncSecrets = async ({
+    deDupeQueue = {},
+    ...dto
+  }: TGetSecrets & { depth?: number; deDupeQueue?: Record<string, boolean> }) => {
+    const deDuplicationKey = uniqueIntegrationKey(dto.environment, dto.secretPath);
+    if (deDupeQueue?.[deDuplicationKey]) {
+      return;
+    }
+    // eslint-disable-next-line
+    deDupeQueue[deDuplicationKey] = true;
     logger.info(
       `syncSecrets: syncing project secrets where [projectId=${dto.projectId}]  [environment=${dto.environment}] [path=${dto.secretPath}]`
     );
     await queueService.queue(QueueName.SecretWebhook, QueueJobs.SecWebhook, dto, {
       jobId: `secret-webhook-${dto.environment}-${dto.projectId}-${dto.secretPath}`,
-      removeOnFail: { count: 5 },
+      removeOnFail: true,
       removeOnComplete: true,
       delay: 1000,
       attempts: 5,
@@ -132,7 +140,7 @@ export const secretQueueFactory = ({
         delay: 3000
       }
     });
-    await syncIntegrations(dto);
+    await syncIntegrations({ ...dto, deDupeQueue });
   };
 
   const removeSecretReminder = async (dto: TRemoveSecretReminderDTO) => {
@@ -326,7 +334,7 @@ export const secretQueueFactory = ({
   };
 
   queueService.start(QueueName.IntegrationSync, async (job) => {
-    const { environment, projectId, secretPath, depth = 1 } = job.data;
+    const { environment, projectId, secretPath, depth = 1, deDupeQueue = {} } = job.data;
 
     const folder = await folderDAL.findBySecretPath(projectId, environment, secretPath);
     if (!folder) {
@@ -349,21 +357,68 @@ export const secretQueueFactory = ({
         const importedFolderIds = unique(imports, (i) => i.folderId).map(({ folderId }) => folderId);
         const importedFolders = await folderDAL.findSecretPathByFolderIds(projectId, importedFolderIds);
         const foldersGroupedById = groupBy(importedFolders, (i) => i.child || i.id);
+        logger.info(
+          `getIntegrationSecrets: Syncing secret due to link change [jobId=${job.id}] [projectId=${job.data.projectId}] [environment=${job.data.environment}]  [secretPath=${job.data.secretPath}] [depth=${depth}]`
+        );
         await Promise.all(
           imports
             .filter(({ folderId }) => Boolean(foldersGroupedById[folderId][0].path))
-            .map(({ folderId }) => {
+            // filter out already synced ones
-              const syncDto = {
+            .filter(
+              ({ folderId }) =>
+                !deDupeQueue[
+                  uniqueIntegrationKey(
+                    foldersGroupedById[folderId][0].environmentSlug,
+                    foldersGroupedById[folderId][0].path
+                  )
+                ]
+            )
+            .map(({ folderId }) =>
+              syncSecrets({
                 depth: depth + 1,
                 projectId,
                 secretPath: foldersGroupedById[folderId][0].path,
-                environment: foldersGroupedById[folderId][0].environmentSlug
+                environment: foldersGroupedById[folderId][0].environmentSlug,
-              };
+                deDupeQueue
-              logger.info(
+              })
-                `getIntegrationSecrets: Syncing secret due to link change [jobId=${job.id}] [projectId=${job.data.projectId}] [environment=${job.data.environment}]  [secretPath=${job.data.secretPath}] [depth=${depth}]`
+            )
-              );
+        );
-              return syncSecrets(syncDto);
+      }
-            })
+
+      const secretReferences = await secretDAL.findReferencedSecretReferences(
+        projectId,
+        folder.environment.slug,
+        secretPath
+      );
+      if (secretReferences.length) {
+        const referencedFolderIds = unique(secretReferences, (i) => i.folderId).map(({ folderId }) => folderId);
+        const referencedFolders = await folderDAL.findSecretPathByFolderIds(projectId, referencedFolderIds);
+        const referencedFoldersGroupedById = groupBy(referencedFolders, (i) => i.child || i.id);
+        logger.info(
+          `getIntegrationSecrets: Syncing secret due to reference change [jobId=${job.id}] [projectId=${job.data.projectId}] [environment=${job.data.environment}]  [secretPath=${job.data.secretPath}] [depth=${depth}]`
+        );
+        await Promise.all(
+          secretReferences
+            .filter(({ folderId }) => Boolean(referencedFoldersGroupedById[folderId][0].path))
+            // filter out already synced ones
+            .filter(
+              ({ folderId }) =>
+                !deDupeQueue[
+                  uniqueIntegrationKey(
+                    referencedFoldersGroupedById[folderId][0].environmentSlug,
+                    referencedFoldersGroupedById[folderId][0].path
+                  )
+                ]
+            )
+            .map(({ folderId }) =>
+              syncSecrets({
+                depth: depth + 1,
+                projectId,
+                secretPath: referencedFoldersGroupedById[folderId][0].path,
+                environment: referencedFoldersGroupedById[folderId][0].environmentSlug,
+                deDupeQueue
+              })
+            )
         );
       }
     } else {

backend/src/services/secret/secret-service.ts

@@ -2,12 +2,22 @@
 /* eslint-disable no-await-in-loop */
 import { ForbiddenError, subject } from "@casl/ability";
 
-import { SecretEncryptionAlgo, SecretKeyEncoding, SecretsSchema, SecretType } from "@app/db/schemas";
+import {
+  ProjectMembershipRole,
+  SecretEncryptionAlgo,
+  SecretKeyEncoding,
+  SecretsSchema,
+  SecretType
+} from "@app/db/schemas";
 import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service";
 import { ProjectPermissionActions, ProjectPermissionSub } from "@app/ee/services/permission/project-permission";
 import { TSecretSnapshotServiceFactory } from "@app/ee/services/secret-snapshot/secret-snapshot-service";
 import { getConfig } from "@app/lib/config/env";
-import { buildSecretBlindIndexFromName, encryptSymmetric128BitHexKeyUTF8 } from "@app/lib/crypto";
+import {
+  buildSecretBlindIndexFromName,
+  decryptSymmetric128BitHexKeyUTF8,
+  encryptSymmetric128BitHexKeyUTF8
+} from "@app/lib/crypto";
 import { BadRequestError } from "@app/lib/errors";
 import { groupBy, pick } from "@app/lib/fn";
 import { logger } from "@app/lib/logger";
@@ -27,12 +37,14 @@ import {
   fnSecretBlindIndexCheck,
   fnSecretBulkInsert,
   fnSecretBulkUpdate,
+  getAllNestedSecretReferences,
   interpolateSecrets,
   recursivelyGetSecretPaths
 } from "./secret-fns";
 import { TSecretQueueFactory } from "./secret-queue";
 import {
   TAttachSecretTagsDTO,
+  TBackFillSecretReferencesDTO,
   TCreateBulkSecretDTO,
   TCreateManySecretRawDTO,
   TCreateSecretDTO,
@@ -91,6 +103,22 @@ export const secretServiceFactory = ({
   secretImportDAL,
   secretVersionTagDAL
 }: TSecretServiceFactoryDep) => {
+  const getSecretReference = async (projectId: string) => {
+    // if bot key missing means e2e still exist
+    const botKey = await projectBotService.getBotKey(projectId).catch(() => null);
+    return (el: { ciphertext?: string; iv: string; tag: string }) =>
+      botKey
+        ? getAllNestedSecretReferences(
+            decryptSymmetric128BitHexKeyUTF8({
+              ciphertext: el.ciphertext || "",
+              iv: el.iv,
+              tag: el.tag,
+              key: botKey
+            })
+          )
+        : undefined;
+  };
+
   // utility function to get secret blind index data
   const interalGenSecBlindIndexByName = async (projectId: string, secretName: string) => {
     const appCfg = getConfig();
@@ -225,6 +253,7 @@ export const secretServiceFactory = ({
     if ((inputSecret.tags || []).length !== tags.length) throw new BadRequestError({ message: "Tag not found" });
 
     const { secretName, type, ...el } = inputSecret;
+    const references = await getSecretReference(projectId);
     const secret = await secretDAL.transaction((tx) =>
       fnSecretBulkInsert({
         folderId,
@@ -237,7 +266,12 @@ export const secretServiceFactory = ({
             userId: inputSecret.type === SecretType.Personal ? actorId : null,
             algorithm: SecretEncryptionAlgo.AES_256_GCM,
             keyEncoding: SecretKeyEncoding.UTF8,
-            tags: inputSecret.tags
+            tags: inputSecret.tags,
+            references: references({
+              ciphertext: inputSecret.secretValueCiphertext,
+              iv: inputSecret.secretValueIV,
+              tag: inputSecret.secretValueTag
+            })
           }
         ],
         secretDAL,
@@ -251,7 +285,7 @@ export const secretServiceFactory = ({
     await snapshotService.performSnapshot(folderId);
     await secretQueueService.syncSecrets({ secretPath: path, projectId, environment });
     // TODO(akhilmhdh-pg): licence check, posthog service and snapshot
-    return { ...secret[0], environment, workspace: projectId, tags };
+    return { ...secret[0], environment, workspace: projectId, tags, secretPath: path };
   };
 
   const updateSecret = async ({
@@ -335,6 +369,7 @@ export const secretServiceFactory = ({
 
     const { secretName, ...el } = inputSecret;
 
+    const references = await getSecretReference(projectId);
     const updatedSecret = await secretDAL.transaction(async (tx) =>
       fnSecretBulkUpdate({
         folderId,
@@ -360,7 +395,12 @@ export const secretServiceFactory = ({
                 "secretReminderRepeatDays",
                 "tags"
               ]),
-              secretBlindIndex: newSecretNameBlindIndex || keyName2BlindIndex[secretName]
+              secretBlindIndex: newSecretNameBlindIndex || keyName2BlindIndex[secretName],
+              references: references({
+                ciphertext: inputSecret.secretValueCiphertext,
+                iv: inputSecret.secretValueIV,
+                tag: inputSecret.secretValueTag
+              })
             }
           }
         ],
@@ -375,7 +415,7 @@ export const secretServiceFactory = ({
     await snapshotService.performSnapshot(folderId);
     await secretQueueService.syncSecrets({ secretPath: path, projectId, environment });
     // TODO(akhilmhdh-pg): licence check, posthog service and snapshot
-    return { ...updatedSecret[0], workspace: projectId, environment };
+    return { ...updatedSecret[0], workspace: projectId, environment, secretPath: path };
   };
 
   const deleteSecret = async ({
@@ -444,7 +484,7 @@ export const secretServiceFactory = ({
     await secretQueueService.syncSecrets({ secretPath: path, projectId, environment });
 
     // TODO(akhilmhdh-pg): licence check, posthog service and snapshot
-    return { ...deletedSecret[0], _id: deletedSecret[0].id, workspace: projectId, environment };
+    return { ...deletedSecret[0], _id: deletedSecret[0].id, workspace: projectId, environment, secretPath: path };
   };
 
   const getSecrets = async ({
@@ -641,7 +681,8 @@ export const secretServiceFactory = ({
             return {
               ...importedSecrets[i].secrets[j],
               workspace: projectId,
-              environment: importedSecrets[i].environment
+              environment: importedSecrets[i].environment,
+              secretPath: importedSecrets[i].secretPath
             };
           }
         }
@@ -649,7 +690,7 @@ export const secretServiceFactory = ({
     }
     if (!secret) throw new BadRequestError({ message: "Secret not found" });
 
-    return { ...secret, workspace: projectId, environment };
+    return { ...secret, workspace: projectId, environment, secretPath: path };
   };
 
   const createManySecret = async ({
@@ -700,6 +741,7 @@ export const secretServiceFactory = ({
     const tags = tagIds.length ? await secretTagDAL.findManyTagsById(projectId, tagIds) : [];
     if (tags.length !== tagIds.length) throw new BadRequestError({ message: "Tag not found" });
 
+    const references = await getSecretReference(projectId);
     const newSecrets = await secretDAL.transaction(async (tx) =>
       fnSecretBulkInsert({
         inputSecrets: inputSecrets.map(({ secretName, ...el }) => ({
@@ -708,7 +750,12 @@ export const secretServiceFactory = ({
           secretBlindIndex: keyName2BlindIndex[secretName],
           type: SecretType.Shared,
           algorithm: SecretEncryptionAlgo.AES_256_GCM,
-          keyEncoding: SecretKeyEncoding.UTF8
+          keyEncoding: SecretKeyEncoding.UTF8,
+          references: references({
+            ciphertext: el.secretValueCiphertext,
+            iv: el.secretValueIV,
+            tag: el.secretValueTag
+          })
         })),
         folderId,
         secretDAL,
@@ -783,6 +830,8 @@ export const secretServiceFactory = ({
     const tagIds = inputSecrets.flatMap(({ tags = [] }) => tags);
     const tags = tagIds.length ? await secretTagDAL.findManyTagsById(projectId, tagIds) : [];
     if (tagIds.length !== tags.length) throw new BadRequestError({ message: "Tag not found" });
+
+    const references = await getSecretReference(projectId);
     const secrets = await secretDAL.transaction(async (tx) =>
       fnSecretBulkUpdate({
         folderId,
@@ -799,7 +848,15 @@ export const secretServiceFactory = ({
                 ? newKeyName2BlindIndex[newSecretName]
                 : keyName2BlindIndex[secretName],
             algorithm: SecretEncryptionAlgo.AES_256_GCM,
-            keyEncoding: SecretKeyEncoding.UTF8
+            keyEncoding: SecretKeyEncoding.UTF8,
+            references:
+              el.secretValueIV && el.secretValueTag
+                ? references({
+                    ciphertext: el.secretValueCiphertext,
+                    iv: el.secretValueIV,
+                    tag: el.secretValueTag
+                  })
+                : undefined
           }
         })),
         secretDAL,
@@ -924,34 +981,40 @@ export const secretServiceFactory = ({
       });
 
       const batchSecretsExpand = async (
-        secretBatch: {
+        secretBatch: { secretKey: string; secretValue: string; secretComment?: string; secretPath: string }[]
-          secretKey: string;
-          secretValue: string;
-          secretComment?: string;
-        }[]
       ) => {
-        const secretRecord: Record<
+        // Group secrets by secretPath
-          string,
+        const secretsByPath: Record<string, { secretKey: string; secretValue: string; secretComment?: string }[]> = {};
-          {
+
-            value: string;
+        secretBatch.forEach((secret) => {
-            comment?: string;
+          if (!secretsByPath[secret.secretPath]) {
-            skipMultilineEncoding?: boolean;
+            secretsByPath[secret.secretPath] = [];
           }
-        > = {};
+          secretsByPath[secret.secretPath].push(secret);
-
-        secretBatch.forEach((decryptedSecret) => {
-          secretRecord[decryptedSecret.secretKey] = {
-            value: decryptedSecret.secretValue,
-            comment: decryptedSecret.secretComment
-          };
         });
 
-        await expandSecrets(secretRecord);
+        // Expand secrets for each group
+        for (const secPath in secretsByPath) {
+          if (!Object.hasOwn(secretsByPath, path)) {
+            // eslint-disable-next-line no-continue
+            continue;
+          }
 
-        secretBatch.forEach((decryptedSecret, index) => {
+          const secretRecord: Record<string, { value: string; comment?: string; skipMultilineEncoding?: boolean }> = {};
-          // eslint-disable-next-line no-param-reassign
+          secretsByPath[secPath].forEach((decryptedSecret) => {
-          secretBatch[index].secretValue = secretRecord[decryptedSecret.secretKey].value;
+            secretRecord[decryptedSecret.secretKey] = {
-        });
+              value: decryptedSecret.secretValue,
+              comment: decryptedSecret.secretComment
+            };
+          });
+
+          await expandSecrets(secretRecord);
+
+          secretsByPath[secPath].forEach((decryptedSecret) => {
+            // eslint-disable-next-line no-param-reassign
+            decryptedSecret.secretValue = secretRecord[decryptedSecret.secretKey].value;
+          });
+        }
       };
 
       // expand secrets
@@ -999,6 +1062,7 @@ export const secretServiceFactory = ({
       includeImports,
       version
     });
+
     return decryptSecretRaw(secret, botKey);
   };
 
@@ -1171,7 +1235,9 @@ export const secretServiceFactory = ({
     await snapshotService.performSnapshot(secrets[0].folderId);
     await secretQueueService.syncSecrets({ secretPath, projectId, environment });
 
-    return secrets.map((secret) => decryptSecretRaw({ ...secret, workspace: projectId, environment }, botKey));
+    return secrets.map((secret) =>
+      decryptSecretRaw({ ...secret, workspace: projectId, environment, secretPath }, botKey)
+    );
   };
 
   const updateManySecretsRaw = async ({
@@ -1223,7 +1289,9 @@ export const secretServiceFactory = ({
     await snapshotService.performSnapshot(secrets[0].folderId);
     await secretQueueService.syncSecrets({ secretPath, projectId, environment });
 
-    return secrets.map((secret) => decryptSecretRaw({ ...secret, workspace: projectId, environment }, botKey));
+    return secrets.map((secret) =>
+      decryptSecretRaw({ ...secret, workspace: projectId, environment, secretPath }, botKey)
+    );
   };
 
   const deleteManySecretsRaw = async ({
@@ -1257,7 +1325,9 @@ export const secretServiceFactory = ({
     await snapshotService.performSnapshot(secrets[0].folderId);
     await secretQueueService.syncSecrets({ secretPath, projectId, environment });
 
-    return secrets.map((secret) => decryptSecretRaw({ ...secret, workspace: projectId, environment }, botKey));
+    return secrets.map((secret) =>
+      decryptSecretRaw({ ...secret, workspace: projectId, environment, secretPath }, botKey)
+    );
   };
 
   const getSecretVersions = async ({
@@ -1488,6 +1558,52 @@ export const secretServiceFactory = ({
     };
   };
 
+  // this is a backfilling API for secret references
+  // what it does is it will go through all the secret values and parse all references
+  // populate the secret reference to do sync integrations
+  const backfillSecretReferences = async ({
+    projectId,
+    actor,
+    actorId,
+    actorOrgId,
+    actorAuthMethod
+  }: TBackFillSecretReferencesDTO) => {
+    const { hasRole } = await permissionService.getProjectPermission(
+      actor,
+      actorId,
+      projectId,
+      actorAuthMethod,
+      actorOrgId
+    );
+
+    if (!hasRole(ProjectMembershipRole.Admin))
+      throw new BadRequestError({ message: "Only admins are allowed to take this action" });
+
+    const botKey = await projectBotService.getBotKey(projectId);
+    if (!botKey)
+      throw new BadRequestError({ message: "Please upgrade your project first", name: "bot_not_found_error" });
+
+    await secretDAL.transaction(async (tx) => {
+      const secrets = await secretDAL.findAllProjectSecretValues(projectId, tx);
+      await secretDAL.upsertSecretReferences(
+        secrets.map(({ id, secretValueCiphertext, secretValueIV, secretValueTag }) => ({
+          secretId: id,
+          references: getAllNestedSecretReferences(
+            decryptSymmetric128BitHexKeyUTF8({
+              ciphertext: secretValueCiphertext,
+              iv: secretValueIV,
+              tag: secretValueTag,
+              key: botKey
+            })
+          )
+        })),
+        tx
+      );
+    });
+
+    return { message: "Successfully backfilled secret references" };
+  };
+
   return {
     attachTags,
     detachTags,
@@ -1508,6 +1624,7 @@ export const secretServiceFactory = ({
     updateManySecretsRaw,
     deleteManySecretsRaw,
     getSecretVersions,
+    backfillSecretReferences,
     // external services function
     fnSecretBulkDelete,
     fnSecretBulkUpdate,

backend/src/services/secret/secret-types.ts

@@ -223,11 +223,13 @@ export type TGetSecretVersionsDTO = Omit<TProjectPermission, "projectId"> & {
   secretId: string;
 };
 
+export type TSecretReference = { environment: string; secretPath: string };
+
 export type TFnSecretBulkInsert = {
   folderId: string;
   tx?: Knex;
-  inputSecrets: Array<Omit<TSecretsInsert, "folderId"> & { tags?: string[] }>;
+  inputSecrets: Array<Omit<TSecretsInsert, "folderId"> & { tags?: string[]; references?: TSecretReference[] }>;
-  secretDAL: Pick<TSecretDALFactory, "insertMany">;
+  secretDAL: Pick<TSecretDALFactory, "insertMany" | "upsertSecretReferences">;
   secretVersionDAL: Pick<TSecretVersionDALFactory, "insertMany">;
   secretTagDAL: Pick<TSecretTagDALFactory, "saveTagsToSecret">;
   secretVersionTagDAL: Pick<TSecretVersionTagDALFactory, "insertMany">;
@@ -236,8 +238,11 @@ export type TFnSecretBulkInsert = {
 export type TFnSecretBulkUpdate = {
   folderId: string;
   projectId: string;
-  inputSecrets: { filter: Partial<TSecrets>; data: TSecretsUpdate & { tags?: string[] } }[];
+  inputSecrets: {
-  secretDAL: Pick<TSecretDALFactory, "bulkUpdate">;
+    filter: Partial<TSecrets>;
+    data: TSecretsUpdate & { tags?: string[]; references?: TSecretReference[] };
+  }[];
+  secretDAL: Pick<TSecretDALFactory, "bulkUpdate" | "upsertSecretReferences">;
   secretVersionDAL: Pick<TSecretVersionDALFactory, "insertMany">;
   secretTagDAL: Pick<TSecretTagDALFactory, "saveTagsToSecret" | "deleteTagsManySecret">;
   secretVersionTagDAL: Pick<TSecretVersionTagDALFactory, "insertMany">;
@@ -294,6 +299,8 @@ export type TRemoveSecretReminderDTO = {
   repeatDays: number;
 };
 
+export type TBackFillSecretReferencesDTO = TProjectPermission;
+
 // ---
 
 export type TCreateManySecretsRawFnFactory = {

docs/api-reference/endpoints/universal-auth/revoke-access-token.mdx

@@ -0,0 +1,4 @@
+---
+title: "Revoke Access Token"
+openapi: "POST /api/v1/auth/token/revoke"
+---

docs/cli/commands/export.mdx

@@ -128,6 +128,12 @@ infisical export --template=<path to template>
 
   </Accordion>
 
+  <Accordion title="--include-imports">
+    By default imported secrets are available, you can disable it by setting this option to false.
+
+    Default value: `true`
+  </Accordion>
+
   <Accordion title="--format">
     Format of the output file. Accepted values: `dotenv`, `dotenv-export`, `csv`, `json` and `yaml`
 

docs/cli/commands/run.mdx

@@ -126,6 +126,12 @@ $ infisical run -- npm run dev
 
   </Accordion>
 
+  <Accordion title="--include-imports">
+    By default imported secrets are available, you can disable it by setting this option to false.
+
+    Default value: `true`
+  </Accordion>
+
 {" "}
 
 <Accordion title="--env">

docs/cli/faq.mdx

@@ -13,6 +13,7 @@ If none of the available stores work for you, you can try using the `file` store
 If you are still experiencing trouble, please seek support. 
 
 [Learn more about vault command](./commands/vault)
+
 </Accordion>
 
 <Accordion title="Can I fetch secrets with Infisical if I am offline?">

docs/documentation/platform/identities/gcp-auth.mdx

@@ -123,7 +123,7 @@ access the Infisical API using the GCP ID Token authentication method.
                 <CodeGroup>
                 ```bash curl
                 curl -H "Metadata-Flavor: Google" \
-                'http://metadata/computeMetadata/v1/instance/service-accounts/default/identity?audience=<identityId>'
+                'http://metadata/computeMetadata/v1/instance/service-accounts/default/identity?audience=<identityId>&format=full'
                 ```
                 </CodeGroup>
 

docs/documentation/platform/secret-versioning.mdx

@@ -3,7 +3,7 @@ title: "Secret Versioning"
 description: "Learn how secret versioning works in Infisical."
 ---
 
-Every time a secret change is persformed, a new version of the same secret is created. 
+Every time a secret change is performed, a new version of the same secret is created. 
 
 Such versions can be accessed visually by opening up the [secret sidebar](/documentation/platform/project#drawer) (as seen below) or [retrieved via API](/api-reference/endpoints/secrets/read) 
 by specifying the `version` query parameter.

docs/mint.json

@@ -417,7 +417,8 @@
             "api-reference/endpoints/universal-auth/create-client-secret",
             "api-reference/endpoints/universal-auth/list-client-secrets",
             "api-reference/endpoints/universal-auth/revoke-client-secret",
-            "api-reference/endpoints/universal-auth/renew-access-token"
+            "api-reference/endpoints/universal-auth/renew-access-token",
+            "api-reference/endpoints/universal-auth/revoke-access-token"
           ]
         },
         {

frontend/src/hooks/api/secrets/index.ts

@@ -1,4 +1,5 @@
 export {
+  useBackfillSecretReference,
   useCreateSecretBatch,
   useCreateSecretV3,
   useDeleteSecretBatch,

frontend/src/hooks/api/secrets/mutations.tsx

@@ -87,11 +87,11 @@ export const useCreateSecretV3 = ({
 
       const randomBytes = latestFileKey
         ? decryptAssymmetric({
-            ciphertext: latestFileKey.encryptedKey,
+          ciphertext: latestFileKey.encryptedKey,
-            nonce: latestFileKey.nonce,
+          nonce: latestFileKey.nonce,
-            publicKey: latestFileKey.sender.publicKey,
+          publicKey: latestFileKey.sender.publicKey,
-            privateKey: PRIVATE_KEY
+          privateKey: PRIVATE_KEY
-          })
+        })
         : crypto.randomBytes(16).toString("hex");
 
       const reqBody = {
@@ -148,11 +148,11 @@ export const useUpdateSecretV3 = ({
 
       const randomBytes = latestFileKey
         ? decryptAssymmetric({
-            ciphertext: latestFileKey.encryptedKey,
+          ciphertext: latestFileKey.encryptedKey,
-            nonce: latestFileKey.nonce,
+          nonce: latestFileKey.nonce,
-            publicKey: latestFileKey.sender.publicKey,
+          publicKey: latestFileKey.sender.publicKey,
-            privateKey: PRIVATE_KEY
+          privateKey: PRIVATE_KEY
-          })
+        })
         : crypto.randomBytes(16).toString("hex");
 
       const reqBody = {
@@ -244,11 +244,11 @@ export const useCreateSecretBatch = ({
       const PRIVATE_KEY = localStorage.getItem("PRIVATE_KEY") as string;
       const randomBytes = latestFileKey
         ? decryptAssymmetric({
-            ciphertext: latestFileKey.encryptedKey,
+          ciphertext: latestFileKey.encryptedKey,
-            nonce: latestFileKey.nonce,
+          nonce: latestFileKey.nonce,
-            publicKey: latestFileKey.sender.publicKey,
+          publicKey: latestFileKey.sender.publicKey,
-            privateKey: PRIVATE_KEY
+          privateKey: PRIVATE_KEY
-          })
+        })
         : crypto.randomBytes(16).toString("hex");
 
       const reqBody = {
@@ -297,11 +297,11 @@ export const useUpdateSecretBatch = ({
       const PRIVATE_KEY = localStorage.getItem("PRIVATE_KEY") as string;
       const randomBytes = latestFileKey
         ? decryptAssymmetric({
-            ciphertext: latestFileKey.encryptedKey,
+          ciphertext: latestFileKey.encryptedKey,
-            nonce: latestFileKey.nonce,
+          nonce: latestFileKey.nonce,
-            publicKey: latestFileKey.sender.publicKey,
+          publicKey: latestFileKey.sender.publicKey,
-            privateKey: PRIVATE_KEY
+          privateKey: PRIVATE_KEY
-          })
+        })
         : crypto.randomBytes(16).toString("hex");
 
       const reqBody = {
@@ -379,3 +379,13 @@ export const createSecret = async (dto: CreateSecretDTO) => {
   const { data } = await apiRequest.post(`/api/v3/secrets/${dto.secretKey}`, dto);
   return data;
 };
+
+export const useBackfillSecretReference = () =>
+  useMutation<{ message: string }, {}, { projectId: string }>({
+    mutationFn: async ({ projectId }) => {
+      const { data } = await apiRequest.post("/api/v3/secrets/backfill-secret-references", {
+        projectId
+      });
+      return data.message;
+    }
+  });

frontend/src/views/Settings/ProjectSettingsPage/components/BackfillSecretReferenceSection/BackfillSecretReferenceSection.tsx

@@ -0,0 +1,43 @@
+import { createNotification } from "@app/components/notifications";
+import { Button } from "@app/components/v2";
+import { useProjectPermission, useWorkspace } from "@app/context";
+import { useBackfillSecretReference } from "@app/hooks/api";
+import { ProjectMembershipRole } from "@app/hooks/api/roles/types";
+
+export const BackfillSecretReferenceSecretion = () => {
+	const { currentWorkspace } = useWorkspace();
+	const { membership } = useProjectPermission();
+	const backfillSecretReferences = useBackfillSecretReference();
+
+	if (!currentWorkspace) return null;
+
+	const handleBackfill = async () => {
+		if (backfillSecretReferences.isLoading) return;
+		try {
+			await backfillSecretReferences.mutateAsync({ projectId: currentWorkspace.id || "" });
+			createNotification({ text: "Successfully re-indexed secret references", type: "success" });
+		} catch {
+			createNotification({ text: "Failed to re-index secret references", type: "error" });
+		}
+	};
+
+	const isAdmin = membership.roles.includes(ProjectMembershipRole.Admin);
+	return (
+		<div className="mb-6 rounded-lg border border-mineshaft-600 bg-mineshaft-900 p-4">
+			<div className="flex w-full items-center justify-between">
+				<p className="text-xl font-semibold">Index Secret References</p>
+			</div>
+			<p className="mb-4 mt-2 max-w-2xl text-sm text-gray-400">
+				This will index all secret references, enabling integrations to be triggered when their values change going forward.
+			</p>
+			<Button
+				variant="outline_bg"
+				isLoading={backfillSecretReferences.isLoading}
+				onClick={handleBackfill}
+				isDisabled={!isAdmin}
+			>
+				Index Secret References
+			</Button>
+		</div>
+	);
+};

frontend/src/views/Settings/ProjectSettingsPage/components/BackfillSecretReferenceSection/index.tsx

@@ -0,0 +1 @@
+export { BackfillSecretReferenceSecretion } from "./BackfillSecretReferenceSection";

frontend/src/views/Settings/ProjectSettingsPage/components/ProjectGeneralTab/ProjectGeneralTab.tsx

@@ -1,4 +1,5 @@
 import { AutoCapitalizationSection } from "../AutoCapitalizationSection";
+import { BackfillSecretReferenceSecretion } from "../BackfillSecretReferenceSection";
 import { DeleteProjectSection } from "../DeleteProjectSection";
 import { E2EESection } from "../E2EESection";
 import { EnvironmentSection } from "../EnvironmentSection";
@@ -13,6 +14,7 @@ export const ProjectGeneralTab = () => {
       <SecretTagsSection />
       <AutoCapitalizationSection />
       <E2EESection />
+      <BackfillSecretReferenceSecretion />
       <DeleteProjectSection />
     </div>
   );

frontend/src/views/Settings/ProjectSettingsPage/components/index.tsx

@@ -1,4 +1,5 @@
 export { AutoCapitalizationSection } from "./AutoCapitalizationSection";
+export { BackfillSecretReferenceSecretion } from "./BackfillSecretReferenceSection";
 export { DeleteProjectSection } from "./DeleteProjectSection";
 export { E2EESection } from "./E2EESection";
 export { EnvironmentSection } from "./EnvironmentSection";