Merge pull request #1825 from akhilmhdh/feat/sync-integration-inline

Secret reference and integration sync support

backend/src/@types/knex.d.ts

@@ -231,6 +231,7 @@ import {
   TWebhooksInsert,
   TWebhooksUpdate
 } from "@app/db/schemas";
+import { TSecretReferences, TSecretReferencesInsert, TSecretReferencesUpdate } from "@app/db/schemas/secret-references";
 
 declare module "knex/types/tables" {
   interface Tables {
@@ -304,6 +305,11 @@ declare module "knex/types/tables" {
     >;
     [TableName.ProjectKeys]: Knex.CompositeTableType<TProjectKeys, TProjectKeysInsert, TProjectKeysUpdate>;
     [TableName.Secret]: Knex.CompositeTableType<TSecrets, TSecretsInsert, TSecretsUpdate>;
+    [TableName.SecretReference]: Knex.CompositeTableType<
+      TSecretReferences,
+      TSecretReferencesInsert,
+      TSecretReferencesUpdate
+    >;
     [TableName.SecretBlindIndex]: Knex.CompositeTableType<
       TSecretBlindIndexes,
       TSecretBlindIndexesInsert,

backend/src/db/migrations/20240514141809_inline-secret-reference-sync.ts

@@ -0,0 +1,24 @@
+import { Knex } from "knex";
+
+import { TableName } from "../schemas";
+import { createOnUpdateTrigger, dropOnUpdateTrigger } from "../utils";
+
+export async function up(knex: Knex): Promise<void> {
+  if (!(await knex.schema.hasTable(TableName.SecretReference))) {
+    await knex.schema.createTable(TableName.SecretReference, (t) => {
+      t.uuid("id", { primaryKey: true }).defaultTo(knex.fn.uuid());
+      t.string("environment").notNullable();
+      t.string("secretPath").notNullable();
+      t.uuid("secretId").notNullable();
+      t.foreign("secretId").references("id").inTable(TableName.Secret).onDelete("CASCADE");
+      t.timestamps(true, true, true);
+    });
+
+    await createOnUpdateTrigger(knex, TableName.SecretReference);
+  }
+}
+
+export async function down(knex: Knex): Promise<void> {
+  await knex.schema.dropTableIfExists(TableName.SecretReference);
+  await dropOnUpdateTrigger(knex, TableName.SecretReference);
+}

backend/src/db/schemas/models.ts

@@ -28,6 +28,7 @@ export enum TableName {
   ProjectUserMembershipRole = "project_user_membership_roles",
   ProjectKeys = "project_keys",
   Secret = "secrets",
+  SecretReference = "secret_references",
   SecretBlindIndex = "secret_blind_indexes",
   SecretVersion = "secret_versions",
   SecretFolder = "secret_folders",

backend/src/db/schemas/secret-references.ts

@@ -0,0 +1,21 @@
+// Code generated by automation script, DO NOT EDIT.
+// Automated by pulling database and generating zod schema
+// To update. Just run npm run generate:schema
+// Written by akhilmhdh.
+
+import { z } from "zod";
+
+import { TImmutableDBKeys } from "./models";
+
+export const SecretReferencesSchema = z.object({
+  id: z.string().uuid(),
+  environment: z.string(),
+  secretPath: z.string(),
+  secretId: z.string().uuid(),
+  createdAt: z.date(),
+  updatedAt: z.date()
+});
+
+export type TSecretReferences = z.infer<typeof SecretReferencesSchema>;
+export type TSecretReferencesInsert = Omit<z.input<typeof SecretReferencesSchema>, TImmutableDBKeys>;
+export type TSecretReferencesUpdate = Partial<Omit<z.input<typeof SecretReferencesSchema>, TImmutableDBKeys>>;

backend/src/ee/services/secret-approval-request/secret-approval-request-service.ts

@@ -7,12 +7,15 @@ import {
   SecretType,
   TSecretApprovalRequestsSecretsInsert
 } from "@app/db/schemas";
+import { decryptSymmetric128BitHexKeyUTF8 } from "@app/lib/crypto";
 import { BadRequestError, UnauthorizedError } from "@app/lib/errors";
 import { groupBy, pick, unique } from "@app/lib/fn";
 import { alphaNumericNanoId } from "@app/lib/nanoid";
 import { ActorType } from "@app/services/auth/auth-type";
 import { TProjectDALFactory } from "@app/services/project/project-dal";
+import { TProjectBotServiceFactory } from "@app/services/project-bot/project-bot-service";
 import { TSecretDALFactory } from "@app/services/secret/secret-dal";
+import { getAllNestedSecretReferences } from "@app/services/secret/secret-fns";
 import { TSecretQueueFactory } from "@app/services/secret/secret-queue";
 import { TSecretServiceFactory } from "@app/services/secret/secret-service";
 import { TSecretVersionDALFactory } from "@app/services/secret/secret-version-dal";
@@ -53,6 +56,7 @@ type TSecretApprovalRequestServiceFactoryDep = {
   secretVersionDAL: Pick<TSecretVersionDALFactory, "findLatestVersionMany" | "insertMany">;
   secretVersionTagDAL: Pick<TSecretVersionTagDALFactory, "insertMany">;
   projectDAL: Pick<TProjectDALFactory, "checkProjectUpgradeStatus">;
+  projectBotService: Pick<TProjectBotServiceFactory, "getBotKey">;
   secretService: Pick<
     TSecretServiceFactory,
     | "fnSecretBulkInsert"
@@ -80,7 +84,8 @@ export const secretApprovalRequestServiceFactory = ({
   snapshotService,
   secretService,
   secretVersionDAL,
-  secretQueueService
+  secretQueueService,
+  projectBotService
 }: TSecretApprovalRequestServiceFactoryDep) => {
   const requestCount = async ({ projectId, actor, actorId, actorOrgId, actorAuthMethod }: TApprovalRequestCountDTO) => {
     if (actor === ActorType.SERVICE) throw new BadRequestError({ message: "Cannot use service token" });
@@ -352,7 +357,7 @@ export const secretApprovalRequestServiceFactory = ({
     }
 
     const secretDeletionCommits = secretApprovalSecrets.filter(({ op }) => op === CommitType.Delete);
-
+    const botKey = await projectBotService.getBotKey(projectId).catch(() => null);
     const mergeStatus = await secretApprovalRequestDAL.transaction(async (tx) => {
       const newSecrets = secretCreationCommits.length
         ? await secretService.fnSecretBulkInsert({
@@ -379,7 +384,17 @@ export const secretApprovalRequestServiceFactory = ({
               ]),
               tags: el?.tags.map(({ id }) => id),
               version: 1,
-              type: SecretType.Shared
+              type: SecretType.Shared,
+              references: botKey
+                ? getAllNestedSecretReferences(
+                    decryptSymmetric128BitHexKeyUTF8({
+                      ciphertext: el.secretValueCiphertext,
+                      iv: el.secretValueIV,
+                      tag: el.secretValueTag,
+                      key: botKey
+                    })
+                  )
+                : undefined
             })),
             secretDAL,
             secretVersionDAL,
@@ -414,7 +429,17 @@ export const secretApprovalRequestServiceFactory = ({
                   "secretReminderNote",
                   "secretReminderRepeatDays",
                   "secretBlindIndex"
-                ])
+                ]),
+                references: botKey
+                  ? getAllNestedSecretReferences(
+                      decryptSymmetric128BitHexKeyUTF8({
+                        ciphertext: el.secretValueCiphertext,
+                        iv: el.secretValueIV,
+                        tag: el.secretValueTag,
+                        key: botKey
+                      })
+                    )
+                  : undefined
               }
             })),
             secretDAL,

backend/src/queue/queue-service.ts

@@ -65,7 +65,13 @@ export type TQueueJobTypes = {
   };
   [QueueName.IntegrationSync]: {
     name: QueueJobs.IntegrationSync;
-    payload: { projectId: string; environment: string; secretPath: string; depth?: number };
+    payload: {
+      projectId: string;
+      environment: string;
+      secretPath: string;
+      depth?: number;
+      deDupeQueue?: Record<string, boolean>;
+    };
   };
   [QueueName.SecretFullRepoScan]: {
     name: QueueJobs.SecretScan;

backend/src/server/routes/index.ts

@@ -607,6 +607,7 @@ export const registerRoutes = async (
   });
   const sarService = secretApprovalRequestServiceFactory({
     permissionService,
+    projectBotService,
     folderDAL,
     secretDAL,
     secretTagDAL,

backend/src/server/routes/v3/secret-router.ts

@@ -1926,4 +1926,41 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => {
       return { secrets };
     }
   });
+
+  server.route({
+    method: "POST",
+    url: "/backfill-secret-references",
+    config: {
+      rateLimit: secretsLimit
+    },
+    schema: {
+      description: "Backfill secret references",
+      security: [
+        {
+          bearerAuth: []
+        }
+      ],
+      body: z.object({
+        projectId: z.string().trim().min(1)
+      }),
+      response: {
+        200: z.object({
+          message: z.string()
+        })
+      }
+    },
+    onRequest: verifyAuth([AuthMode.JWT]),
+    handler: async (req) => {
+      const { projectId } = req.body;
+      const message = await server.services.secret.backfillSecretReferences({
+        actorId: req.permission.id,
+        actor: req.permission.type,
+        actorAuthMethod: req.permission.authMethod,
+        actorOrgId: req.permission.orgId,
+        projectId
+      });
+
+      return message;
+    }
+  });
 };

backend/src/services/secret/secret-dal.ts

@@ -243,6 +243,74 @@ export const secretDALFactory = (db: TDbClient) => {
     }
   };
 
+  const upsertSecretReferences = async (
+    data: {
+      secretId: string;
+      references: Array<{ environment: string; secretPath: string }>;
+    }[] = [],
+    tx?: Knex
+  ) => {
+    try {
+      if (!data.length) return;
+
+      await (tx || db)(TableName.SecretReference)
+        .whereIn(
+          "secretId",
+          data.map(({ secretId }) => secretId)
+        )
+        .delete();
+      const newSecretReferences = data
+        .filter(({ references }) => references.length)
+        .flatMap(({ secretId, references }) =>
+          references.map(({ environment, secretPath }) => ({
+            secretPath,
+            secretId,
+            environment
+          }))
+        );
+      if (!newSecretReferences.length) return;
+      const secretReferences = await (tx || db)(TableName.SecretReference).insert(newSecretReferences);
+      return secretReferences;
+    } catch (error) {
+      throw new DatabaseError({ error, name: "UpsertSecretReference" });
+    }
+  };
+
+  const findReferencedSecretReferences = async (projectId: string, envSlug: string, secretPath: string, tx?: Knex) => {
+    try {
+      const docs = await (tx || db)(TableName.SecretReference)
+        .where({
+          secretPath,
+          environment: envSlug
+        })
+        .join(TableName.Secret, `${TableName.Secret}.id`, `${TableName.SecretReference}.secretId`)
+        .join(TableName.SecretFolder, `${TableName.Secret}.folderId`, `${TableName.SecretFolder}.id`)
+        .join(TableName.Environment, `${TableName.SecretFolder}.envId`, `${TableName.Environment}.id`)
+        .where("projectId", projectId)
+        .select(selectAllTableCols(TableName.SecretReference))
+        .select("folderId");
+      return docs;
+    } catch (error) {
+      throw new DatabaseError({ error, name: "FindReferencedSecretReferences" });
+    }
+  };
+
+  // special query to backfill secret value
+  const findAllProjectSecretValues = async (projectId: string, tx?: Knex) => {
+    try {
+      const docs = await (tx || db)(TableName.Secret)
+        .join(TableName.SecretFolder, `${TableName.Secret}.folderId`, `${TableName.SecretFolder}.id`)
+        .join(TableName.Environment, `${TableName.SecretFolder}.envId`, `${TableName.Environment}.id`)
+        .where("projectId", projectId)
+        // not empty
+        .whereNotNull("secretValueCiphertext")
+        .select("secretValueTag", "secretValueCiphertext", "secretValueIV", `${TableName.Secret}.id` as "id");
+      return docs;
+    } catch (error) {
+      throw new DatabaseError({ error, name: "FindAllProjectSecretValues" });
+    }
+  };
+
   return {
     ...secretOrm,
     update,
@@ -252,6 +320,9 @@ export const secretDALFactory = (db: TDbClient) => {
     getSecretTags,
     findByFolderId,
     findByFolderIds,
-    findByBlindIndexes
+    findByBlindIndexes,
+    upsertSecretReferences,
+    findReferencedSecretReferences,
+    findAllProjectSecretValues
   };
 };

backend/src/services/secret/secret-fns.ts

@@ -194,6 +194,7 @@ type TInterpolateSecretArg = {
   folderDAL: Pick<TSecretFolderDALFactory, "findBySecretPath">;
 };
 
+const INTERPOLATION_SYNTAX_REG = /\${([^}]+)}/g;
 export const interpolateSecrets = ({ projectId, secretEncKey, secretDAL, folderDAL }: TInterpolateSecretArg) => {
   const fetchSecretsCrossEnv = () => {
     const fetchCache: Record<string, Record<string, string>> = {};
@@ -235,7 +236,6 @@ export const interpolateSecrets = ({ projectId, secretEncKey, secretDAL, folderD
     };
   };
 
-  const INTERPOLATION_SYNTAX_REG = /\${([^}]+)}/g;
   const recursivelyExpandSecret = async (
     expandedSec: Record<string, string>,
     interpolatedSec: Record<string, string>,
@@ -396,6 +396,37 @@ export const decryptSecretRaw = (
   };
 };
 
+/**
+ * Grabs and processes nested secret references from a string
+ *
+ * This function looks for patterns that match the interpolation syntax in the input string.
+ * It filters out references that include nested paths, splits them into environment and
+ * secret path parts, and then returns an array of objects with the environment and the
+ * joined secret path.
+ *
+ * @param {string} maybeSecretReference - The string that has the potential secret references.
+ * @returns {Array<{ environment: string, secretPath: string }>} - An array of objects
+ * with the environment and joined secret path.
+ *
+ * @example
+ * const value = "Hello ${dev.someFolder.OtherFolder.SECRET_NAME} and ${prod.anotherFolder.SECRET_NAME}";
+ * const result = getAllNestedSecretReferences(value);
+ * // result will be:
+ * // [
+ * //   { environment: 'dev', secretPath: '/someFolder/OtherFolder' },
+ * //   { environment: 'prod', secretPath: '/anotherFolder' }
+ * // ]
+ */
+export const getAllNestedSecretReferences = (maybeSecretReference: string) => {
+  const references = Array.from(maybeSecretReference.matchAll(INTERPOLATION_SYNTAX_REG), (m) => m[1]);
+  return references
+    .filter((el) => el.includes("."))
+    .map((el) => {
+      const [environment, ...secretPathList] = el.split(".");
+      return { environment, secretPath: path.join("/", ...secretPathList.slice(0, -1)) };
+    });
+};
+
 /**
  * Checks and handles secrets using a blind index method.
  * The function generates mappings between secret names and their blind indexes, validates user IDs for personal secrets, and retrieves secrets from the database based on their blind indexes.
@@ -467,7 +498,7 @@ export const fnSecretBulkInsert = async ({
   tx
 }: TFnSecretBulkInsert) => {
   const newSecrets = await secretDAL.insertMany(
-    inputSecrets.map(({ tags, ...el }) => ({ ...el, folderId })),
+    inputSecrets.map(({ tags, references, ...el }) => ({ ...el, folderId })),
     tx
   );
   const newSecretGroupByBlindIndex = groupBy(newSecrets, (item) => item.secretBlindIndex as string);
@@ -478,13 +509,19 @@ export const fnSecretBulkInsert = async ({
     }))
   );
   const secretVersions = await secretVersionDAL.insertMany(
-    inputSecrets.map(({ tags, ...el }) => ({
+    inputSecrets.map(({ tags, references, ...el }) => ({
       ...el,
       folderId,
       secretId: newSecretGroupByBlindIndex[el.secretBlindIndex as string][0].id
     })),
     tx
   );
+  await secretDAL.upsertSecretReferences(
+    inputSecrets.map(({ references = [], secretBlindIndex }) => ({
+      secretId: newSecretGroupByBlindIndex[secretBlindIndex as string][0].id,
+      references
+    }))
+  );
   if (newSecretTags.length) {
     const secTags = await secretTagDAL.saveTagsToSecret(newSecretTags, tx);
     const secVersionsGroupBySecId = groupBy(secretVersions, (i) => i.secretId);
@@ -509,7 +546,7 @@ export const fnSecretBulkUpdate = async ({
   secretVersionTagDAL
 }: TFnSecretBulkUpdate) => {
   const newSecrets = await secretDAL.bulkUpdate(
-    inputSecrets.map(({ filter, data: { tags, ...data } }) => ({
+    inputSecrets.map(({ filter, data: { tags, references, ...data } }) => ({
       filter: { ...filter, folderId },
       data
     })),
@@ -522,6 +559,14 @@ export const fnSecretBulkUpdate = async ({
     })),
     tx
   );
+  await secretDAL.upsertSecretReferences(
+    inputSecrets
+      .filter(({ data: { references } }) => Boolean(references))
+      .map(({ data: { references = [] } }, i) => ({
+        secretId: newSecrets[i].id,
+        references
+      }))
+  );
   const secsUpdatedTag = inputSecrets.flatMap(({ data: { tags } }, i) =>
     tags !== undefined ? { tags, secretId: newSecrets[i].id } : []
   );
@@ -591,50 +636,39 @@ export const createManySecretsRawFnFactory = ({
       folderId,
       isNew: true,
       blindIndexCfg,
+      userId,
       secretDAL
     });
 
-    const inputSecrets = await Promise.all(
+    const inputSecrets = secrets.map((secret) => {
-      secrets.map(async (secret) => {
+      const secretKeyEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretName, botKey);
-        const secretKeyEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretName, botKey);
+      const secretValueEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretValue || "", botKey);
-        const secretValueEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretValue || "", botKey);
+      const secretReferences = getAllNestedSecretReferences(secret.secretValue || "");
-        const secretCommentEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretComment || "", botKey);
+      const secretCommentEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretComment || "", botKey);
 
-        if (secret.type === SecretType.Personal) {
+      return {
-          if (!userId) throw new BadRequestError({ message: "Missing user id for personal secret" });
+        type: secret.type,
-          const sharedExist = await secretDAL.findOne({
+        userId: secret.type === SecretType.Personal ? userId : null,
-            secretBlindIndex: keyName2BlindIndex[secret.secretName],
+        secretName: secret.secretName,
-            folderId,
+        secretKeyCiphertext: secretKeyEncrypted.ciphertext,
-            type: SecretType.Shared
+        secretKeyIV: secretKeyEncrypted.iv,
-          });
+        secretKeyTag: secretKeyEncrypted.tag,
+        secretValueCiphertext: secretValueEncrypted.ciphertext,
+        secretValueIV: secretValueEncrypted.iv,
+        secretValueTag: secretValueEncrypted.tag,
+        secretCommentCiphertext: secretCommentEncrypted.ciphertext,
+        secretCommentIV: secretCommentEncrypted.iv,
+        secretCommentTag: secretCommentEncrypted.tag,
+        skipMultilineEncoding: secret.skipMultilineEncoding,
+        tags: secret.tags,
+        references: secretReferences
+      };
+    });
 
-          if (!sharedExist)
+    // get all tags
-            throw new BadRequestError({
+    const tagIds = inputSecrets.flatMap(({ tags = [] }) => tags);
-              message: "Failed to create personal secret override for no corresponding shared secret"
+    const tags = tagIds.length ? await secretTagDAL.findManyTagsById(projectId, tagIds) : [];
-            });
+    if (tags.length !== tagIds.length) throw new BadRequestError({ message: "Tag not found" });
-        }
-
-        const tags = secret.tags ? await secretTagDAL.findManyTagsById(projectId, secret.tags) : [];
-        if ((secret.tags || []).length !== tags.length) throw new BadRequestError({ message: "Tag not found" });
-
-        return {
-          type: secret.type,
-          userId: secret.type === SecretType.Personal ? userId : null,
-          secretName: secret.secretName,
-          secretKeyCiphertext: secretKeyEncrypted.ciphertext,
-          secretKeyIV: secretKeyEncrypted.iv,
-          secretKeyTag: secretKeyEncrypted.tag,
-          secretValueCiphertext: secretValueEncrypted.ciphertext,
-          secretValueIV: secretValueEncrypted.iv,
-          secretValueTag: secretValueEncrypted.tag,
-          secretCommentCiphertext: secretCommentEncrypted.ciphertext,
-          secretCommentIV: secretCommentEncrypted.iv,
-          secretCommentTag: secretCommentEncrypted.tag,
-          skipMultilineEncoding: secret.skipMultilineEncoding,
-          tags: secret.tags
-        };
-      })
-    );
 
     const newSecrets = await secretDAL.transaction(async (tx) =>
       fnSecretBulkInsert({
@@ -703,56 +737,35 @@ export const updateManySecretsRawFnFactory = ({
       userId
     });
 
-    const inputSecrets = await Promise.all(
+    const inputSecrets = secrets.map((secret) => {
-      secrets.map(async (secret) => {
+      if (secret.newSecretName === "") {
-        if (secret.newSecretName === "") {
+        throw new BadRequestError({ message: "New secret name cannot be empty" });
-          throw new BadRequestError({ message: "New secret name cannot be empty" });
+      }
-        }
 
-        const secretKeyEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretName, botKey);
+      const secretKeyEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretName, botKey);
-        const secretValueEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretValue || "", botKey);
+      const secretValueEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretValue || "", botKey);
-        const secretCommentEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretComment || "", botKey);
+      const secretReferences = getAllNestedSecretReferences(secret.secretValue || "");
+      const secretCommentEncrypted = encryptSymmetric128BitHexKeyUTF8(secret.secretComment || "", botKey);
 
-        if (secret.type === SecretType.Personal) {
+      return {
-          if (!userId) throw new BadRequestError({ message: "Missing user id for personal secret" });
+        type: secret.type,
-
+        userId: secret.type === SecretType.Personal ? userId : null,
-          const sharedExist = await secretDAL.findOne({
+        secretName: secret.secretName,
-            secretBlindIndex: keyName2BlindIndex[secret.secretName],
+        newSecretName: secret.newSecretName,
-            folderId,
+        secretKeyCiphertext: secretKeyEncrypted.ciphertext,
-            type: SecretType.Shared
+        secretKeyIV: secretKeyEncrypted.iv,
-          });
+        secretKeyTag: secretKeyEncrypted.tag,
-
+        secretValueCiphertext: secretValueEncrypted.ciphertext,
-          if (!sharedExist)
+        secretValueIV: secretValueEncrypted.iv,
-            throw new BadRequestError({
+        secretValueTag: secretValueEncrypted.tag,
-              message: "Failed to update personal secret override for no corresponding shared secret"
+        secretCommentCiphertext: secretCommentEncrypted.ciphertext,
-            });
+        secretCommentIV: secretCommentEncrypted.iv,
-
+        secretCommentTag: secretCommentEncrypted.tag,
-          if (secret.newSecretName)
+        skipMultilineEncoding: secret.skipMultilineEncoding,
-            throw new BadRequestError({ message: "Personal secret cannot change the key name" });
+        tags: secret.tags,
-        }
+        references: secretReferences
-
+      };
-        const tags = secret.tags ? await secretTagDAL.findManyTagsById(projectId, secret.tags) : [];
+    });
-        if ((secret.tags || []).length !== tags.length) throw new BadRequestError({ message: "Tag not found" });
-
-        return {
-          type: secret.type,
-          userId: secret.type === SecretType.Personal ? userId : null,
-          secretName: secret.secretName,
-          newSecretName: secret.newSecretName,
-          secretKeyCiphertext: secretKeyEncrypted.ciphertext,
-          secretKeyIV: secretKeyEncrypted.iv,
-          secretKeyTag: secretKeyEncrypted.tag,
-          secretValueCiphertext: secretValueEncrypted.ciphertext,
-          secretValueIV: secretValueEncrypted.iv,
-          secretValueTag: secretValueEncrypted.tag,
-          secretCommentCiphertext: secretCommentEncrypted.ciphertext,
-          secretCommentIV: secretCommentEncrypted.iv,
-          secretCommentTag: secretCommentEncrypted.tag,
-          skipMultilineEncoding: secret.skipMultilineEncoding,
-          tags: secret.tags
-        };
-      })
-    );
 
     const tagIds = inputSecrets.flatMap(({ tags = [] }) => tags);
     const tags = tagIds.length ? await secretTagDAL.findManyTagsById(projectId, tagIds) : [];

backend/src/services/secret/secret-queue.ts

@@ -59,6 +59,7 @@ export type TGetSecrets = {
 };
 
 const MAX_SYNC_SECRET_DEPTH = 5;
+const uniqueIntegrationKey = (environment: string, secretPath: string) => `integration-${environment}-${secretPath}`;
 
 export const secretQueueFactory = ({
   queueService,
@@ -102,28 +103,35 @@ export const secretQueueFactory = ({
     folderDAL
   });
 
-  const syncIntegrations = async (dto: TGetSecrets) => {
+  const syncIntegrations = async (dto: TGetSecrets & { deDupeQueue?: Record<string, boolean> }) => {
     await queueService.queue(QueueName.IntegrationSync, QueueJobs.IntegrationSync, dto, {
-      attempts: 5,
+      attempts: 3,
       delay: 1000,
       backoff: {
         type: "exponential",
         delay: 3000
       },
       removeOnComplete: true,
-      removeOnFail: {
+      removeOnFail: true
-        count: 5 // keep the most recent  jobs
-      }
     });
   };
 
-  const syncSecrets = async (dto: TGetSecrets & { depth?: number }) => {
+  const syncSecrets = async ({
+    deDupeQueue = {},
+    ...dto
+  }: TGetSecrets & { depth?: number; deDupeQueue?: Record<string, boolean> }) => {
+    const deDuplicationKey = uniqueIntegrationKey(dto.environment, dto.secretPath);
+    if (deDupeQueue?.[deDuplicationKey]) {
+      return;
+    }
+    // eslint-disable-next-line
+    deDupeQueue[deDuplicationKey] = true;
     logger.info(
       `syncSecrets: syncing project secrets where [projectId=${dto.projectId}]  [environment=${dto.environment}] [path=${dto.secretPath}]`
     );
     await queueService.queue(QueueName.SecretWebhook, QueueJobs.SecWebhook, dto, {
       jobId: `secret-webhook-${dto.environment}-${dto.projectId}-${dto.secretPath}`,
-      removeOnFail: { count: 5 },
+      removeOnFail: true,
       removeOnComplete: true,
       delay: 1000,
       attempts: 5,
@@ -132,7 +140,7 @@ export const secretQueueFactory = ({
         delay: 3000
       }
     });
-    await syncIntegrations(dto);
+    await syncIntegrations({ ...dto, deDupeQueue });
   };
 
   const removeSecretReminder = async (dto: TRemoveSecretReminderDTO) => {
@@ -326,7 +334,7 @@ export const secretQueueFactory = ({
   };
 
   queueService.start(QueueName.IntegrationSync, async (job) => {
-    const { environment, projectId, secretPath, depth = 1 } = job.data;
+    const { environment, projectId, secretPath, depth = 1, deDupeQueue = {} } = job.data;
 
     const folder = await folderDAL.findBySecretPath(projectId, environment, secretPath);
     if (!folder) {
@@ -349,21 +357,68 @@ export const secretQueueFactory = ({
         const importedFolderIds = unique(imports, (i) => i.folderId).map(({ folderId }) => folderId);
         const importedFolders = await folderDAL.findSecretPathByFolderIds(projectId, importedFolderIds);
         const foldersGroupedById = groupBy(importedFolders, (i) => i.child || i.id);
+        logger.info(
+          `getIntegrationSecrets: Syncing secret due to link change [jobId=${job.id}] [projectId=${job.data.projectId}] [environment=${job.data.environment}]  [secretPath=${job.data.secretPath}] [depth=${depth}]`
+        );
         await Promise.all(
           imports
             .filter(({ folderId }) => Boolean(foldersGroupedById[folderId][0].path))
-            .map(({ folderId }) => {
+            // filter out already synced ones
-              const syncDto = {
+            .filter(
+              ({ folderId }) =>
+                !deDupeQueue[
+                  uniqueIntegrationKey(
+                    foldersGroupedById[folderId][0].environmentSlug,
+                    foldersGroupedById[folderId][0].path
+                  )
+                ]
+            )
+            .map(({ folderId }) =>
+              syncSecrets({
                 depth: depth + 1,
                 projectId,
                 secretPath: foldersGroupedById[folderId][0].path,
-                environment: foldersGroupedById[folderId][0].environmentSlug
+                environment: foldersGroupedById[folderId][0].environmentSlug,
-              };
+                deDupeQueue
-              logger.info(
+              })
-                `getIntegrationSecrets: Syncing secret due to link change [jobId=${job.id}] [projectId=${job.data.projectId}] [environment=${job.data.environment}]  [secretPath=${job.data.secretPath}] [depth=${depth}]`
+            )
-              );
+        );
-              return syncSecrets(syncDto);
+      }
-            })
+
+      const secretReferences = await secretDAL.findReferencedSecretReferences(
+        projectId,
+        folder.environment.slug,
+        secretPath
+      );
+      if (secretReferences.length) {
+        const referencedFolderIds = unique(secretReferences, (i) => i.folderId).map(({ folderId }) => folderId);
+        const referencedFolders = await folderDAL.findSecretPathByFolderIds(projectId, referencedFolderIds);
+        const referencedFoldersGroupedById = groupBy(referencedFolders, (i) => i.child || i.id);
+        logger.info(
+          `getIntegrationSecrets: Syncing secret due to reference change [jobId=${job.id}] [projectId=${job.data.projectId}] [environment=${job.data.environment}]  [secretPath=${job.data.secretPath}] [depth=${depth}]`
+        );
+        await Promise.all(
+          secretReferences
+            .filter(({ folderId }) => Boolean(referencedFoldersGroupedById[folderId][0].path))
+            // filter out already synced ones
+            .filter(
+              ({ folderId }) =>
+                !deDupeQueue[
+                  uniqueIntegrationKey(
+                    referencedFoldersGroupedById[folderId][0].environmentSlug,
+                    referencedFoldersGroupedById[folderId][0].path
+                  )
+                ]
+            )
+            .map(({ folderId }) =>
+              syncSecrets({
+                depth: depth + 1,
+                projectId,
+                secretPath: referencedFoldersGroupedById[folderId][0].path,
+                environment: referencedFoldersGroupedById[folderId][0].environmentSlug,
+                deDupeQueue
+              })
+            )
         );
       }
     } else {

backend/src/services/secret/secret-service.ts

@@ -2,12 +2,22 @@
 /* eslint-disable no-await-in-loop */
 import { ForbiddenError, subject } from "@casl/ability";
 
-import { SecretEncryptionAlgo, SecretKeyEncoding, SecretsSchema, SecretType } from "@app/db/schemas";
+import {
+  ProjectMembershipRole,
+  SecretEncryptionAlgo,
+  SecretKeyEncoding,
+  SecretsSchema,
+  SecretType
+} from "@app/db/schemas";
 import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service";
 import { ProjectPermissionActions, ProjectPermissionSub } from "@app/ee/services/permission/project-permission";
 import { TSecretSnapshotServiceFactory } from "@app/ee/services/secret-snapshot/secret-snapshot-service";
 import { getConfig } from "@app/lib/config/env";
-import { buildSecretBlindIndexFromName, encryptSymmetric128BitHexKeyUTF8 } from "@app/lib/crypto";
+import {
+  buildSecretBlindIndexFromName,
+  decryptSymmetric128BitHexKeyUTF8,
+  encryptSymmetric128BitHexKeyUTF8
+} from "@app/lib/crypto";
 import { BadRequestError } from "@app/lib/errors";
 import { groupBy, pick } from "@app/lib/fn";
 import { logger } from "@app/lib/logger";
@@ -27,12 +37,14 @@ import {
   fnSecretBlindIndexCheck,
   fnSecretBulkInsert,
   fnSecretBulkUpdate,
+  getAllNestedSecretReferences,
   interpolateSecrets,
   recursivelyGetSecretPaths
 } from "./secret-fns";
 import { TSecretQueueFactory } from "./secret-queue";
 import {
   TAttachSecretTagsDTO,
+  TBackFillSecretReferencesDTO,
   TCreateBulkSecretDTO,
   TCreateManySecretRawDTO,
   TCreateSecretDTO,
@@ -91,6 +103,22 @@ export const secretServiceFactory = ({
   secretImportDAL,
   secretVersionTagDAL
 }: TSecretServiceFactoryDep) => {
+  const getSecretReference = async (projectId: string) => {
+    // if bot key missing means e2e still exist
+    const botKey = await projectBotService.getBotKey(projectId).catch(() => null);
+    return (el: { ciphertext?: string; iv: string; tag: string }) =>
+      botKey
+        ? getAllNestedSecretReferences(
+            decryptSymmetric128BitHexKeyUTF8({
+              ciphertext: el.ciphertext || "",
+              iv: el.iv,
+              tag: el.tag,
+              key: botKey
+            })
+          )
+        : undefined;
+  };
+
   // utility function to get secret blind index data
   const interalGenSecBlindIndexByName = async (projectId: string, secretName: string) => {
     const appCfg = getConfig();
@@ -225,6 +253,7 @@ export const secretServiceFactory = ({
     if ((inputSecret.tags || []).length !== tags.length) throw new BadRequestError({ message: "Tag not found" });
 
     const { secretName, type, ...el } = inputSecret;
+    const references = await getSecretReference(projectId);
     const secret = await secretDAL.transaction((tx) =>
       fnSecretBulkInsert({
         folderId,
@@ -237,7 +266,12 @@ export const secretServiceFactory = ({
             userId: inputSecret.type === SecretType.Personal ? actorId : null,
             algorithm: SecretEncryptionAlgo.AES_256_GCM,
             keyEncoding: SecretKeyEncoding.UTF8,
-            tags: inputSecret.tags
+            tags: inputSecret.tags,
+            references: references({
+              ciphertext: inputSecret.secretValueCiphertext,
+              iv: inputSecret.secretValueIV,
+              tag: inputSecret.secretValueTag
+            })
           }
         ],
         secretDAL,
@@ -335,6 +369,7 @@ export const secretServiceFactory = ({
 
     const { secretName, ...el } = inputSecret;
 
+    const references = await getSecretReference(projectId);
     const updatedSecret = await secretDAL.transaction(async (tx) =>
       fnSecretBulkUpdate({
         folderId,
@@ -360,7 +395,12 @@ export const secretServiceFactory = ({
                 "secretReminderRepeatDays",
                 "tags"
               ]),
-              secretBlindIndex: newSecretNameBlindIndex || keyName2BlindIndex[secretName]
+              secretBlindIndex: newSecretNameBlindIndex || keyName2BlindIndex[secretName],
+              references: references({
+                ciphertext: inputSecret.secretValueCiphertext,
+                iv: inputSecret.secretValueIV,
+                tag: inputSecret.secretValueTag
+              })
             }
           }
         ],
@@ -700,6 +740,7 @@ export const secretServiceFactory = ({
     const tags = tagIds.length ? await secretTagDAL.findManyTagsById(projectId, tagIds) : [];
     if (tags.length !== tagIds.length) throw new BadRequestError({ message: "Tag not found" });
 
+    const references = await getSecretReference(projectId);
     const newSecrets = await secretDAL.transaction(async (tx) =>
       fnSecretBulkInsert({
         inputSecrets: inputSecrets.map(({ secretName, ...el }) => ({
@@ -708,7 +749,12 @@ export const secretServiceFactory = ({
           secretBlindIndex: keyName2BlindIndex[secretName],
           type: SecretType.Shared,
           algorithm: SecretEncryptionAlgo.AES_256_GCM,
-          keyEncoding: SecretKeyEncoding.UTF8
+          keyEncoding: SecretKeyEncoding.UTF8,
+          references: references({
+            ciphertext: el.secretValueCiphertext,
+            iv: el.secretValueIV,
+            tag: el.secretValueTag
+          })
         })),
         folderId,
         secretDAL,
@@ -783,6 +829,8 @@ export const secretServiceFactory = ({
     const tagIds = inputSecrets.flatMap(({ tags = [] }) => tags);
     const tags = tagIds.length ? await secretTagDAL.findManyTagsById(projectId, tagIds) : [];
     if (tagIds.length !== tags.length) throw new BadRequestError({ message: "Tag not found" });
+
+    const references = await getSecretReference(projectId);
     const secrets = await secretDAL.transaction(async (tx) =>
       fnSecretBulkUpdate({
         folderId,
@@ -799,7 +847,15 @@ export const secretServiceFactory = ({
                 ? newKeyName2BlindIndex[newSecretName]
                 : keyName2BlindIndex[secretName],
             algorithm: SecretEncryptionAlgo.AES_256_GCM,
-            keyEncoding: SecretKeyEncoding.UTF8
+            keyEncoding: SecretKeyEncoding.UTF8,
+            references:
+              el.secretValueIV && el.secretValueTag
+                ? references({
+                    ciphertext: el.secretValueCiphertext,
+                    iv: el.secretValueIV,
+                    tag: el.secretValueTag
+                  })
+                : undefined
           }
         })),
         secretDAL,
@@ -1488,6 +1544,51 @@ export const secretServiceFactory = ({
     };
   };
 
+  // this is a backfilling API for secret references
+  // what it does is it will go through all the secret values and parse all references
+  // populate the secret reference to do sync integrations
+  const backfillSecretReferences = async ({
+    projectId,
+    actor,
+    actorId,
+    actorOrgId,
+    actorAuthMethod
+  }: TBackFillSecretReferencesDTO) => {
+    const { hasRole } = await permissionService.getProjectPermission(
+      actor,
+      actorId,
+      projectId,
+      actorAuthMethod,
+      actorOrgId
+    );
+
+    if (!hasRole(ProjectMembershipRole.Admin))
+      throw new BadRequestError({ message: "Only admins are allowed to take this action" });
+
+    const botKey = await projectBotService.getBotKey(projectId);
+    if (!botKey)
+      throw new BadRequestError({ message: "Please upgrade your project first", name: "bot_not_found_error" });
+
+    await secretDAL.transaction(async (tx) => {
+      const secrets = await secretDAL.findAllProjectSecretValues(projectId, tx);
+      await secretDAL.upsertSecretReferences(
+        secrets.map(({ id, secretValueCiphertext, secretValueIV, secretValueTag }) => ({
+          secretId: id,
+          references: getAllNestedSecretReferences(
+            decryptSymmetric128BitHexKeyUTF8({
+              ciphertext: secretValueCiphertext,
+              iv: secretValueIV,
+              tag: secretValueTag,
+              key: botKey
+            })
+          )
+        }))
+      );
+    });
+
+    return { message: "Successfully backfilled secret references" };
+  };
+
   return {
     attachTags,
     detachTags,
@@ -1508,6 +1609,7 @@ export const secretServiceFactory = ({
     updateManySecretsRaw,
     deleteManySecretsRaw,
     getSecretVersions,
+    backfillSecretReferences,
     // external services function
     fnSecretBulkDelete,
     fnSecretBulkUpdate,

backend/src/services/secret/secret-types.ts

@@ -223,11 +223,13 @@ export type TGetSecretVersionsDTO = Omit<TProjectPermission, "projectId"> & {
   secretId: string;
 };
 
+export type TSecretReference = { environment: string; secretPath: string };
+
 export type TFnSecretBulkInsert = {
   folderId: string;
   tx?: Knex;
-  inputSecrets: Array<Omit<TSecretsInsert, "folderId"> & { tags?: string[] }>;
+  inputSecrets: Array<Omit<TSecretsInsert, "folderId"> & { tags?: string[]; references?: TSecretReference[] }>;
-  secretDAL: Pick<TSecretDALFactory, "insertMany">;
+  secretDAL: Pick<TSecretDALFactory, "insertMany" | "upsertSecretReferences">;
   secretVersionDAL: Pick<TSecretVersionDALFactory, "insertMany">;
   secretTagDAL: Pick<TSecretTagDALFactory, "saveTagsToSecret">;
   secretVersionTagDAL: Pick<TSecretVersionTagDALFactory, "insertMany">;
@@ -236,8 +238,11 @@ export type TFnSecretBulkInsert = {
 export type TFnSecretBulkUpdate = {
   folderId: string;
   projectId: string;
-  inputSecrets: { filter: Partial<TSecrets>; data: TSecretsUpdate & { tags?: string[] } }[];
+  inputSecrets: {
-  secretDAL: Pick<TSecretDALFactory, "bulkUpdate">;
+    filter: Partial<TSecrets>;
+    data: TSecretsUpdate & { tags?: string[]; references?: TSecretReference[] };
+  }[];
+  secretDAL: Pick<TSecretDALFactory, "bulkUpdate" | "upsertSecretReferences">;
   secretVersionDAL: Pick<TSecretVersionDALFactory, "insertMany">;
   secretTagDAL: Pick<TSecretTagDALFactory, "saveTagsToSecret" | "deleteTagsManySecret">;
   secretVersionTagDAL: Pick<TSecretVersionTagDALFactory, "insertMany">;
@@ -294,6 +299,8 @@ export type TRemoveSecretReminderDTO = {
   repeatDays: number;
 };
 
+export type TBackFillSecretReferencesDTO = TProjectPermission;
+
 // ---
 
 export type TCreateManySecretsRawFnFactory = {

frontend/src/hooks/api/secrets/index.ts

@@ -1,4 +1,5 @@
 export {
+  useBackfillSecretReference,
   useCreateSecretBatch,
   useCreateSecretV3,
   useDeleteSecretBatch,

frontend/src/hooks/api/secrets/mutations.tsx

@@ -87,11 +87,11 @@ export const useCreateSecretV3 = ({
 
       const randomBytes = latestFileKey
         ? decryptAssymmetric({
-            ciphertext: latestFileKey.encryptedKey,
+          ciphertext: latestFileKey.encryptedKey,
-            nonce: latestFileKey.nonce,
+          nonce: latestFileKey.nonce,
-            publicKey: latestFileKey.sender.publicKey,
+          publicKey: latestFileKey.sender.publicKey,
-            privateKey: PRIVATE_KEY
+          privateKey: PRIVATE_KEY
-          })
+        })
         : crypto.randomBytes(16).toString("hex");
 
       const reqBody = {
@@ -148,11 +148,11 @@ export const useUpdateSecretV3 = ({
 
       const randomBytes = latestFileKey
         ? decryptAssymmetric({
-            ciphertext: latestFileKey.encryptedKey,
+          ciphertext: latestFileKey.encryptedKey,
-            nonce: latestFileKey.nonce,
+          nonce: latestFileKey.nonce,
-            publicKey: latestFileKey.sender.publicKey,
+          publicKey: latestFileKey.sender.publicKey,
-            privateKey: PRIVATE_KEY
+          privateKey: PRIVATE_KEY
-          })
+        })
         : crypto.randomBytes(16).toString("hex");
 
       const reqBody = {
@@ -244,11 +244,11 @@ export const useCreateSecretBatch = ({
       const PRIVATE_KEY = localStorage.getItem("PRIVATE_KEY") as string;
       const randomBytes = latestFileKey
         ? decryptAssymmetric({
-            ciphertext: latestFileKey.encryptedKey,
+          ciphertext: latestFileKey.encryptedKey,
-            nonce: latestFileKey.nonce,
+          nonce: latestFileKey.nonce,
-            publicKey: latestFileKey.sender.publicKey,
+          publicKey: latestFileKey.sender.publicKey,
-            privateKey: PRIVATE_KEY
+          privateKey: PRIVATE_KEY
-          })
+        })
         : crypto.randomBytes(16).toString("hex");
 
       const reqBody = {
@@ -297,11 +297,11 @@ export const useUpdateSecretBatch = ({
       const PRIVATE_KEY = localStorage.getItem("PRIVATE_KEY") as string;
       const randomBytes = latestFileKey
         ? decryptAssymmetric({
-            ciphertext: latestFileKey.encryptedKey,
+          ciphertext: latestFileKey.encryptedKey,
-            nonce: latestFileKey.nonce,
+          nonce: latestFileKey.nonce,
-            publicKey: latestFileKey.sender.publicKey,
+          publicKey: latestFileKey.sender.publicKey,
-            privateKey: PRIVATE_KEY
+          privateKey: PRIVATE_KEY
-          })
+        })
         : crypto.randomBytes(16).toString("hex");
 
       const reqBody = {
@@ -379,3 +379,13 @@ export const createSecret = async (dto: CreateSecretDTO) => {
   const { data } = await apiRequest.post(`/api/v3/secrets/${dto.secretKey}`, dto);
   return data;
 };
+
+export const useBackfillSecretReference = () =>
+  useMutation<{ message: string }, {}, { projectId: string }>({
+    mutationFn: async ({ projectId }) => {
+      const { data } = await apiRequest.post("/api/v3/secrets/backfill-secret-references", {
+        projectId
+      });
+      return data.message;
+    }
+  });

frontend/src/views/Settings/ProjectSettingsPage/components/BackfillSecretReferenceSection/BackfillSecretReferenceSection.tsx

@@ -0,0 +1,43 @@
+import { createNotification } from "@app/components/notifications";
+import { Button } from "@app/components/v2";
+import { useProjectPermission, useWorkspace } from "@app/context";
+import { useBackfillSecretReference } from "@app/hooks/api";
+import { ProjectMembershipRole } from "@app/hooks/api/roles/types";
+
+export const BackfillSecretReferenceSecretion = () => {
+	const { currentWorkspace } = useWorkspace();
+	const { membership } = useProjectPermission();
+	const backfillSecretReferences = useBackfillSecretReference();
+
+	if (!currentWorkspace) return null;
+
+	const handleBackfill = async () => {
+		if (backfillSecretReferences.isLoading) return;
+		try {
+			await backfillSecretReferences.mutateAsync({ projectId: currentWorkspace.id || "" });
+			createNotification({ text: "Successfully re-indexed secret references", type: "success" });
+		} catch {
+			createNotification({ text: "Failed to re-index secret references", type: "error" });
+		}
+	};
+
+	const isAdmin = membership.roles.includes(ProjectMembershipRole.Admin);
+	return (
+		<div className="mb-6 rounded-lg border border-mineshaft-600 bg-mineshaft-900 p-4">
+			<div className="flex w-full items-center justify-between">
+				<p className="text-xl font-semibold">Index Secret References</p>
+			</div>
+			<p className="mb-4 mt-2 max-w-2xl text-sm text-gray-400">
+				This will index all secret references, enabling integrations to be triggered when their values change going forward.
+			</p>
+			<Button
+				variant="outline_bg"
+				isLoading={backfillSecretReferences.isLoading}
+				onClick={handleBackfill}
+				isDisabled={!isAdmin}
+			>
+				Index Secret References
+			</Button>
+		</div>
+	);
+};

frontend/src/views/Settings/ProjectSettingsPage/components/BackfillSecretReferenceSection/index.tsx

@@ -0,0 +1 @@
+export { BackfillSecretReferenceSecretion } from "./BackfillSecretReferenceSection";

frontend/src/views/Settings/ProjectSettingsPage/components/ProjectGeneralTab/ProjectGeneralTab.tsx

@@ -1,4 +1,5 @@
 import { AutoCapitalizationSection } from "../AutoCapitalizationSection";
+import { BackfillSecretReferenceSecretion } from "../BackfillSecretReferenceSection";
 import { DeleteProjectSection } from "../DeleteProjectSection";
 import { E2EESection } from "../E2EESection";
 import { EnvironmentSection } from "../EnvironmentSection";
@@ -13,6 +14,7 @@ export const ProjectGeneralTab = () => {
       <SecretTagsSection />
       <AutoCapitalizationSection />
       <E2EESection />
+      <BackfillSecretReferenceSecretion />
       <DeleteProjectSection />
     </div>
   );

frontend/src/views/Settings/ProjectSettingsPage/components/index.tsx

@@ -1,4 +1,5 @@
 export { AutoCapitalizationSection } from "./AutoCapitalizationSection";
+export { BackfillSecretReferenceSecretion } from "./BackfillSecretReferenceSection";
 export { DeleteProjectSection } from "./DeleteProjectSection";
 export { E2EESection } from "./E2EESection";
 export { EnvironmentSection } from "./EnvironmentSection";