Add resync on integration import creation/deletion and update forward/backward recursive logic for syncing dependent imports

backend/src/server/routes/index.ts

@@ -445,14 +445,6 @@ export const registerRoutes = async (
     projectEnvDAL,
     snapshotService
   });
-  const secretImportService = secretImportServiceFactory({
-    projectEnvDAL,
-    folderDAL,
-    permissionService,
-    secretImportDAL,
-    projectDAL,
-    secretDAL
-  });
   const integrationAuthService = integrationAuthServiceFactory({
     integrationAuthDAL,
     integrationDAL,
@@ -480,6 +472,15 @@ export const registerRoutes = async (
     secretTagDAL,
     secretVersionTagDAL
   });
+  const secretImportService = secretImportServiceFactory({
+    projectEnvDAL,
+    folderDAL,
+    permissionService,
+    secretImportDAL,
+    projectDAL,
+    secretDAL,
+    secretQueueService
+  });
   const secretBlindIndexService = secretBlindIndexServiceFactory({
     permissionService,
     secretDAL,

backend/src/services/secret-folder/secret-folder-dal.ts

@@ -259,10 +259,12 @@ export const secretFolderDALFactory = (db: TDbClient) => {
   const findSecretPathByFolderIds = async (projectId: string, folderIds: string[], tx?: Knex) => {
     try {
       const folders = await sqlFindSecretPathByFolderId(tx || db, projectId, folderIds);
+
       const rootFolders = groupBy(
         folders.filter(({ parentId }) => parentId === null),
         (i) => i.child || i.id // root condition then child and parent will null
       );
+
       return folderIds.map((folderId) => rootFolders[folderId]?.[0]);
     } catch (error) {
       throw new DatabaseError({ error, name: "Find by secret path" });

backend/src/services/secret-import/secret-import-service.ts

@@ -7,6 +7,7 @@ import { BadRequestError } from "@app/lib/errors";
 import { TProjectDALFactory } from "../project/project-dal";
 import { TProjectEnvDALFactory } from "../project-env/project-env-dal";
 import { TSecretDALFactory } from "../secret/secret-dal";
+import { TSecretQueueFactory } from "../secret/secret-queue";
 import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal";
 import { TSecretImportDALFactory } from "./secret-import-dal";
 import { fnSecretsFromImports } from "./secret-import-fns";
@@ -25,6 +26,7 @@ type TSecretImportServiceFactoryDep = {
   projectDAL: Pick<TProjectDALFactory, "checkProjectUpgradeStatus">;
   projectEnvDAL: TProjectEnvDALFactory;
   permissionService: Pick<TPermissionServiceFactory, "getProjectPermission">;
+  secretQueueService: Pick<TSecretQueueFactory, "syncSecrets">;
 };
 
 const ERR_SEC_IMP_NOT_FOUND = new BadRequestError({ message: "Secret import not found" });
@@ -37,7 +39,8 @@ export const secretImportServiceFactory = ({
   permissionService,
   folderDAL,
   projectDAL,
-  secretDAL
+  secretDAL,
+  secretQueueService
 }: TSecretImportServiceFactoryDep) => {
   const createImport = async ({
     environment,
@@ -103,6 +106,12 @@ export const secretImportServiceFactory = ({
       );
     });
 
+    await secretQueueService.syncSecrets({
+      secretPath: secImport.importPath,
+      projectId,
+      environment: importEnv.slug
+    });
+
     return { ...secImport, importEnv };
   };
 
@@ -208,6 +217,13 @@ export const secretImportServiceFactory = ({
       if (!importEnv) throw new BadRequestError({ error: "Imported env not found", name: "Create import" });
       return { ...doc, importEnv };
     });
+
+    await secretQueueService.syncSecrets({
+      secretPath: path,
+      projectId,
+      environment
+    });
+
     return secImport;
   };
 

backend/src/services/secret/secret-queue.ts

@@ -23,7 +23,6 @@ import { TProjectEnvDALFactory } from "../project-env/project-env-dal";
 import { TProjectMembershipDALFactory } from "../project-membership/project-membership-dal";
 import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal";
 import { TSecretImportDALFactory } from "../secret-import/secret-import-dal";
-import { fnSecretsFromImports } from "../secret-import/secret-import-fns";
 import { SmtpTemplates, TSmtpService } from "../smtp/smtp-service";
 import { TWebhookDALFactory } from "../webhook/webhook-dal";
 import { fnTriggerWebhook } from "../webhook/webhook-fns";
@@ -231,62 +230,35 @@ export const secretQueueFactory = ({
     }
   };
 
-  const getIntegrationSecrets = async (dto: TGetSecrets & { folderId: string }, key: string) => {
+  type Content = Record<string, { value: string; comment?: string; skipMultilineEncoding?: boolean }>;
+
+  /**
+   * Return the secrets in a given [folderId] including the secrets from nested imported folders
+   */
+  const getIntegrationSecrets = async (dto: {
+    projectId: string;
+    environment: string;
+    folderId: string;
+    key: string;
+    depth: number;
+  }) => {
+    let content: Content = {};
+    if (dto.depth > MAX_SYNC_SECRET_DEPTH) return content;
+
     const secrets = await secretDAL.findByFolderId(dto.folderId);
-
-    // get imported secrets
-    const secretImport = await secretImportDAL.find({ folderId: dto.folderId });
-    const importedSecrets = await fnSecretsFromImports({
-      allowedImports: secretImport,
-      secretDAL,
-      folderDAL
-    });
-
-    if (!secrets.length && !importedSecrets.length) return {};
-
-    const content: Record<string, { value: string; comment?: string; skipMultilineEncoding?: boolean }> = {};
-
-    importedSecrets.forEach(({ secrets: secs }) => {
-      secs.forEach((secret) => {
-        const secretKey = decryptSymmetric128BitHexKeyUTF8({
-          ciphertext: secret.secretKeyCiphertext,
-          iv: secret.secretKeyIV,
-          tag: secret.secretKeyTag,
-          key
-        });
-        const secretValue = decryptSymmetric128BitHexKeyUTF8({
-          ciphertext: secret.secretValueCiphertext,
-          iv: secret.secretValueIV,
-          tag: secret.secretValueTag,
-          key
-        });
-        content[secretKey] = { value: secretValue };
-        content[secretKey].skipMultilineEncoding = Boolean(secret.skipMultilineEncoding);
-
-        if (secret.secretCommentCiphertext && secret.secretCommentIV && secret.secretCommentTag) {
-          const commentValue = decryptSymmetric128BitHexKeyUTF8({
-            ciphertext: secret.secretCommentCiphertext,
-            iv: secret.secretCommentIV,
-            tag: secret.secretCommentTag,
-            key
-          });
-          content[secretKey].comment = commentValue;
-        }
-      });
-    });
     secrets.forEach((secret) => {
       const secretKey = decryptSymmetric128BitHexKeyUTF8({
         ciphertext: secret.secretKeyCiphertext,
         iv: secret.secretKeyIV,
         tag: secret.secretKeyTag,
-        key
+        key: dto.key
       });
 
       const secretValue = decryptSymmetric128BitHexKeyUTF8({
         ciphertext: secret.secretValueCiphertext,
         iv: secret.secretValueIV,
         tag: secret.secretValueTag,
-        key
+        key: dto.key
       });
 
       content[secretKey] = { value: secretValue };
@@ -296,25 +268,52 @@ export const secretQueueFactory = ({
           ciphertext: secret.secretCommentCiphertext,
           iv: secret.secretCommentIV,
           tag: secret.secretCommentTag,
-          key
+          key: dto.key
         });
         content[secretKey].comment = commentValue;
       }
 
       content[secretKey].skipMultilineEncoding = Boolean(secret.skipMultilineEncoding);
     });
+
     const expandSecrets = interpolateSecrets({
       projectId: dto.projectId,
-      secretEncKey: key,
+      secretEncKey: dto.key,
       folderDAL,
       secretDAL
     });
+
     await expandSecrets(content);
+
+    const secretImport = await secretImportDAL.find({ folderId: dto.folderId });
+    if (!secretImport) return content;
+
+    const importedFolders = await folderDAL.findByManySecretPath(
+      secretImport.map(({ importEnv, importPath }) => ({
+        envId: importEnv.id,
+        secretPath: importPath
+      }))
+    );
+
+    for await (const folder of importedFolders) {
+      if (folder) {
+        const importedSecrets = await getIntegrationSecrets({
+          environment: dto.environment,
+          projectId: dto.projectId,
+          folderId: folder.id,
+          key: dto.key,
+          depth: dto.depth + 1
+        });
+        content = { ...content, ...importedSecrets };
+      }
+    }
+
     return content;
   };
 
   queueService.start(QueueName.IntegrationSync, async (job) => {
     const { environment, projectId, secretPath, depth = 1 } = job.data;
+
     const folder = await folderDAL.findBySecretPath(projectId, environment, secretPath);
     if (!folder) {
       logger.error(new Error("Secret path not found"));
@@ -330,11 +329,12 @@ export const secretQueueFactory = ({
         importPath: secretPath
       };
       const imports = await secretImportDAL.find(linkSourceDto);
+
       if (imports.length) {
         // keep calling sync secret for all the imports made
         const importedFolderIds = unique(imports, (i) => i.folderId).map(({ folderId }) => folderId);
         const importedFolders = await folderDAL.findSecretPathByFolderIds(projectId, importedFolderIds);
-        const foldersGroupedById = groupBy(importedFolders, (i) => i.id);
+        const foldersGroupedById = groupBy(importedFolders, (i) => i.child || i.id);
         await Promise.all(
           imports
             .filter(({ folderId }) => Boolean(foldersGroupedById[folderId][0].path))
@@ -352,8 +352,9 @@ export const secretQueueFactory = ({
       }
     }
 
-    const integrations = await integrationDAL.findByProjectIdV2(projectId, environment);
+    const integrations = await integrationDAL.findByProjectIdV2(projectId, environment); // note: returns array of integrations + integration auths in this environment
     const toBeSyncedIntegrations = integrations.filter(
+      // note: sync only the integrations sourced from secretPath
       ({ secretPath: integrationSecPath, isActive }) => isActive && isSamePath(secretPath, integrationSecPath)
     );
 
@@ -369,7 +370,13 @@ export const secretQueueFactory = ({
 
       const botKey = await projectBotService.getBotKey(projectId);
       const { accessToken, accessId } = await integrationAuthService.getIntegrationAccessToken(integrationAuth, botKey);
-      const secrets = await getIntegrationSecrets({ environment, projectId, secretPath, folderId: folder.id }, botKey);
+      const secrets = await getIntegrationSecrets({
+        environment,
+        projectId,
+        folderId: folder.id,
+        key: botKey,
+        depth: 1
+      });
       const suffixedSecrets: typeof secrets = {};
       const metadata = integration.metadata as Record<string, string>;
       if (metadata) {