Merge pull request #2455 from Infisical/daniel/emails-on-sync-failed

feat(integrations): email when integration sync fails

backend/src/queue/queue-service.ts

@@ -7,7 +7,11 @@ import {
   TScanFullRepoEventPayload,
   TScanPushEventPayload
 } from "@app/ee/services/secret-scanning/secret-scanning-queue/secret-scanning-queue-types";
-import { TSyncSecretsDTO } from "@app/services/secret/secret-types";
+import {
+  TFailedIntegrationSyncEmailsPayload,
+  TIntegrationSyncPayload,
+  TSyncSecretsDTO
+} from "@app/services/secret/secret-types";
 
 export enum QueueName {
   SecretRotation = "secret-rotation",
@@ -42,6 +46,7 @@ export enum QueueJobs {
   SecWebhook = "secret-webhook-trigger",
   TelemetryInstanceStats = "telemetry-self-hosted-stats",
   IntegrationSync = "secret-integration-pull",
+  SendFailedIntegrationSyncEmails = "send-failed-integration-sync-emails",
   SecretScan = "secret-scan",
   UpgradeProjectToGhost = "upgrade-project-to-ghost-job",
   DynamicSecretRevocation = "dynamic-secret-revocation",
@@ -88,17 +93,25 @@ export type TQueueJobTypes = {
     name: QueueJobs.SecWebhook;
     payload: { projectId: string; environment: string; secretPath: string; depth?: number };
   };
-  [QueueName.IntegrationSync]: {
+
-    name: QueueJobs.IntegrationSync;
+  [QueueName.AccessTokenStatusUpdate]:
-    payload: {
+    | {
-      isManual?: boolean;
+        name: QueueJobs.IdentityAccessTokenStatusUpdate;
-      actorId?: string;
+        payload: { identityAccessTokenId: string; numberOfUses: number };
-      projectId: string;
+      }
-      environment: string;
+    | {
-      secretPath: string;
+        name: QueueJobs.ServiceTokenStatusUpdate;
-      depth?: number;
+        payload: { serviceTokenId: string };
-      deDupeQueue?: Record<string, boolean>;
       };
+
+  [QueueName.IntegrationSync]:
+    | {
+        name: QueueJobs.IntegrationSync;
+        payload: TIntegrationSyncPayload;
+      }
+    | {
+        name: QueueJobs.SendFailedIntegrationSyncEmails;
+        payload: TFailedIntegrationSyncEmailsPayload;
       };
   [QueueName.SecretFullRepoScan]: {
     name: QueueJobs.SecretScan;
@@ -153,15 +166,6 @@ export type TQueueJobTypes = {
     name: QueueJobs.ProjectV3Migration;
     payload: { projectId: string };
   };
-  [QueueName.AccessTokenStatusUpdate]:
-    | {
-        name: QueueJobs.IdentityAccessTokenStatusUpdate;
-        payload: { identityAccessTokenId: string; numberOfUses: number };
-      }
-    | {
-        name: QueueJobs.ServiceTokenStatusUpdate;
-        payload: { serviceTokenId: string };
-      };
 };
 
 export type TQueueServiceFactory = ReturnType<typeof queueServiceFactory>;

backend/src/services/secret/secret-queue.ts

@@ -1,7 +1,13 @@
 /* eslint-disable no-await-in-loop */
 import { AxiosError } from "axios";
 
-import { ProjectUpgradeStatus, ProjectVersion, TSecretSnapshotSecretsV2, TSecretVersionsV2 } from "@app/db/schemas";
+import {
+  ProjectMembershipRole,
+  ProjectUpgradeStatus,
+  ProjectVersion,
+  TSecretSnapshotSecretsV2,
+  TSecretVersionsV2
+} from "@app/db/schemas";
 import { TAuditLogServiceFactory } from "@app/ee/services/audit-log/audit-log-service";
 import { Actor, EventType } from "@app/ee/services/audit-log/audit-log-types";
 import { TSecretApprovalRequestDALFactory } from "@app/ee/services/secret-approval-request/secret-approval-request-dal";
@@ -50,7 +56,9 @@ import { TSecretDALFactory } from "./secret-dal";
 import { interpolateSecrets } from "./secret-fns";
 import {
   TCreateSecretReminderDTO,
+  TFailedIntegrationSyncEmailsPayload,
   THandleReminderDTO,
+  TIntegrationSyncPayload,
   TRemoveSecretReminderDTO,
   TSyncSecretsDTO
 } from "./secret-types";
@@ -509,6 +517,19 @@ export const secretQueueFactory = ({
     );
   };
 
+  const sendFailedIntegrationSyncEmails = async (payload: TFailedIntegrationSyncEmailsPayload) => {
+    const appCfg = getConfig();
+    if (!appCfg.isSmtpConfigured) return;
+
+    await queueService.queue(QueueName.IntegrationSync, QueueJobs.SendFailedIntegrationSyncEmails, payload, {
+      jobId: `send-failed-integration-sync-emails-${payload.projectId}-${payload.secretPath}-${payload.environmentSlug}`,
+      delay: 1_000 * 60, // 1 minute
+
+      removeOnFail: true,
+      removeOnComplete: true
+    });
+  };
+
   queueService.start(QueueName.SecretSync, async (job) => {
     const {
       _deDupeQueue: deDupeQueue,
@@ -554,7 +575,46 @@ export const secretQueueFactory = ({
   });
 
   queueService.start(QueueName.IntegrationSync, async (job) => {
-    const { environment, actorId, isManual, projectId, secretPath, depth = 1, deDupeQueue = {} } = job.data;
+    if (job.name === QueueJobs.SendFailedIntegrationSyncEmails) {
+      const appCfg = getConfig();
+
+      const jobPayload = job.data as TFailedIntegrationSyncEmailsPayload;
+
+      const projectMembers = await projectMembershipDAL.findAllProjectMembers(jobPayload.projectId);
+      const project = await projectDAL.findById(jobPayload.projectId);
+
+      // Only send emails to admins, and if its a manual trigger, only send it to the person who triggered it (if actor is admin as well)
+      const filteredProjectMembers = projectMembers
+        .filter((member) => member.roles.some((role) => role.role === ProjectMembershipRole.Admin))
+        .filter((member) =>
+          jobPayload.manuallyTriggeredByUserId ? member.userId === jobPayload.manuallyTriggeredByUserId : true
+        );
+
+      await smtpService.sendMail({
+        recipients: filteredProjectMembers.map((member) => member.user.email!),
+        template: SmtpTemplates.IntegrationSyncFailed,
+        subjectLine: `Integration Sync Failed`,
+        substitutions: {
+          syncMessage: jobPayload.count === 1 ? jobPayload.syncMessage : undefined, // We are only displaying the sync message if its a singular integration, so we can just grab the first one in the array.
+          secretPath: jobPayload.secretPath,
+          environment: jobPayload.environmentName,
+          count: jobPayload.count,
+          projectName: project.name,
+          integrationUrl: `${appCfg.SITE_URL}/integrations/${project.id}`
+        }
+      });
+    }
+
+    if (job.name === QueueJobs.IntegrationSync) {
+      const {
+        environment,
+        actorId,
+        isManual,
+        projectId,
+        secretPath,
+        depth = 1,
+        deDupeQueue = {}
+      } = job.data as TIntegrationSyncPayload;
       if (depth > MAX_SYNC_SECRET_DEPTH) return;
 
       const folder = await folderDAL.findBySecretPath(projectId, environment, secretPath);
@@ -577,7 +637,7 @@ export const secretQueueFactory = ({
         const importedFolders = await folderDAL.findSecretPathByFolderIds(projectId, importedFolderIds);
         const foldersGroupedById = groupBy(importedFolders.filter(Boolean), (i) => i?.id as string);
         logger.info(
-        `getIntegrationSecrets: Syncing secret due to link change [jobId=${job.id}] [projectId=${job.data.projectId}] [environment=${job.data.environment}]  [secretPath=${job.data.secretPath}] [depth=${depth}]`
+          `getIntegrationSecrets: Syncing secret due to link change [jobId=${job.id}] [projectId=${job.data.projectId}] [environment=${environment}]  [secretPath=${job.data.secretPath}] [depth=${depth}]`
         );
         await Promise.all(
           imports
@@ -629,7 +689,7 @@ export const secretQueueFactory = ({
         const referencedFolders = await folderDAL.findSecretPathByFolderIds(projectId, referencedFolderIds);
         const referencedFoldersGroupedById = groupBy(referencedFolders.filter(Boolean), (i) => i?.id as string);
         logger.info(
-        `getIntegrationSecrets: Syncing secret due to reference change [jobId=${job.id}] [projectId=${job.data.projectId}] [environment=${job.data.environment}]  [secretPath=${job.data.secretPath}] [depth=${depth}]`
+          `getIntegrationSecrets: Syncing secret due to reference change [jobId=${job.id}] [projectId=${job.data.projectId}] [environment=${environment}]  [secretPath=${job.data.secretPath}] [depth=${depth}]`
         );
         await Promise.all(
           referencedFolderIds
@@ -663,9 +723,11 @@ export const secretQueueFactory = ({
         ({ secretPath: integrationSecPath, isActive }) => isActive && isSamePath(secretPath, integrationSecPath)
       );
 
+      const integrationsFailedToSync: { integrationId: string; syncMessage?: string }[] = [];
+
       if (!integrations.length) return;
       logger.info(
-      `getIntegrationSecrets: secret integration sync started [jobId=${job.id}] [jobId=${job.id}] [projectId=${job.data.projectId}] [environment=${job.data.environment}]  [secretPath=${job.data.secretPath}] [depth=${job.data.depth}]`
+        `getIntegrationSecrets: secret integration sync started [jobId=${job.id}] [jobId=${job.id}] [projectId=${job.data.projectId}] [environment=${environment}]  [secretPath=${job.data.secretPath}] [depth=${depth}]`
       );
 
       const lock = await keyStore.acquireLock(
@@ -688,7 +750,7 @@ export const secretQueueFactory = ({
         const isStaleSyncIntegration = new Date(job.timestamp) < new Date(lastRunSyncIntegrationTimestamp);
         if (isStaleSyncIntegration) {
           logger.info(
-          `getIntegrationSecrets: secret integration sync stale [jobId=${job.id}] [jobId=${job.id}] [projectId=${job.data.projectId}] [environment=${job.data.environment}]  [secretPath=${job.data.secretPath}] [depth=${job.data.depth}]`
+            `getIntegrationSecrets: secret integration sync stale [jobId=${job.id}] [jobId=${job.id}] [projectId=${job.data.projectId}] [environment=${environment}]  [secretPath=${job.data.secretPath}] [depth=${depth}]`
           );
           return;
         }
@@ -833,10 +895,18 @@ export const secretQueueFactory = ({
               syncMessage: response?.syncMessage ?? "",
               isSynced: response?.isSynced ?? true
             });
+
+            // May be undefined, if it's undefined we assume the sync was successful, hence the strict equality type check.
+            if (response?.isSynced === false) {
+              integrationsFailedToSync.push({
+                integrationId: integration.id,
+                syncMessage: response.syncMessage
+              });
+            }
           } catch (err) {
             logger.error(
               err,
-            `Secret integration sync error [projectId=${job.data.projectId}] [environment=${job.data.environment}]  [secretPath=${job.data.secretPath}]`
+              `Secret integration sync error [projectId=${job.data.projectId}] [environment=${environment}]  [secretPath=${job.data.secretPath}]`
             );
 
             const message =
@@ -863,10 +933,29 @@ export const secretQueueFactory = ({
               syncMessage: message,
               isSynced: false
             });
+
+            integrationsFailedToSync.push({
+              integrationId: integration.id,
+              syncMessage: message
+            });
           }
         }
       } finally {
         await lock.release();
+        if (integrationsFailedToSync.length) {
+          await sendFailedIntegrationSyncEmails({
+            count: integrationsFailedToSync.length,
+            environmentName: folder.environment.name,
+            environmentSlug: environment,
+            ...(isManual &&
+              actorId && {
+                manuallyTriggeredByUserId: actorId
+              }),
+            projectId,
+            secretPath,
+            syncMessage: integrationsFailedToSync[0].syncMessage
+          });
+        }
       }
 
       await keyStore.setItemWithExpiry(
@@ -875,6 +964,7 @@ export const secretQueueFactory = ({
         lockAcquiredTime.toISOString()
       );
       logger.info("Secret integration sync ended: %s", job.id);
+    }
   });
 
   queueService.start(QueueName.SecretReminder, async ({ data }) => {

backend/src/services/secret/secret-types.ts

@@ -1,4 +1,5 @@
 import { Knex } from "knex";
+import { z } from "zod";
 
 import { SecretType, TSecretBlindIndexes, TSecrets, TSecretsInsert, TSecretsUpdate } from "@app/db/schemas";
 import { TProjectPermission } from "@app/lib/types";
@@ -21,6 +22,29 @@ type TPartialSecret = Pick<TSecrets, "id" | "secretReminderRepeatDays" | "secret
 
 type TPartialInputSecret = Pick<TSecrets, "type" | "secretReminderNote" | "secretReminderRepeatDays" | "id">;
 
+export const FailedIntegrationSyncEmailsPayloadSchema = z.object({
+  projectId: z.string(),
+  secretPath: z.string(),
+  environmentName: z.string(),
+  environmentSlug: z.string(),
+
+  count: z.number(),
+  syncMessage: z.string().optional(),
+  manuallyTriggeredByUserId: z.string().optional()
+});
+
+export type TFailedIntegrationSyncEmailsPayload = z.infer<typeof FailedIntegrationSyncEmailsPayloadSchema>;
+
+export type TIntegrationSyncPayload = {
+  isManual?: boolean;
+  actorId?: string;
+  projectId: string;
+  environment: string;
+  secretPath: string;
+  depth?: number;
+  deDupeQueue?: Record<string, boolean>;
+};
+
 export type TCreateSecretDTO = {
   secretName: string;
   path: string;

backend/src/services/smtp/smtp-service.ts

@@ -33,7 +33,8 @@ export enum SmtpTemplates {
   SecretLeakIncident = "secretLeakIncident.handlebars",
   WorkspaceInvite = "workspaceInvitation.handlebars",
   ScimUserProvisioned = "scimUserProvisioned.handlebars",
-  PkiExpirationAlert = "pkiExpirationAlert.handlebars"
+  PkiExpirationAlert = "pkiExpirationAlert.handlebars",
+  IntegrationSyncFailed = "integrationSyncFailed.handlebars"
 }
 
 export enum SmtpHost {

backend/src/services/smtp/templates/integrationSyncFailed.handlebars

@@ -0,0 +1,31 @@
+<html>
+
+  <head>
+    <meta charset="utf-8" />
+    <meta http-equiv="x-ua-compatible" content="ie=edge" />
+    <title>Integration Sync Failed</title>
+  </head>
+
+  <body>
+    <h2>Infisical</h2>
+
+    <div>
+      <p>{{count}} integration(s) failed to sync.</p>
+      <a href="{{integrationUrl}}">
+        View your project integrations.
+      </a>
+    </div>
+
+    <br />
+    <div>
+      <p><strong>Project</strong>: {{projectName}}</p>
+      <p><strong>Environment</strong>: {{environment}}</p>
+      <p><strong>Secret Path</strong>: {{secretPath}}</p>
+    </div>
+
+    {{#if syncMessage}}
+      <p><b>Reason: </b>{{syncMessage}}</p>
+    {{/if}}
+  </body>
+
+</html>