From c6696d4fdec6cdbbe7e38abc80404c3889138907 Mon Sep 17 00:00:00 2001 From: x032205 Date: Wed, 8 Oct 2025 23:46:47 -0400 Subject: [PATCH] alerting for gateways and relays --- ...251008220303_relay-gateway-health-alarm.ts | 29 +++++ backend/src/db/schemas/gateways-v2.ts | 3 +- backend/src/db/schemas/relays.ts | 3 +- .../ee/services/gateway-v2/gateway-v2-dal.ts | 24 +++- .../services/gateway-v2/gateway-v2-service.ts | 91 +++++++++++++- backend/src/ee/services/relay/relay-dal.ts | 42 ++++++- .../src/ee/services/relay/relay-service.ts | 117 +++++++++++++++++- backend/src/server/routes/index.ts | 21 +++- .../notification/notification-types.ts | 4 +- .../smtp/emails/HealthAlertTemplate.tsx | 41 ++++++ backend/src/services/smtp/emails/index.ts | 1 + backend/src/services/smtp/smtp-service.ts | 7 +- 12 files changed, 365 insertions(+), 18 deletions(-) create mode 100644 backend/src/db/migrations/20251008220303_relay-gateway-health-alarm.ts create mode 100644 backend/src/services/smtp/emails/HealthAlertTemplate.tsx diff --git a/backend/src/db/migrations/20251008220303_relay-gateway-health-alarm.ts b/backend/src/db/migrations/20251008220303_relay-gateway-health-alarm.ts new file mode 100644 index 000000000..2abc1cf69 --- /dev/null +++ b/backend/src/db/migrations/20251008220303_relay-gateway-health-alarm.ts @@ -0,0 +1,29 @@ +import { Knex } from "knex"; + +import { TableName } from "../schemas"; + +export async function up(knex: Knex): Promise { + if (!(await knex.schema.hasColumn(TableName.Relay, "healthAlertedAt"))) { + await knex.schema.alterTable(TableName.Relay, (t) => { + t.datetime("healthAlertedAt"); + }); + } + if (!(await knex.schema.hasColumn(TableName.GatewayV2, "healthAlertedAt"))) { + await knex.schema.alterTable(TableName.GatewayV2, (t) => { + t.datetime("healthAlertedAt"); + }); + } +} + +export async function down(knex: Knex): Promise { + if (await knex.schema.hasColumn(TableName.GatewayV2, "healthAlertedAt")) { + await knex.schema.alterTable(TableName.GatewayV2, (t) => { + t.dropColumn("healthAlertedAt"); + }); + } + if (await knex.schema.hasColumn(TableName.Relay, "healthAlertedAt")) { + await knex.schema.alterTable(TableName.Relay, (t) => { + t.dropColumn("healthAlertedAt"); + }); + } +} diff --git a/backend/src/db/schemas/gateways-v2.ts b/backend/src/db/schemas/gateways-v2.ts index 1362793f6..4a48e3ce5 100644 --- a/backend/src/db/schemas/gateways-v2.ts +++ b/backend/src/db/schemas/gateways-v2.ts @@ -18,7 +18,8 @@ export const GatewaysV2Schema = z.object({ relayId: z.string().uuid().nullable().optional(), name: z.string(), heartbeat: z.date().nullable().optional(), - encryptedPamSessionKey: zodBuffer.nullable().optional() + encryptedPamSessionKey: zodBuffer.nullable().optional(), + healthAlertedAt: z.date().nullable().optional() }); export type TGatewaysV2 = z.infer; diff --git a/backend/src/db/schemas/relays.ts b/backend/src/db/schemas/relays.ts index 476e537c8..82a5fa830 100644 --- a/backend/src/db/schemas/relays.ts +++ b/backend/src/db/schemas/relays.ts @@ -15,7 +15,8 @@ export const RelaysSchema = z.object({ identityId: z.string().uuid().nullable().optional(), name: z.string(), host: z.string(), - heartbeat: z.date().nullable().optional() + heartbeat: z.date().nullable().optional(), + healthAlertedAt: z.date().nullable().optional() }); export type TRelays = z.infer; diff --git a/backend/src/ee/services/gateway-v2/gateway-v2-dal.ts b/backend/src/ee/services/gateway-v2/gateway-v2-dal.ts index da9d3c1ef..36feb3809 100644 --- a/backend/src/ee/services/gateway-v2/gateway-v2-dal.ts +++ b/backend/src/ee/services/gateway-v2/gateway-v2-dal.ts @@ -10,11 +10,16 @@ export type TGatewayV2DALFactory = ReturnType; export const gatewayV2DalFactory = (db: TDbClient) => { const orm = ormify(db, TableName.GatewayV2); - const find = async (filter: TFindFilter, { offset, limit, sort, tx }: TFindOpt = {}) => { + const find = async ( + filter: TFindFilter & { isHeartbeatStale?: boolean }, + { offset, limit, sort, tx }: TFindOpt = {} + ) => { try { + const { isHeartbeatStale, ...regularFilter } = filter; + const query = (tx || db.replicaNode())(TableName.GatewayV2) // eslint-disable-next-line @typescript-eslint/no-misused-promises - .where(buildFindFilter(filter, TableName.GatewayV2)) + .where(buildFindFilter(regularFilter, TableName.GatewayV2)) .join(TableName.Identity, `${TableName.Identity}.id`, `${TableName.GatewayV2}.identityId`) .join( TableName.IdentityOrgMembership, @@ -24,6 +29,21 @@ export const gatewayV2DalFactory = (db: TDbClient) => { .select(selectAllTableCols(TableName.GatewayV2)) .select(db.ref("name").withSchema(TableName.Identity).as("identityName")); + if (isHeartbeatStale) { + const oneHourAgo = new Date(); + oneHourAgo.setHours(oneHourAgo.getHours() - 1); + void query.where(`${TableName.GatewayV2}.heartbeat`, "<", oneHourAgo); + void query.where((v) => { + void v + .whereNull(`${TableName.GatewayV2}.healthAlertedAt`) + .orWhere( + `${TableName.GatewayV2}.healthAlertedAt`, + "<", + db.ref("heartbeat").withSchema(TableName.GatewayV2) + ); + }); + } + if (limit) void query.limit(limit); if (offset) void query.offset(offset); if (sort) { diff --git a/backend/src/ee/services/gateway-v2/gateway-v2-service.ts b/backend/src/ee/services/gateway-v2/gateway-v2-service.ts index 4bf6b1aef..128f9a651 100644 --- a/backend/src/ee/services/gateway-v2/gateway-v2-service.ts +++ b/backend/src/ee/services/gateway-v2/gateway-v2-service.ts @@ -2,14 +2,16 @@ import net from "node:net"; import { ForbiddenError } from "@casl/ability"; import * as x509 from "@peculiar/x509"; +import { CronJob } from "cron"; -import { TRelays } from "@app/db/schemas"; +import { OrgMembershipRole, TRelays } from "@app/db/schemas"; import { PgSqlLock } from "@app/keystore/keystore"; import { crypto } from "@app/lib/crypto"; import { DatabaseErrorCode } from "@app/lib/error-codes"; import { BadRequestError, DatabaseError, NotFoundError } from "@app/lib/errors"; import { GatewayProxyProtocol } from "@app/lib/gateway/types"; import { withGatewayV2Proxy } from "@app/lib/gateway-v2/gateway-v2"; +import { logger } from "@app/lib/logger"; import { OrgServiceActor } from "@app/lib/types"; import { ActorAuthMethod, ActorType } from "@app/services/auth/auth-type"; import { constructPemChainFromCerts } from "@app/services/certificate/certificate-fns"; @@ -20,6 +22,10 @@ import { } from "@app/services/certificate-authority/certificate-authority-fns"; import { TKmsServiceFactory } from "@app/services/kms/kms-service"; import { KmsDataKey } from "@app/services/kms/kms-types"; +import { TNotificationServiceFactory } from "@app/services/notification/notification-service"; +import { NotificationType } from "@app/services/notification/notification-types"; +import { TOrgDALFactory } from "@app/services/org/org-dal"; +import { SmtpTemplates, TSmtpService } from "@app/services/smtp/smtp-service"; import { TLicenseServiceFactory } from "../license/license-service"; import { PamResource } from "../pam-resource/pam-resource-enums"; @@ -39,6 +45,9 @@ type TGatewayV2ServiceFactoryDep = { gatewayV2DAL: TGatewayV2DALFactory; relayDAL: TRelayDALFactory; permissionService: TPermissionServiceFactory; + orgDAL: Pick; + notificationService: Pick; + smtpService: Pick; }; export type TGatewayV2ServiceFactory = ReturnType; @@ -50,7 +59,10 @@ export const gatewayV2ServiceFactory = ({ relayService, gatewayV2DAL, relayDAL, - permissionService + permissionService, + orgDAL, + notificationService, + smtpService }: TGatewayV2ServiceFactoryDep) => { const $validateIdentityAccessToGateway = async (orgId: string, actorId: string, actorAuthMethod: ActorAuthMethod) => { const orgLicensePlan = await licenseService.getPlan(orgId); @@ -878,6 +890,78 @@ export const gatewayV2ServiceFactory = ({ }); }; + const $healthcheckNotify = async () => { + const unhealthyGateways = await gatewayV2DAL.find({ + isHeartbeatStale: true + }); + + if (unhealthyGateways.length === 0) return; + + logger.warn( + { gatewayIds: unhealthyGateways.map((g) => g.id) }, + "Found gateways with last heartbeat over an hour ago. Sending notifications." + ); + + await Promise.all(unhealthyGateways.map((gw) => gatewayV2DAL.updateById(gw.id, { healthAlertedAt: new Date() }))); + + const gatewaysByOrg = unhealthyGateways.reduce>((acc, gw) => { + if (!acc[gw.orgId]) { + acc[gw.orgId] = []; + } + acc[gw.orgId].push(gw); + return acc; + }, {}); + + for await (const [orgId, gateways] of Object.entries(gatewaysByOrg)) { + try { + const admins = await orgDAL.findOrgMembersByRole(orgId, OrgMembershipRole.Admin); + if (admins.length === 0) { + logger.warn({ orgId }, "Organization has no admins to notify about unhealthy gateway."); + // eslint-disable-next-line no-continue + continue; + } + + const gatewayNames = gateways.map((g) => `"${g.name}"`).join(", "); + const body = `The following gateway(s) in your organization may be offline as they haven't reported a heartbeat in over an hour: ${gatewayNames}. Please check their status.`; + + await notificationService.createUserNotifications( + admins.map((admin) => ({ + userId: admin.user.id, + orgId, + type: NotificationType.GATEWAY_HEALTH_ALERT, + title: "Gateway Health Alert", + body, + link: "/organization/networking" + })) + ); + + await smtpService.sendMail({ + recipients: admins.map((admin) => admin.user.email).filter((v): v is string => !!v), + subjectLine: "Gateway Health Alert", + substitutions: { + type: "gateway", + names: gatewayNames + }, + template: SmtpTemplates.HealthAlert + }); + } catch (error) { + logger.error(error, `Failed to send gateway health notifications for organization [orgId=${orgId}]`); + } + } + }; + + const initializeHealthcheckNotify = async () => { + logger.info("Setting up background notification process for gateway v2 health-checks"); + + await $healthcheckNotify(); + + // run every 5 minutes + const job = new CronJob("*/5 * * * *", $healthcheckNotify); + job.start(); + + return job; + }; + return { listGateways, registerGateway, @@ -885,6 +969,7 @@ export const gatewayV2ServiceFactory = ({ getPAMConnectionDetails, deleteGatewayById, heartbeat, - getPamSessionKey + getPamSessionKey, + initializeHealthcheckNotify }; }; diff --git a/backend/src/ee/services/relay/relay-dal.ts b/backend/src/ee/services/relay/relay-dal.ts index 9107e0807..cef5e643a 100644 --- a/backend/src/ee/services/relay/relay-dal.ts +++ b/backend/src/ee/services/relay/relay-dal.ts @@ -1,11 +1,47 @@ import { TDbClient } from "@app/db"; -import { TableName } from "@app/db/schemas"; -import { ormify } from "@app/lib/knex"; +import { TableName, TRelays } from "@app/db/schemas"; +import { DatabaseError } from "@app/lib/errors"; +import { buildFindFilter, ormify, TFindFilter, TFindOpt } from "@app/lib/knex"; export type TRelayDALFactory = ReturnType; export const relayDalFactory = (db: TDbClient) => { const orm = ormify(db, TableName.Relay); - return orm; + const find = async ( + filter: TFindFilter & { isHeartbeatStale?: boolean }, + { offset, limit, sort, tx }: TFindOpt = {} + ) => { + try { + const { isHeartbeatStale, ...regularFilter } = filter; + + const query = (tx || db.replicaNode())(TableName.Relay) + // eslint-disable-next-line @typescript-eslint/no-misused-promises + .where(buildFindFilter(regularFilter, TableName.Relay)); + + if (isHeartbeatStale) { + const oneHourAgo = new Date(); + oneHourAgo.setHours(oneHourAgo.getHours() - 1); + void query.where(`${TableName.Relay}.heartbeat`, "<", oneHourAgo); + void query.where((v) => { + void v + .whereNull(`${TableName.Relay}.healthAlertedAt`) + .orWhere(`${TableName.Relay}.healthAlertedAt`, "<", db.ref("heartbeat").withSchema(TableName.Relay)); + }); + } + + if (limit) void query.limit(limit); + if (offset) void query.offset(offset); + if (sort) { + void query.orderBy(sort.map(([column, order, nulls]) => ({ column: column as string, order, nulls }))); + } + + const docs = await query; + return docs; + } catch (error) { + throw new DatabaseError({ error, name: `${TableName.Relay}: Find` }); + } + }; + + return { ...orm, find }; }; diff --git a/backend/src/ee/services/relay/relay-service.ts b/backend/src/ee/services/relay/relay-service.ts index 43e7cd1de..661a3d304 100644 --- a/backend/src/ee/services/relay/relay-service.ts +++ b/backend/src/ee/services/relay/relay-service.ts @@ -2,12 +2,14 @@ import { isIP } from "node:net"; import { ForbiddenError } from "@casl/ability"; import * as x509 from "@peculiar/x509"; +import { CronJob } from "cron"; -import { TRelays } from "@app/db/schemas"; +import { OrgMembershipRole, TRelays } from "@app/db/schemas"; import { PgSqlLock } from "@app/keystore/keystore"; import { crypto } from "@app/lib/crypto"; import { BadRequestError, ForbiddenRequestError, NotFoundError } from "@app/lib/errors"; import { createRelayConnection } from "@app/lib/gateway-v2/gateway-v2"; +import { logger } from "@app/lib/logger"; import { ActorAuthMethod, ActorType } from "@app/services/auth/auth-type"; import { constructPemChainFromCerts, prependCertToPemChain } from "@app/services/certificate/certificate-fns"; import { CertExtendedKeyUsage, CertKeyAlgorithm, CertKeyUsage } from "@app/services/certificate/certificate-types"; @@ -17,6 +19,11 @@ import { } from "@app/services/certificate-authority/certificate-authority-fns"; import { TKmsServiceFactory } from "@app/services/kms/kms-service"; import { KmsDataKey } from "@app/services/kms/kms-types"; +import { TNotificationServiceFactory } from "@app/services/notification/notification-service"; +import { NotificationType } from "@app/services/notification/notification-types"; +import { TOrgDALFactory } from "@app/services/org/org-dal"; +import { SmtpTemplates, TSmtpService } from "@app/services/smtp/smtp-service"; +import { TUserDALFactory } from "@app/services/user/user-dal"; import { verifyHostInputValidity } from "../dynamic-secret/dynamic-secret-fns"; import { TLicenseServiceFactory } from "../license/license-service"; @@ -40,7 +47,11 @@ export const relayServiceFactory = ({ relayDAL, kmsService, licenseService, - permissionService + permissionService, + orgDAL, + notificationService, + smtpService, + userDAL }: { instanceRelayConfigDAL: TInstanceRelayConfigDALFactory; orgRelayConfigDAL: TOrgRelayConfigDALFactory; @@ -48,6 +59,10 @@ export const relayServiceFactory = ({ kmsService: TKmsServiceFactory; licenseService: TLicenseServiceFactory; permissionService: TPermissionServiceFactory; + orgDAL: Pick; + notificationService: Pick; + smtpService: Pick; + userDAL: Pick; }) => { const $getInstanceCAs = async () => { const instanceConfig = await instanceRelayConfigDAL.transaction(async (tx) => { @@ -1193,12 +1208,108 @@ export const relayServiceFactory = ({ return deletedRelay; }; + const $healthcheckNotify = async () => { + const oneHourAgo = new Date(); + oneHourAgo.setHours(oneHourAgo.getHours() - 1); + const unhealthyRelays = await relayDAL.find({ + isHeartbeatStale: true + }); + + if (unhealthyRelays.length === 0) return; + + logger.warn( + { relayIds: unhealthyRelays.map((g) => g.id) }, + "Found relays with last heartbeat over an hour ago. Sending notifications." + ); + + await Promise.all(unhealthyRelays.map((r) => relayDAL.updateById(r.id, { healthAlertedAt: new Date() }))); + + const relaysByOrg = unhealthyRelays.reduce>((acc, r) => { + const key = r.orgId ?? "instance"; + if (!acc[key]) { + acc[key] = []; + } + acc[key].push(r); + return acc; + }, {}); + + for await (const [orgId, relays] of Object.entries(relaysByOrg)) { + try { + if (orgId === "instance") { + const superAdmins = await userDAL.find({ + superAdmin: true + }); + + const recipients = superAdmins.map((admin) => admin.email).filter((v): v is string => !!v); + + if (recipients.length > 0) { + const relayNames = relays.map((r) => `"${r.name}"`).join(", "); + await smtpService.sendMail({ + recipients, + subjectLine: "Relay Health Alert", + substitutions: { + type: "relay", + names: relayNames + }, + template: SmtpTemplates.HealthAlert + }); + } + } else { + const admins = await orgDAL.findOrgMembersByRole(orgId, OrgMembershipRole.Admin); + if (admins.length === 0) { + // eslint-disable-next-line no-continue + continue; + } + + const relayNames = relays.map((r) => `"${r.name}"`).join(", "); + const body = `The following relay(s) in your organization may be offline as they haven't reported a heartbeat in over an hour: ${relayNames}. Please check their status.`; + + await notificationService.createUserNotifications( + admins.map((admin) => ({ + userId: admin.user.id, + orgId, + type: NotificationType.RELAY_HEALTH_ALERT, + title: "Relay Health Alert", + body, + link: "/organization/networking" + })) + ); + + await smtpService.sendMail({ + recipients: admins.map((admin) => admin.user.email).filter((v): v is string => !!v), + subjectLine: "Relay Health Alert", + substitutions: { + type: "relay", + names: relayNames + }, + template: SmtpTemplates.HealthAlert + }); + } + } catch (error) { + logger.error(error, `Failed to send relay health notifications for organization [orgId=${orgId}]`); + } + } + }; + + const initializeHealthcheckNotify = async () => { + logger.info("Setting up background notification process for relay health-checks"); + + await $healthcheckNotify(); + + // run every 5 minutes + const job = new CronJob("*/5 * * * *", $healthcheckNotify); + job.start(); + + return job; + }; + return { registerRelay, getCredentialsForGateway, getCredentialsForClient, getRelays, deleteRelay, - heartbeat + heartbeat, + initializeHealthcheckNotify }; }; diff --git a/backend/src/server/routes/index.ts b/backend/src/server/routes/index.ts index 406daa63d..d16ee4034 100644 --- a/backend/src/server/routes/index.ts +++ b/backend/src/server/routes/index.ts @@ -1121,7 +1121,11 @@ export const registerRoutes = async ( relayDAL, kmsService, licenseService, - permissionService + permissionService, + orgDAL, + notificationService, + smtpService, + userDAL }); const gatewayV2Service = gatewayV2ServiceFactory({ @@ -1131,7 +1135,10 @@ export const registerRoutes = async ( orgGatewayConfigV2DAL, gatewayV2DAL, relayDAL, - permissionService + permissionService, + orgDAL, + notificationService, + smtpService }); const secretSyncQueue = secretSyncQueueFactory({ @@ -2330,6 +2337,16 @@ export const registerRoutes = async ( cronJobs.push(configSyncJob); } + const gatewayHealthcheckNotifyJob = await gatewayV2Service.initializeHealthcheckNotify(); + if (gatewayHealthcheckNotifyJob) { + cronJobs.push(gatewayHealthcheckNotifyJob); + } + + const relayHealthcheckNotifyJob = await relayService.initializeHealthcheckNotify(); + if (relayHealthcheckNotifyJob) { + cronJobs.push(relayHealthcheckNotifyJob); + } + const oauthConfigSyncJob = await initializeOauthConfigSync(); if (oauthConfigSyncJob) { cronJobs.push(oauthConfigSyncJob); diff --git a/backend/src/services/notification/notification-types.ts b/backend/src/services/notification/notification-types.ts index a3657c680..84cf35a50 100644 --- a/backend/src/services/notification/notification-types.ts +++ b/backend/src/services/notification/notification-types.ts @@ -15,7 +15,9 @@ export enum NotificationType { DIRECT_PROJECT_ACCESS_ISSUED_TO_ADMIN = "direct-project-access-issued-to-admin", PROJECT_ACCESS_REQUEST = "project-access-request", PROJECT_INVITATION = "project-invitation", - SECRET_SYNC_FAILED = "secret-sync-failed" + SECRET_SYNC_FAILED = "secret-sync-failed", + GATEWAY_HEALTH_ALERT = "gateway-health-alert", + RELAY_HEALTH_ALERT = "relay-health-alert" } export interface TCreateUserNotificationDTO { diff --git a/backend/src/services/smtp/emails/HealthAlertTemplate.tsx b/backend/src/services/smtp/emails/HealthAlertTemplate.tsx new file mode 100644 index 000000000..9b9a1f591 --- /dev/null +++ b/backend/src/services/smtp/emails/HealthAlertTemplate.tsx @@ -0,0 +1,41 @@ +import { Heading, Section, Text } from "@react-email/components"; + +import { BaseEmailWrapper, BaseEmailWrapperProps } from "./BaseEmailWrapper"; +import { BaseLink } from "./BaseLink"; + +interface HealthAlertTemplateProps extends Omit { + type: "gateway" | "relay"; + names: string; +} + +export const HealthAlertTemplate = ({ siteUrl, names, type }: HealthAlertTemplateProps) => { + return ( + + + {type === "gateway" ? "Gateway" : "Relay"} Health Alert + +
+ + The following {type}(s) in your organization may be offline as they haven't reported a + heartbeat in over an hour: {names}. + + + If your issue persists, you can contact the Infisical team at{" "} + support@infisical.com. + +
+ + ); +}; + +export default HealthAlertTemplate; + +HealthAlertTemplate.PreviewProps = { + type: "gateway", + names: '"gateway1", "gateway2"', + siteUrl: "https://infisical.com" +} as HealthAlertTemplateProps; diff --git a/backend/src/services/smtp/emails/index.ts b/backend/src/services/smtp/emails/index.ts index d7204085c..06ac31ab6 100644 --- a/backend/src/services/smtp/emails/index.ts +++ b/backend/src/services/smtp/emails/index.ts @@ -6,6 +6,7 @@ export * from "./EmailVerificationTemplate"; export * from "./ExternalImportFailedTemplate"; export * from "./ExternalImportStartedTemplate"; export * from "./ExternalImportSucceededTemplate"; +export * from "./HealthAlertTemplate"; export * from "./IntegrationSyncFailedTemplate"; export * from "./NewDeviceLoginTemplate"; export * from "./OAuthPasswordResetTemplate"; diff --git a/backend/src/services/smtp/smtp-service.ts b/backend/src/services/smtp/smtp-service.ts index 63645cf67..652f56567 100644 --- a/backend/src/services/smtp/smtp-service.ts +++ b/backend/src/services/smtp/smtp-service.ts @@ -15,6 +15,7 @@ import { ExternalImportFailedTemplate, ExternalImportStartedTemplate, ExternalImportSucceededTemplate, + HealthAlertTemplate, IntegrationSyncFailedTemplate, NewDeviceLoginTemplate, OAuthPasswordResetTemplate, @@ -85,7 +86,8 @@ export enum SmtpTemplates { ServiceTokenExpired = "serviceTokenExpired", SecretScanningV2ScanFailed = "secretScanningV2ScanFailed", SecretScanningV2SecretsDetected = "secretScanningV2SecretsDetected", - AccountDeletionConfirmation = "accountDeletionConfirmation" + AccountDeletionConfirmation = "accountDeletionConfirmation", + HealthAlert = "healthAlert" } export enum SmtpHost { @@ -131,7 +133,8 @@ const EmailTemplateMap: Record> = { [SmtpTemplates.PkiExpirationAlert]: PkiExpirationAlertTemplate, [SmtpTemplates.SecretScanningV2ScanFailed]: SecretScanningScanFailedTemplate, [SmtpTemplates.SecretScanningV2SecretsDetected]: SecretScanningSecretsDetectedTemplate, - [SmtpTemplates.AccountDeletionConfirmation]: AccountDeletionConfirmationTemplate + [SmtpTemplates.AccountDeletionConfirmation]: AccountDeletionConfirmationTemplate, + [SmtpTemplates.HealthAlert]: HealthAlertTemplate }; export const smtpServiceFactory = (cfg: TSmtpConfig) => {