Merge pull request #4636 from Infisical/ENG-3802

feat(gateway-v2, relays): health check or relays & health alerts

backend/src/db/migrations/20251008003912_relay-heartbeat.ts

@@ -0,0 +1,19 @@
+import { Knex } from "knex";
+
+import { TableName } from "../schemas";
+
+export async function up(knex: Knex): Promise<void> {
+  if (!(await knex.schema.hasColumn(TableName.Relay, "heartbeat"))) {
+    await knex.schema.alterTable(TableName.Relay, (t) => {
+      t.datetime("heartbeat");
+    });
+  }
+}
+
+export async function down(knex: Knex): Promise<void> {
+  if (await knex.schema.hasColumn(TableName.Relay, "heartbeat")) {
+    await knex.schema.alterTable(TableName.Relay, (t) => {
+      t.dropColumn("heartbeat");
+    });
+  }
+}

backend/src/db/migrations/20251008220303_relay-gateway-health-alarm.ts

@@ -0,0 +1,29 @@
+import { Knex } from "knex";
+
+import { TableName } from "../schemas";
+
+export async function up(knex: Knex): Promise<void> {
+  if (!(await knex.schema.hasColumn(TableName.Relay, "healthAlertedAt"))) {
+    await knex.schema.alterTable(TableName.Relay, (t) => {
+      t.datetime("healthAlertedAt");
+    });
+  }
+  if (!(await knex.schema.hasColumn(TableName.GatewayV2, "healthAlertedAt"))) {
+    await knex.schema.alterTable(TableName.GatewayV2, (t) => {
+      t.datetime("healthAlertedAt");
+    });
+  }
+}
+
+export async function down(knex: Knex): Promise<void> {
+  if (await knex.schema.hasColumn(TableName.GatewayV2, "healthAlertedAt")) {
+    await knex.schema.alterTable(TableName.GatewayV2, (t) => {
+      t.dropColumn("healthAlertedAt");
+    });
+  }
+  if (await knex.schema.hasColumn(TableName.Relay, "healthAlertedAt")) {
+    await knex.schema.alterTable(TableName.Relay, (t) => {
+      t.dropColumn("healthAlertedAt");
+    });
+  }
+}

backend/src/db/schemas/gateways-v2.ts

@@ -18,7 +18,8 @@ export const GatewaysV2Schema = z.object({
   relayId: z.string().uuid().nullable().optional(),
   name: z.string(),
   heartbeat: z.date().nullable().optional(),
-  encryptedPamSessionKey: zodBuffer.nullable().optional()
+  encryptedPamSessionKey: zodBuffer.nullable().optional(),
+  healthAlertedAt: z.date().nullable().optional()
 });
 
 export type TGatewaysV2 = z.infer<typeof GatewaysV2Schema>;

backend/src/db/schemas/relays.ts

@@ -14,7 +14,9 @@ export const RelaysSchema = z.object({
   orgId: z.string().uuid().nullable().optional(),
   identityId: z.string().uuid().nullable().optional(),
   name: z.string(),
-  host: z.string()
+  host: z.string(),
+  heartbeat: z.date().nullable().optional(),
+  healthAlertedAt: z.date().nullable().optional()
 });
 
 export type TRelays = z.infer<typeof RelaysSchema>;

backend/src/ee/routes/v1/relay-router.ts

@@ -146,4 +146,85 @@ export const registerRelayRouter = async (server: FastifyZodProvider) => {
       });
     }
   });
+
+  server.route({
+    method: "POST",
+    url: "/heartbeat-instance-relay",
+    config: {
+      rateLimit: writeLimit
+    },
+    schema: {
+      body: z.object({
+        name: slugSchema({ min: 1, max: 32, field: "name" })
+      }),
+      response: {
+        200: z.object({
+          message: z.string()
+        })
+      }
+    },
+    onRequest: (req, _, next) => {
+      const authHeader = req.headers.authorization;
+
+      if (!appCfg.RELAY_AUTH_SECRET) {
+        throw new UnauthorizedError({
+          message: "Relay authentication not configured"
+        });
+      }
+
+      if (!authHeader) {
+        throw new UnauthorizedError({
+          message: "Missing authorization header"
+        });
+      }
+
+      const expectedHeader = `Bearer ${appCfg.RELAY_AUTH_SECRET}`;
+      if (
+        authHeader.length === expectedHeader.length &&
+        crypto.nativeCrypto.timingSafeEqual(Buffer.from(authHeader), Buffer.from(expectedHeader))
+      ) {
+        return next();
+      }
+
+      throw new UnauthorizedError({
+        message: "Invalid relay auth secret"
+      });
+    },
+    handler: async (req) => {
+      await server.services.relay.heartbeat({
+        name: req.body.name
+      });
+
+      return { message: "Successfully triggered heartbeat" };
+    }
+  });
+
+  server.route({
+    method: "POST",
+    url: "/heartbeat-org-relay",
+    config: {
+      rateLimit: writeLimit
+    },
+    schema: {
+      body: z.object({
+        name: slugSchema({ min: 1, max: 32, field: "name" })
+      }),
+      response: {
+        200: z.object({
+          message: z.string()
+        })
+      }
+    },
+    onRequest: verifyAuth([AuthMode.IDENTITY_ACCESS_TOKEN]),
+    handler: async (req) => {
+      await server.services.relay.heartbeat({
+        name: req.body.name,
+        identityId: req.permission.id,
+        orgId: req.permission.orgId,
+        actorAuthMethod: req.permission.authMethod
+      });
+
+      return { message: "Successfully triggered heartbeat" };
+    }
+  });
 };

backend/src/ee/services/gateway-v2/gateway-v2-dal.ts

@@ -10,15 +10,34 @@ export type TGatewayV2DALFactory = ReturnType<typeof gatewayV2DalFactory>;
 export const gatewayV2DalFactory = (db: TDbClient) => {
   const orm = ormify(db, TableName.GatewayV2);
 
-  const find = async (filter: TFindFilter<TGatewaysV2>, { offset, limit, sort, tx }: TFindOpt<TGatewaysV2> = {}) => {
+  const find = async (
+    filter: TFindFilter<TGatewaysV2> & { isHeartbeatStale?: boolean },
+    { offset, limit, sort, tx }: TFindOpt<TGatewaysV2> = {}
+  ) => {
     try {
+      const { isHeartbeatStale, ...regularFilter } = filter;
+
       const query = (tx || db.replicaNode())(TableName.GatewayV2)
         // eslint-disable-next-line @typescript-eslint/no-misused-promises
-        .where(buildFindFilter(filter, TableName.GatewayV2))
+        .where(buildFindFilter(regularFilter, TableName.GatewayV2))
         .join(TableName.Identity, `${TableName.Identity}.id`, `${TableName.GatewayV2}.identityId`)
         .select(selectAllTableCols(TableName.GatewayV2))
         .select(db.ref("name").withSchema(TableName.Identity).as("identityName"));
 
+      if (isHeartbeatStale) {
+        const oneHourAgo = new Date(Date.now() - 60 * 60 * 1000);
+        void query.where(`${TableName.GatewayV2}.heartbeat`, "<", oneHourAgo);
+        void query.where((v) => {
+          void v
+            .whereNull(`${TableName.GatewayV2}.healthAlertedAt`)
+            .orWhere(
+              `${TableName.GatewayV2}.healthAlertedAt`,
+              "<",
+              db.ref("heartbeat").withSchema(TableName.GatewayV2)
+            );
+        });
+      }
+
       if (limit) void query.limit(limit);
       if (offset) void query.offset(offset);
       if (sort) {

backend/src/ee/services/gateway-v2/gateway-v2-service.ts

@@ -2,14 +2,17 @@ import net from "node:net";
 
 import { ForbiddenError } from "@casl/ability";
 import * as x509 from "@peculiar/x509";
+import { CronJob } from "cron";
 
-import { TRelays } from "@app/db/schemas";
+import { OrgMembershipRole, TRelays } from "@app/db/schemas";
 import { PgSqlLock } from "@app/keystore/keystore";
 import { crypto } from "@app/lib/crypto";
 import { DatabaseErrorCode } from "@app/lib/error-codes";
 import { BadRequestError, DatabaseError, NotFoundError } from "@app/lib/errors";
+import { groupBy } from "@app/lib/fn";
 import { GatewayProxyProtocol } from "@app/lib/gateway/types";
 import { withGatewayV2Proxy } from "@app/lib/gateway-v2/gateway-v2";
+import { logger } from "@app/lib/logger";
 import { OrgServiceActor } from "@app/lib/types";
 import { ActorAuthMethod, ActorType } from "@app/services/auth/auth-type";
 import { constructPemChainFromCerts } from "@app/services/certificate/certificate-fns";
@@ -20,6 +23,10 @@ import {
 } from "@app/services/certificate-authority/certificate-authority-fns";
 import { TKmsServiceFactory } from "@app/services/kms/kms-service";
 import { KmsDataKey } from "@app/services/kms/kms-types";
+import { TNotificationServiceFactory } from "@app/services/notification/notification-service";
+import { NotificationType } from "@app/services/notification/notification-types";
+import { TOrgDALFactory } from "@app/services/org/org-dal";
+import { SmtpTemplates, TSmtpService } from "@app/services/smtp/smtp-service";
 
 import { TLicenseServiceFactory } from "../license/license-service";
 import { PamResource } from "../pam-resource/pam-resource-enums";
@@ -39,6 +46,9 @@ type TGatewayV2ServiceFactoryDep = {
   gatewayV2DAL: TGatewayV2DALFactory;
   relayDAL: TRelayDALFactory;
   permissionService: TPermissionServiceFactory;
+  orgDAL: Pick<TOrgDALFactory, "findOrgMembersByRole">;
+  notificationService: Pick<TNotificationServiceFactory, "createUserNotifications">;
+  smtpService: Pick<TSmtpService, "sendMail">;
 };
 
 export type TGatewayV2ServiceFactory = ReturnType<typeof gatewayV2ServiceFactory>;
@@ -50,7 +60,10 @@ export const gatewayV2ServiceFactory = ({
   relayService,
   gatewayV2DAL,
   relayDAL,
-  permissionService
+  permissionService,
+  orgDAL,
+  notificationService,
+  smtpService
 }: TGatewayV2ServiceFactoryDep) => {
   const $validateIdentityAccessToGateway = async (orgId: string, actorId: string, actorAuthMethod: ActorAuthMethod) => {
     const orgLicensePlan = await licenseService.getPlan(orgId);
@@ -878,6 +891,72 @@ export const gatewayV2ServiceFactory = ({
     });
   };
 
+  const $healthcheckNotify = async () => {
+    const unhealthyGateways = await gatewayV2DAL.find({
+      isHeartbeatStale: true
+    });
+
+    if (unhealthyGateways.length === 0) return;
+
+    logger.warn(
+      { gatewayIds: unhealthyGateways.map((g) => g.id) },
+      "Found gateways with last heartbeat over an hour ago. Sending notifications."
+    );
+
+    const gatewaysByOrg = groupBy(unhealthyGateways, (gw) => gw.orgId);
+
+    for await (const [orgId, gateways] of Object.entries(gatewaysByOrg)) {
+      try {
+        const admins = await orgDAL.findOrgMembersByRole(orgId, OrgMembershipRole.Admin);
+        if (admins.length === 0) {
+          logger.warn({ orgId }, "Organization has no admins to notify about unhealthy gateway.");
+          // eslint-disable-next-line no-continue
+          continue;
+        }
+
+        const gatewayNames = gateways.map((g) => `"${g.name}"`).join(", ");
+        const body = `The following gateway(s) in your organization may be offline as they haven't reported a heartbeat in over an hour: ${gatewayNames}. Please check their status.`;
+
+        await notificationService.createUserNotifications(
+          admins.map((admin) => ({
+            userId: admin.user.id,
+            orgId,
+            type: NotificationType.GATEWAY_HEALTH_ALERT,
+            title: "Gateway Health Alert",
+            body,
+            link: "/organization/networking"
+          }))
+        );
+
+        await smtpService.sendMail({
+          recipients: admins.map((admin) => admin.user.email).filter((v): v is string => !!v),
+          subjectLine: "Gateway Health Alert",
+          substitutions: {
+            type: "gateway",
+            names: gatewayNames
+          },
+          template: SmtpTemplates.HealthAlert
+        });
+
+        await Promise.all(gateways.map((gw) => gatewayV2DAL.updateById(gw.id, { healthAlertedAt: new Date() })));
+      } catch (error) {
+        logger.error(error, `Failed to send gateway health notifications for organization [orgId=${orgId}]`);
+      }
+    }
+  };
+
+  const initializeHealthcheckNotify = async () => {
+    logger.info("Setting up background notification process for gateway v2 health-checks");
+
+    await $healthcheckNotify();
+
+    // run every 5 minutes
+    const job = new CronJob("*/5 * * * *", $healthcheckNotify);
+    job.start();
+
+    return job;
+  };
+
   return {
     listGateways,
     registerGateway,
@@ -885,6 +964,7 @@ export const gatewayV2ServiceFactory = ({
     getPAMConnectionDetails,
     deleteGatewayById,
     heartbeat,
-    getPamSessionKey
+    getPamSessionKey,
+    initializeHealthcheckNotify
   };
 };

backend/src/ee/services/relay/relay-dal.ts

@@ -1,11 +1,47 @@
 import { TDbClient } from "@app/db";
-import { TableName } from "@app/db/schemas";
+import { TableName, TRelays } from "@app/db/schemas";
-import { ormify } from "@app/lib/knex";
+import { DatabaseError } from "@app/lib/errors";
+import { buildFindFilter, ormify, TFindFilter, TFindOpt } from "@app/lib/knex";
 
 export type TRelayDALFactory = ReturnType<typeof relayDalFactory>;
 
 export const relayDalFactory = (db: TDbClient) => {
   const orm = ormify(db, TableName.Relay);
 
-  return orm;
+  const find = async (
+    filter: TFindFilter<TRelays> & { isHeartbeatStale?: boolean },
+    { offset, limit, sort, tx }: TFindOpt<TRelays> = {}
+  ) => {
+    try {
+      const { isHeartbeatStale, ...regularFilter } = filter;
+
+      const query = (tx || db.replicaNode())(TableName.Relay)
+        // eslint-disable-next-line @typescript-eslint/no-misused-promises
+        .where(buildFindFilter(regularFilter, TableName.Relay));
+
+      if (isHeartbeatStale) {
+        const oneHourAgo = new Date(Date.now() - 60 * 60 * 1000);
+        void query.whereNotNull(`${TableName.Relay}.heartbeat`);
+        void query.where(`${TableName.Relay}.heartbeat`, "<", oneHourAgo);
+        void query.where((v) => {
+          void v
+            .whereNull(`${TableName.Relay}.healthAlertedAt`)
+            .orWhere(`${TableName.Relay}.healthAlertedAt`, "<", db.ref("heartbeat").withSchema(TableName.Relay));
+        });
+      }
+
+      if (limit) void query.limit(limit);
+      if (offset) void query.offset(offset);
+      if (sort) {
+        void query.orderBy(sort.map(([column, order, nulls]) => ({ column: column as string, order, nulls })));
+      }
+
+      const docs = await query;
+      return docs;
+    } catch (error) {
+      throw new DatabaseError({ error, name: `${TableName.Relay}: Find` });
+    }
+  };
+
+  return { ...orm, find };
 };

backend/src/ee/services/relay/relay-service.ts

@@ -2,11 +2,15 @@ import { isIP } from "node:net";
 
 import { ForbiddenError } from "@casl/ability";
 import * as x509 from "@peculiar/x509";
+import { CronJob } from "cron";
 
-import { TRelays } from "@app/db/schemas";
+import { OrgMembershipRole, TRelays } from "@app/db/schemas";
 import { PgSqlLock } from "@app/keystore/keystore";
 import { crypto } from "@app/lib/crypto";
-import { BadRequestError, NotFoundError } from "@app/lib/errors";
+import { BadRequestError, ForbiddenRequestError, NotFoundError } from "@app/lib/errors";
+import { groupBy } from "@app/lib/fn";
+import { createRelayConnection } from "@app/lib/gateway-v2/gateway-v2";
+import { logger } from "@app/lib/logger";
 import { ActorAuthMethod, ActorType } from "@app/services/auth/auth-type";
 import { constructPemChainFromCerts, prependCertToPemChain } from "@app/services/certificate/certificate-fns";
 import { CertExtendedKeyUsage, CertKeyAlgorithm, CertKeyUsage } from "@app/services/certificate/certificate-types";
@@ -16,6 +20,11 @@ import {
 } from "@app/services/certificate-authority/certificate-authority-fns";
 import { TKmsServiceFactory } from "@app/services/kms/kms-service";
 import { KmsDataKey } from "@app/services/kms/kms-types";
+import { TNotificationServiceFactory } from "@app/services/notification/notification-service";
+import { NotificationType } from "@app/services/notification/notification-types";
+import { TOrgDALFactory } from "@app/services/org/org-dal";
+import { SmtpTemplates, TSmtpService } from "@app/services/smtp/smtp-service";
+import { TUserDALFactory } from "@app/services/user/user-dal";
 
 import { verifyHostInputValidity } from "../dynamic-secret/dynamic-secret-fns";
 import { TLicenseServiceFactory } from "../license/license-service";
@@ -39,7 +48,11 @@ export const relayServiceFactory = ({
   relayDAL,
   kmsService,
   licenseService,
-  permissionService
+  permissionService,
+  orgDAL,
+  notificationService,
+  smtpService,
+  userDAL
 }: {
   instanceRelayConfigDAL: TInstanceRelayConfigDALFactory;
   orgRelayConfigDAL: TOrgRelayConfigDALFactory;
@@ -47,6 +60,10 @@ export const relayServiceFactory = ({
   kmsService: TKmsServiceFactory;
   licenseService: TLicenseServiceFactory;
   permissionService: TPermissionServiceFactory;
+  orgDAL: Pick<TOrgDALFactory, "findOrgMembersByRole">;
+  notificationService: Pick<TNotificationServiceFactory, "createUserNotifications">;
+  smtpService: Pick<TSmtpService, "sendMail">;
+  userDAL: Pick<TUserDALFactory, "find">;
 }) => {
   const $getInstanceCAs = async () => {
     const instanceConfig = await instanceRelayConfigDAL.transaction(async (tx) => {
@@ -1056,6 +1073,78 @@ export const relayServiceFactory = ({
     });
   };
 
+  const heartbeat = async ({
+    name,
+    identityId,
+    actorAuthMethod,
+    orgId
+  }: {
+    name: string;
+    identityId?: string;
+    actorAuthMethod?: ActorAuthMethod;
+    orgId?: string;
+  }) => {
+    const relay = await relayDAL.findOne({
+      name,
+      orgId: orgId ?? null
+    });
+
+    if (!relay) {
+      throw new NotFoundError({ message: `Relay with name ${name} not found.` });
+    }
+
+    let clientOrgId: string;
+    let clientOrgName: string;
+
+    if (relay.orgId) {
+      if (!identityId || !orgId || relay.orgId !== orgId) {
+        throw new ForbiddenRequestError({
+          message: "You do not have permission to perform this action on this relay."
+        });
+      }
+
+      const { permission } = await permissionService.getOrgPermission(
+        ActorType.IDENTITY,
+        identityId,
+        orgId,
+        actorAuthMethod!,
+        orgId
+      );
+      ForbiddenError.from(permission).throwUnlessCan(
+        OrgPermissionRelayActions.CreateRelays,
+        OrgPermissionSubjects.Relay
+      );
+      clientOrgId = orgId;
+      clientOrgName = orgId;
+    } else {
+      clientOrgId = "00000000-0000-0000-0000-000000000000";
+      clientOrgName = "heartbeat";
+    }
+
+    const relayClientCredentials = await getCredentialsForClient({
+      relayId: relay.id,
+      orgId: clientOrgId,
+      orgName: clientOrgName,
+      gatewayId: "00000000-0000-0000-0000-000000000000",
+      gatewayName: "heartbeat",
+      duration: 60 * 1000 // 1 minute
+    });
+
+    try {
+      await createRelayConnection({
+        relayHost: relayClientCredentials.relayHost,
+        clientCertificate: relayClientCredentials.clientCertificate,
+        clientPrivateKey: relayClientCredentials.clientPrivateKey,
+        serverCertificateChain: relayClientCredentials.serverCertificateChain
+      });
+
+      await relayDAL.updateById(relay.id, { heartbeat: new Date() });
+    } catch (err) {
+      const error = err as Error;
+      throw new BadRequestError({ message: `Relay ${name} is not reachable: ${error.message}` });
+    }
+  };
+
   const getRelays = async ({
     actorId,
     actor,
@@ -1120,11 +1209,99 @@ export const relayServiceFactory = ({
     return deletedRelay;
   };
 
+  const $healthcheckNotify = async () => {
+    const unhealthyRelays = await relayDAL.find({
+      isHeartbeatStale: true
+    });
+
+    if (unhealthyRelays.length === 0) return;
+
+    logger.warn(
+      { relayIds: unhealthyRelays.map((g) => g.id) },
+      "Found relays with last heartbeat over an hour ago. Sending notifications."
+    );
+
+    const relaysByOrg = groupBy(unhealthyRelays, (r) => r.orgId ?? "instance");
+
+    for await (const [orgId, relays] of Object.entries(relaysByOrg)) {
+      try {
+        if (orgId === "instance") {
+          const superAdmins = await userDAL.find({
+            superAdmin: true
+          });
+
+          const recipients = superAdmins.map((admin) => admin.email).filter((v): v is string => !!v);
+
+          if (recipients.length > 0) {
+            const relayNames = relays.map((r) => `"${r.name}"`).join(", ");
+            await smtpService.sendMail({
+              recipients,
+              subjectLine: "Relay Health Alert",
+              substitutions: {
+                type: "instance-relay",
+                names: relayNames
+              },
+              template: SmtpTemplates.HealthAlert
+            });
+          }
+        } else {
+          const admins = await orgDAL.findOrgMembersByRole(orgId, OrgMembershipRole.Admin);
+          if (admins.length === 0) {
+            // eslint-disable-next-line no-continue
+            continue;
+          }
+
+          const relayNames = relays.map((r) => `"${r.name}"`).join(", ");
+          const body = `The following relay(s) in your organization may be offline as they haven't reported a heartbeat in over an hour: ${relayNames}. Please check their status.`;
+
+          await notificationService.createUserNotifications(
+            admins.map((admin) => ({
+              userId: admin.user.id,
+              orgId,
+              type: NotificationType.RELAY_HEALTH_ALERT,
+              title: "Relay Health Alert",
+              body,
+              link: "/organization/networking"
+            }))
+          );
+
+          await smtpService.sendMail({
+            recipients: admins.map((admin) => admin.user.email).filter((v): v is string => !!v),
+            subjectLine: "Relay Health Alert",
+            substitutions: {
+              type: "relay",
+              names: relayNames
+            },
+            template: SmtpTemplates.HealthAlert
+          });
+        }
+
+        await Promise.all(relays.map((r) => relayDAL.updateById(r.id, { healthAlertedAt: new Date() })));
+      } catch (error) {
+        logger.error(error, `Failed to send relay health notifications for organization [orgId=${orgId}]`);
+      }
+    }
+  };
+
+  const initializeHealthcheckNotify = async () => {
+    logger.info("Setting up background notification process for relay health-checks");
+
+    await $healthcheckNotify();
+
+    // run every 5 minutes
+    const job = new CronJob("*/5 * * * *", $healthcheckNotify);
+    job.start();
+
+    return job;
+  };
+
   return {
     registerRelay,
     getCredentialsForGateway,
     getCredentialsForClient,
     getRelays,
-    deleteRelay
+    deleteRelay,
+    heartbeat,
+    initializeHealthcheckNotify
   };
 };

backend/src/lib/gateway-v2/gateway-v2.ts

@@ -18,7 +18,7 @@ interface IGatewayRelayServer {
   getRelayError: () => string;
 }
 
-const createRelayConnection = async ({
+export const createRelayConnection = async ({
   relayHost,
   clientCertificate,
   clientPrivateKey,

backend/src/server/routes/index.ts

@@ -1160,7 +1160,11 @@ export const registerRoutes = async (
     relayDAL,
     kmsService,
     licenseService,
-    permissionService
+    permissionService,
+    orgDAL,
+    notificationService,
+    smtpService,
+    userDAL
   });
 
   const gatewayV2Service = gatewayV2ServiceFactory({
@@ -1170,7 +1174,10 @@ export const registerRoutes = async (
     orgGatewayConfigV2DAL,
     gatewayV2DAL,
     relayDAL,
-    permissionService
+    permissionService,
+    orgDAL,
+    notificationService,
+    smtpService
   });
 
   const secretSyncQueue = secretSyncQueueFactory({
@@ -2363,6 +2370,16 @@ export const registerRoutes = async (
     cronJobs.push(configSyncJob);
   }
 
+  const gatewayHealthcheckNotifyJob = await gatewayV2Service.initializeHealthcheckNotify();
+  if (gatewayHealthcheckNotifyJob) {
+    cronJobs.push(gatewayHealthcheckNotifyJob);
+  }
+
+  const relayHealthcheckNotifyJob = await relayService.initializeHealthcheckNotify();
+  if (relayHealthcheckNotifyJob) {
+    cronJobs.push(relayHealthcheckNotifyJob);
+  }
+
   const oauthConfigSyncJob = await initializeOauthConfigSync();
   if (oauthConfigSyncJob) {
     cronJobs.push(oauthConfigSyncJob);

backend/src/services/notification/notification-types.ts

@@ -15,7 +15,9 @@ export enum NotificationType {
   DIRECT_PROJECT_ACCESS_ISSUED_TO_ADMIN = "direct-project-access-issued-to-admin",
   PROJECT_ACCESS_REQUEST = "project-access-request",
   PROJECT_INVITATION = "project-invitation",
-  SECRET_SYNC_FAILED = "secret-sync-failed"
+  SECRET_SYNC_FAILED = "secret-sync-failed",
+  GATEWAY_HEALTH_ALERT = "gateway-health-alert",
+  RELAY_HEALTH_ALERT = "relay-health-alert"
 }
 
 export interface TCreateUserNotificationDTO {

backend/src/services/smtp/emails/HealthAlertTemplate.tsx

@@ -0,0 +1,47 @@
+import { Heading, Section, Text } from "@react-email/components";
+
+import { BaseEmailWrapper, BaseEmailWrapperProps } from "./BaseEmailWrapper";
+import { BaseLink } from "./BaseLink";
+
+interface HealthAlertTemplateProps extends Omit<BaseEmailWrapperProps, "title" | "preview" | "children"> {
+  type: "gateway" | "relay" | "instance-relay";
+  names: string;
+}
+
+export const HealthAlertTemplate = ({ siteUrl, names, type }: HealthAlertTemplateProps) => {
+  return (
+    <BaseEmailWrapper
+      title={`${type === "gateway" ? "Gateway" : "Relay"} Health Alert`}
+      preview={`Some ${type}s in your organization have failed their health check.`}
+      siteUrl={siteUrl}
+    >
+      <Heading className="text-black text-[18px] leading-[28px] text-center font-normal p-0 mx-0">
+        {type === "gateway" ? "Gateway" : "Relay"} Health Alert
+      </Heading>
+      <Section className="px-[24px] mt-[36px] pt-[12px] pb-[8px] border border-solid border-gray-200 rounded-md bg-gray-50">
+        <Text className="text-black text-[14px] leading-[24px]">
+          The following <strong>{type}</strong>(s) in your organization may be offline as they haven't reported a
+          heartbeat in over an hour: <strong>{names}</strong>.
+        </Text>
+        <Text className="text-black text-[14px] leading-[24px]">
+          {type === "instance-relay" && (
+            <>
+              If the issue persists, you can contact the Infisical team at{" "}
+              <BaseLink href="mailto:[email protected]">support@infisical.com</BaseLink>.
+            </>
+          )}
+          {type === "relay" && <>Please contact your relay administrators.</>}
+          {type === "gateway" && <>Please contact your gateway administrators.</>}
+        </Text>
+      </Section>
+    </BaseEmailWrapper>
+  );
+};
+
+export default HealthAlertTemplate;
+
+HealthAlertTemplate.PreviewProps = {
+  type: "gateway",
+  names: '"gateway1", "gateway2"',
+  siteUrl: "https://infisical.com"
+} as HealthAlertTemplateProps;

backend/src/services/smtp/emails/index.ts

@@ -6,6 +6,7 @@ export * from "./EmailVerificationTemplate";
 export * from "./ExternalImportFailedTemplate";
 export * from "./ExternalImportStartedTemplate";
 export * from "./ExternalImportSucceededTemplate";
+export * from "./HealthAlertTemplate";
 export * from "./IntegrationSyncFailedTemplate";
 export * from "./NewDeviceLoginTemplate";
 export * from "./OAuthPasswordResetTemplate";

backend/src/services/smtp/smtp-service.ts

@@ -15,6 +15,7 @@ import {
   ExternalImportFailedTemplate,
   ExternalImportStartedTemplate,
   ExternalImportSucceededTemplate,
+  HealthAlertTemplate,
   IntegrationSyncFailedTemplate,
   NewDeviceLoginTemplate,
   OAuthPasswordResetTemplate,
@@ -85,7 +86,8 @@ export enum SmtpTemplates {
   ServiceTokenExpired = "serviceTokenExpired",
   SecretScanningV2ScanFailed = "secretScanningV2ScanFailed",
   SecretScanningV2SecretsDetected = "secretScanningV2SecretsDetected",
-  AccountDeletionConfirmation = "accountDeletionConfirmation"
+  AccountDeletionConfirmation = "accountDeletionConfirmation",
+  HealthAlert = "healthAlert"
 }
 
 export enum SmtpHost {
@@ -131,7 +133,8 @@ const EmailTemplateMap: Record<SmtpTemplates, React.FC<any>> = {
   [SmtpTemplates.PkiExpirationAlert]: PkiExpirationAlertTemplate,
   [SmtpTemplates.SecretScanningV2ScanFailed]: SecretScanningScanFailedTemplate,
   [SmtpTemplates.SecretScanningV2SecretsDetected]: SecretScanningSecretsDetectedTemplate,
-  [SmtpTemplates.AccountDeletionConfirmation]: AccountDeletionConfirmationTemplate
+  [SmtpTemplates.AccountDeletionConfirmation]: AccountDeletionConfirmationTemplate,
+  [SmtpTemplates.HealthAlert]: HealthAlertTemplate
 };
 
 export const smtpServiceFactory = (cfg: TSmtpConfig) => {

frontend/src/hooks/api/relays/types.ts

@@ -6,6 +6,7 @@ export type TRelay = {
   identityId: string | null;
   name: string;
   host: string;
+  heartbeat: string;
 };
 
 export type TDeleteRelayDTO = {

frontend/src/pages/organization/NetworkingPage/components/RelayTab/RelayTab.tsx

@@ -5,6 +5,7 @@ import {
   faCopy,
   faDoorClosed,
   faEllipsisV,
+  faInfoCircle,
   faMagnifyingGlass,
   faSearch,
   faTrash
@@ -41,6 +42,25 @@ import { withPermission } from "@app/hoc";
 import { usePopUp } from "@app/hooks";
 import { useDeleteRelayById, useGetRelays } from "@app/hooks/api/relays";
 
+const RelayHealthStatus = ({ heartbeat }: { heartbeat?: string }) => {
+  const heartbeatDate = heartbeat ? new Date(heartbeat) : null;
+  const now = new Date();
+  const oneHourAgo = new Date(now.getTime() - 60 * 60 * 1000);
+
+  const isHealthy = !heartbeatDate || heartbeatDate >= oneHourAgo;
+  const tooltipContent = heartbeatDate
+    ? `Last heartbeat: ${heartbeatDate.toLocaleString()}`
+    : "No heartbeat data available";
+
+  return (
+    <Tooltip content={tooltipContent}>
+      <span className={`cursor-default ${isHealthy ? "text-green-400" : "text-red-400"}`}>
+        {isHealthy ? "Healthy" : "Unreachable"}
+      </span>
+    </Tooltip>
+  );
+};
+
 export const RelayTab = withPermission(
   () => {
     const [search, setSearch] = useState("");
@@ -106,6 +126,16 @@ export const RelayTab = withPermission(
                   <Th className="w-1/3">Name</Th>
                   <Th>Host</Th>
                   <Th>Created</Th>
+                  <Th>
+                    Health Check
+                    <Tooltip
+                      asChild={false}
+                      className="normal-case"
+                      content="The last known healthcheck. Triggers every 1 hour."
+                    >
+                      <FontAwesomeIcon icon={faInfoCircle} className="ml-2" />
+                    </Tooltip>
+                  </Th>
                   <Th className="w-5" />
                 </Tr>
               </THead>
@@ -129,6 +159,9 @@ export const RelayTab = withPermission(
                     </Td>
                     <Td>{el.host}</Td>
                     <Td>{formatRelative(new Date(el.createdAt), new Date())}</Td>
+                    <Td>
+                      <RelayHealthStatus heartbeat={el.heartbeat} />
+                    </Td>
                     <Td className="w-5">
                       <Tooltip className="max-w-sm text-center" content="Options">
                         <DropdownMenu>