diff --git a/backend/src/db/migrations/20251008003912_relay-heartbeat.ts b/backend/src/db/migrations/20251008003912_relay-heartbeat.ts new file mode 100644 index 000000000..f19c540cb --- /dev/null +++ b/backend/src/db/migrations/20251008003912_relay-heartbeat.ts @@ -0,0 +1,19 @@ +import { Knex } from "knex"; + +import { TableName } from "../schemas"; + +export async function up(knex: Knex): Promise { + if (!(await knex.schema.hasColumn(TableName.Relay, "heartbeat"))) { + await knex.schema.alterTable(TableName.Relay, (t) => { + t.datetime("heartbeat"); + }); + } +} + +export async function down(knex: Knex): Promise { + if (await knex.schema.hasColumn(TableName.Relay, "heartbeat")) { + await knex.schema.alterTable(TableName.Relay, (t) => { + t.dropColumn("heartbeat"); + }); + } +} diff --git a/backend/src/db/migrations/20251008220303_relay-gateway-health-alarm.ts b/backend/src/db/migrations/20251008220303_relay-gateway-health-alarm.ts new file mode 100644 index 000000000..2abc1cf69 --- /dev/null +++ b/backend/src/db/migrations/20251008220303_relay-gateway-health-alarm.ts @@ -0,0 +1,29 @@ +import { Knex } from "knex"; + +import { TableName } from "../schemas"; + +export async function up(knex: Knex): Promise { + if (!(await knex.schema.hasColumn(TableName.Relay, "healthAlertedAt"))) { + await knex.schema.alterTable(TableName.Relay, (t) => { + t.datetime("healthAlertedAt"); + }); + } + if (!(await knex.schema.hasColumn(TableName.GatewayV2, "healthAlertedAt"))) { + await knex.schema.alterTable(TableName.GatewayV2, (t) => { + t.datetime("healthAlertedAt"); + }); + } +} + +export async function down(knex: Knex): Promise { + if (await knex.schema.hasColumn(TableName.GatewayV2, "healthAlertedAt")) { + await knex.schema.alterTable(TableName.GatewayV2, (t) => { + t.dropColumn("healthAlertedAt"); + }); + } + if (await knex.schema.hasColumn(TableName.Relay, "healthAlertedAt")) { + await knex.schema.alterTable(TableName.Relay, (t) => { + t.dropColumn("healthAlertedAt"); + }); + } +} diff --git a/backend/src/db/schemas/gateways-v2.ts b/backend/src/db/schemas/gateways-v2.ts index 1362793f6..4a48e3ce5 100644 --- a/backend/src/db/schemas/gateways-v2.ts +++ b/backend/src/db/schemas/gateways-v2.ts @@ -18,7 +18,8 @@ export const GatewaysV2Schema = z.object({ relayId: z.string().uuid().nullable().optional(), name: z.string(), heartbeat: z.date().nullable().optional(), - encryptedPamSessionKey: zodBuffer.nullable().optional() + encryptedPamSessionKey: zodBuffer.nullable().optional(), + healthAlertedAt: z.date().nullable().optional() }); export type TGatewaysV2 = z.infer; diff --git a/backend/src/db/schemas/relays.ts b/backend/src/db/schemas/relays.ts index 4bb615e96..82a5fa830 100644 --- a/backend/src/db/schemas/relays.ts +++ b/backend/src/db/schemas/relays.ts @@ -14,7 +14,9 @@ export const RelaysSchema = z.object({ orgId: z.string().uuid().nullable().optional(), identityId: z.string().uuid().nullable().optional(), name: z.string(), - host: z.string() + host: z.string(), + heartbeat: z.date().nullable().optional(), + healthAlertedAt: z.date().nullable().optional() }); export type TRelays = z.infer; diff --git a/backend/src/ee/routes/v1/relay-router.ts b/backend/src/ee/routes/v1/relay-router.ts index f3d006b10..766025c21 100644 --- a/backend/src/ee/routes/v1/relay-router.ts +++ b/backend/src/ee/routes/v1/relay-router.ts @@ -146,4 +146,85 @@ export const registerRelayRouter = async (server: FastifyZodProvider) => { }); } }); + + server.route({ + method: "POST", + url: "/heartbeat-instance-relay", + config: { + rateLimit: writeLimit + }, + schema: { + body: z.object({ + name: slugSchema({ min: 1, max: 32, field: "name" }) + }), + response: { + 200: z.object({ + message: z.string() + }) + } + }, + onRequest: (req, _, next) => { + const authHeader = req.headers.authorization; + + if (!appCfg.RELAY_AUTH_SECRET) { + throw new UnauthorizedError({ + message: "Relay authentication not configured" + }); + } + + if (!authHeader) { + throw new UnauthorizedError({ + message: "Missing authorization header" + }); + } + + const expectedHeader = `Bearer ${appCfg.RELAY_AUTH_SECRET}`; + if ( + authHeader.length === expectedHeader.length && + crypto.nativeCrypto.timingSafeEqual(Buffer.from(authHeader), Buffer.from(expectedHeader)) + ) { + return next(); + } + + throw new UnauthorizedError({ + message: "Invalid relay auth secret" + }); + }, + handler: async (req) => { + await server.services.relay.heartbeat({ + name: req.body.name + }); + + return { message: "Successfully triggered heartbeat" }; + } + }); + + server.route({ + method: "POST", + url: "/heartbeat-org-relay", + config: { + rateLimit: writeLimit + }, + schema: { + body: z.object({ + name: slugSchema({ min: 1, max: 32, field: "name" }) + }), + response: { + 200: z.object({ + message: z.string() + }) + } + }, + onRequest: verifyAuth([AuthMode.IDENTITY_ACCESS_TOKEN]), + handler: async (req) => { + await server.services.relay.heartbeat({ + name: req.body.name, + identityId: req.permission.id, + orgId: req.permission.orgId, + actorAuthMethod: req.permission.authMethod + }); + + return { message: "Successfully triggered heartbeat" }; + } + }); }; diff --git a/backend/src/ee/services/gateway-v2/gateway-v2-dal.ts b/backend/src/ee/services/gateway-v2/gateway-v2-dal.ts index e0e7e582e..1896192cd 100644 --- a/backend/src/ee/services/gateway-v2/gateway-v2-dal.ts +++ b/backend/src/ee/services/gateway-v2/gateway-v2-dal.ts @@ -10,15 +10,34 @@ export type TGatewayV2DALFactory = ReturnType; export const gatewayV2DalFactory = (db: TDbClient) => { const orm = ormify(db, TableName.GatewayV2); - const find = async (filter: TFindFilter, { offset, limit, sort, tx }: TFindOpt = {}) => { + const find = async ( + filter: TFindFilter & { isHeartbeatStale?: boolean }, + { offset, limit, sort, tx }: TFindOpt = {} + ) => { try { + const { isHeartbeatStale, ...regularFilter } = filter; + const query = (tx || db.replicaNode())(TableName.GatewayV2) // eslint-disable-next-line @typescript-eslint/no-misused-promises - .where(buildFindFilter(filter, TableName.GatewayV2)) + .where(buildFindFilter(regularFilter, TableName.GatewayV2)) .join(TableName.Identity, `${TableName.Identity}.id`, `${TableName.GatewayV2}.identityId`) .select(selectAllTableCols(TableName.GatewayV2)) .select(db.ref("name").withSchema(TableName.Identity).as("identityName")); + if (isHeartbeatStale) { + const oneHourAgo = new Date(Date.now() - 60 * 60 * 1000); + void query.where(`${TableName.GatewayV2}.heartbeat`, "<", oneHourAgo); + void query.where((v) => { + void v + .whereNull(`${TableName.GatewayV2}.healthAlertedAt`) + .orWhere( + `${TableName.GatewayV2}.healthAlertedAt`, + "<", + db.ref("heartbeat").withSchema(TableName.GatewayV2) + ); + }); + } + if (limit) void query.limit(limit); if (offset) void query.offset(offset); if (sort) { diff --git a/backend/src/ee/services/gateway-v2/gateway-v2-service.ts b/backend/src/ee/services/gateway-v2/gateway-v2-service.ts index 4bf6b1aef..dce5bc299 100644 --- a/backend/src/ee/services/gateway-v2/gateway-v2-service.ts +++ b/backend/src/ee/services/gateway-v2/gateway-v2-service.ts @@ -2,14 +2,17 @@ import net from "node:net"; import { ForbiddenError } from "@casl/ability"; import * as x509 from "@peculiar/x509"; +import { CronJob } from "cron"; -import { TRelays } from "@app/db/schemas"; +import { OrgMembershipRole, TRelays } from "@app/db/schemas"; import { PgSqlLock } from "@app/keystore/keystore"; import { crypto } from "@app/lib/crypto"; import { DatabaseErrorCode } from "@app/lib/error-codes"; import { BadRequestError, DatabaseError, NotFoundError } from "@app/lib/errors"; +import { groupBy } from "@app/lib/fn"; import { GatewayProxyProtocol } from "@app/lib/gateway/types"; import { withGatewayV2Proxy } from "@app/lib/gateway-v2/gateway-v2"; +import { logger } from "@app/lib/logger"; import { OrgServiceActor } from "@app/lib/types"; import { ActorAuthMethod, ActorType } from "@app/services/auth/auth-type"; import { constructPemChainFromCerts } from "@app/services/certificate/certificate-fns"; @@ -20,6 +23,10 @@ import { } from "@app/services/certificate-authority/certificate-authority-fns"; import { TKmsServiceFactory } from "@app/services/kms/kms-service"; import { KmsDataKey } from "@app/services/kms/kms-types"; +import { TNotificationServiceFactory } from "@app/services/notification/notification-service"; +import { NotificationType } from "@app/services/notification/notification-types"; +import { TOrgDALFactory } from "@app/services/org/org-dal"; +import { SmtpTemplates, TSmtpService } from "@app/services/smtp/smtp-service"; import { TLicenseServiceFactory } from "../license/license-service"; import { PamResource } from "../pam-resource/pam-resource-enums"; @@ -39,6 +46,9 @@ type TGatewayV2ServiceFactoryDep = { gatewayV2DAL: TGatewayV2DALFactory; relayDAL: TRelayDALFactory; permissionService: TPermissionServiceFactory; + orgDAL: Pick; + notificationService: Pick; + smtpService: Pick; }; export type TGatewayV2ServiceFactory = ReturnType; @@ -50,7 +60,10 @@ export const gatewayV2ServiceFactory = ({ relayService, gatewayV2DAL, relayDAL, - permissionService + permissionService, + orgDAL, + notificationService, + smtpService }: TGatewayV2ServiceFactoryDep) => { const $validateIdentityAccessToGateway = async (orgId: string, actorId: string, actorAuthMethod: ActorAuthMethod) => { const orgLicensePlan = await licenseService.getPlan(orgId); @@ -878,6 +891,72 @@ export const gatewayV2ServiceFactory = ({ }); }; + const $healthcheckNotify = async () => { + const unhealthyGateways = await gatewayV2DAL.find({ + isHeartbeatStale: true + }); + + if (unhealthyGateways.length === 0) return; + + logger.warn( + { gatewayIds: unhealthyGateways.map((g) => g.id) }, + "Found gateways with last heartbeat over an hour ago. Sending notifications." + ); + + const gatewaysByOrg = groupBy(unhealthyGateways, (gw) => gw.orgId); + + for await (const [orgId, gateways] of Object.entries(gatewaysByOrg)) { + try { + const admins = await orgDAL.findOrgMembersByRole(orgId, OrgMembershipRole.Admin); + if (admins.length === 0) { + logger.warn({ orgId }, "Organization has no admins to notify about unhealthy gateway."); + // eslint-disable-next-line no-continue + continue; + } + + const gatewayNames = gateways.map((g) => `"${g.name}"`).join(", "); + const body = `The following gateway(s) in your organization may be offline as they haven't reported a heartbeat in over an hour: ${gatewayNames}. Please check their status.`; + + await notificationService.createUserNotifications( + admins.map((admin) => ({ + userId: admin.user.id, + orgId, + type: NotificationType.GATEWAY_HEALTH_ALERT, + title: "Gateway Health Alert", + body, + link: "/organization/networking" + })) + ); + + await smtpService.sendMail({ + recipients: admins.map((admin) => admin.user.email).filter((v): v is string => !!v), + subjectLine: "Gateway Health Alert", + substitutions: { + type: "gateway", + names: gatewayNames + }, + template: SmtpTemplates.HealthAlert + }); + + await Promise.all(gateways.map((gw) => gatewayV2DAL.updateById(gw.id, { healthAlertedAt: new Date() }))); + } catch (error) { + logger.error(error, `Failed to send gateway health notifications for organization [orgId=${orgId}]`); + } + } + }; + + const initializeHealthcheckNotify = async () => { + logger.info("Setting up background notification process for gateway v2 health-checks"); + + await $healthcheckNotify(); + + // run every 5 minutes + const job = new CronJob("*/5 * * * *", $healthcheckNotify); + job.start(); + + return job; + }; + return { listGateways, registerGateway, @@ -885,6 +964,7 @@ export const gatewayV2ServiceFactory = ({ getPAMConnectionDetails, deleteGatewayById, heartbeat, - getPamSessionKey + getPamSessionKey, + initializeHealthcheckNotify }; }; diff --git a/backend/src/ee/services/relay/relay-dal.ts b/backend/src/ee/services/relay/relay-dal.ts index 9107e0807..8e7eac8de 100644 --- a/backend/src/ee/services/relay/relay-dal.ts +++ b/backend/src/ee/services/relay/relay-dal.ts @@ -1,11 +1,47 @@ import { TDbClient } from "@app/db"; -import { TableName } from "@app/db/schemas"; -import { ormify } from "@app/lib/knex"; +import { TableName, TRelays } from "@app/db/schemas"; +import { DatabaseError } from "@app/lib/errors"; +import { buildFindFilter, ormify, TFindFilter, TFindOpt } from "@app/lib/knex"; export type TRelayDALFactory = ReturnType; export const relayDalFactory = (db: TDbClient) => { const orm = ormify(db, TableName.Relay); - return orm; + const find = async ( + filter: TFindFilter & { isHeartbeatStale?: boolean }, + { offset, limit, sort, tx }: TFindOpt = {} + ) => { + try { + const { isHeartbeatStale, ...regularFilter } = filter; + + const query = (tx || db.replicaNode())(TableName.Relay) + // eslint-disable-next-line @typescript-eslint/no-misused-promises + .where(buildFindFilter(regularFilter, TableName.Relay)); + + if (isHeartbeatStale) { + const oneHourAgo = new Date(Date.now() - 60 * 60 * 1000); + void query.whereNotNull(`${TableName.Relay}.heartbeat`); + void query.where(`${TableName.Relay}.heartbeat`, "<", oneHourAgo); + void query.where((v) => { + void v + .whereNull(`${TableName.Relay}.healthAlertedAt`) + .orWhere(`${TableName.Relay}.healthAlertedAt`, "<", db.ref("heartbeat").withSchema(TableName.Relay)); + }); + } + + if (limit) void query.limit(limit); + if (offset) void query.offset(offset); + if (sort) { + void query.orderBy(sort.map(([column, order, nulls]) => ({ column: column as string, order, nulls }))); + } + + const docs = await query; + return docs; + } catch (error) { + throw new DatabaseError({ error, name: `${TableName.Relay}: Find` }); + } + }; + + return { ...orm, find }; }; diff --git a/backend/src/ee/services/relay/relay-service.ts b/backend/src/ee/services/relay/relay-service.ts index 696bd1f4f..41fe91bfc 100644 --- a/backend/src/ee/services/relay/relay-service.ts +++ b/backend/src/ee/services/relay/relay-service.ts @@ -2,11 +2,15 @@ import { isIP } from "node:net"; import { ForbiddenError } from "@casl/ability"; import * as x509 from "@peculiar/x509"; +import { CronJob } from "cron"; -import { TRelays } from "@app/db/schemas"; +import { OrgMembershipRole, TRelays } from "@app/db/schemas"; import { PgSqlLock } from "@app/keystore/keystore"; import { crypto } from "@app/lib/crypto"; -import { BadRequestError, NotFoundError } from "@app/lib/errors"; +import { BadRequestError, ForbiddenRequestError, NotFoundError } from "@app/lib/errors"; +import { groupBy } from "@app/lib/fn"; +import { createRelayConnection } from "@app/lib/gateway-v2/gateway-v2"; +import { logger } from "@app/lib/logger"; import { ActorAuthMethod, ActorType } from "@app/services/auth/auth-type"; import { constructPemChainFromCerts, prependCertToPemChain } from "@app/services/certificate/certificate-fns"; import { CertExtendedKeyUsage, CertKeyAlgorithm, CertKeyUsage } from "@app/services/certificate/certificate-types"; @@ -16,6 +20,11 @@ import { } from "@app/services/certificate-authority/certificate-authority-fns"; import { TKmsServiceFactory } from "@app/services/kms/kms-service"; import { KmsDataKey } from "@app/services/kms/kms-types"; +import { TNotificationServiceFactory } from "@app/services/notification/notification-service"; +import { NotificationType } from "@app/services/notification/notification-types"; +import { TOrgDALFactory } from "@app/services/org/org-dal"; +import { SmtpTemplates, TSmtpService } from "@app/services/smtp/smtp-service"; +import { TUserDALFactory } from "@app/services/user/user-dal"; import { verifyHostInputValidity } from "../dynamic-secret/dynamic-secret-fns"; import { TLicenseServiceFactory } from "../license/license-service"; @@ -39,7 +48,11 @@ export const relayServiceFactory = ({ relayDAL, kmsService, licenseService, - permissionService + permissionService, + orgDAL, + notificationService, + smtpService, + userDAL }: { instanceRelayConfigDAL: TInstanceRelayConfigDALFactory; orgRelayConfigDAL: TOrgRelayConfigDALFactory; @@ -47,6 +60,10 @@ export const relayServiceFactory = ({ kmsService: TKmsServiceFactory; licenseService: TLicenseServiceFactory; permissionService: TPermissionServiceFactory; + orgDAL: Pick; + notificationService: Pick; + smtpService: Pick; + userDAL: Pick; }) => { const $getInstanceCAs = async () => { const instanceConfig = await instanceRelayConfigDAL.transaction(async (tx) => { @@ -1056,6 +1073,78 @@ export const relayServiceFactory = ({ }); }; + const heartbeat = async ({ + name, + identityId, + actorAuthMethod, + orgId + }: { + name: string; + identityId?: string; + actorAuthMethod?: ActorAuthMethod; + orgId?: string; + }) => { + const relay = await relayDAL.findOne({ + name, + orgId: orgId ?? null + }); + + if (!relay) { + throw new NotFoundError({ message: `Relay with name ${name} not found.` }); + } + + let clientOrgId: string; + let clientOrgName: string; + + if (relay.orgId) { + if (!identityId || !orgId || relay.orgId !== orgId) { + throw new ForbiddenRequestError({ + message: "You do not have permission to perform this action on this relay." + }); + } + + const { permission } = await permissionService.getOrgPermission( + ActorType.IDENTITY, + identityId, + orgId, + actorAuthMethod!, + orgId + ); + ForbiddenError.from(permission).throwUnlessCan( + OrgPermissionRelayActions.CreateRelays, + OrgPermissionSubjects.Relay + ); + clientOrgId = orgId; + clientOrgName = orgId; + } else { + clientOrgId = "00000000-0000-0000-0000-000000000000"; + clientOrgName = "heartbeat"; + } + + const relayClientCredentials = await getCredentialsForClient({ + relayId: relay.id, + orgId: clientOrgId, + orgName: clientOrgName, + gatewayId: "00000000-0000-0000-0000-000000000000", + gatewayName: "heartbeat", + duration: 60 * 1000 // 1 minute + }); + + try { + await createRelayConnection({ + relayHost: relayClientCredentials.relayHost, + clientCertificate: relayClientCredentials.clientCertificate, + clientPrivateKey: relayClientCredentials.clientPrivateKey, + serverCertificateChain: relayClientCredentials.serverCertificateChain + }); + + await relayDAL.updateById(relay.id, { heartbeat: new Date() }); + } catch (err) { + const error = err as Error; + throw new BadRequestError({ message: `Relay ${name} is not reachable: ${error.message}` }); + } + }; + const getRelays = async ({ actorId, actor, @@ -1120,11 +1209,99 @@ export const relayServiceFactory = ({ return deletedRelay; }; + const $healthcheckNotify = async () => { + const unhealthyRelays = await relayDAL.find({ + isHeartbeatStale: true + }); + + if (unhealthyRelays.length === 0) return; + + logger.warn( + { relayIds: unhealthyRelays.map((g) => g.id) }, + "Found relays with last heartbeat over an hour ago. Sending notifications." + ); + + const relaysByOrg = groupBy(unhealthyRelays, (r) => r.orgId ?? "instance"); + + for await (const [orgId, relays] of Object.entries(relaysByOrg)) { + try { + if (orgId === "instance") { + const superAdmins = await userDAL.find({ + superAdmin: true + }); + + const recipients = superAdmins.map((admin) => admin.email).filter((v): v is string => !!v); + + if (recipients.length > 0) { + const relayNames = relays.map((r) => `"${r.name}"`).join(", "); + await smtpService.sendMail({ + recipients, + subjectLine: "Relay Health Alert", + substitutions: { + type: "instance-relay", + names: relayNames + }, + template: SmtpTemplates.HealthAlert + }); + } + } else { + const admins = await orgDAL.findOrgMembersByRole(orgId, OrgMembershipRole.Admin); + if (admins.length === 0) { + // eslint-disable-next-line no-continue + continue; + } + + const relayNames = relays.map((r) => `"${r.name}"`).join(", "); + const body = `The following relay(s) in your organization may be offline as they haven't reported a heartbeat in over an hour: ${relayNames}. Please check their status.`; + + await notificationService.createUserNotifications( + admins.map((admin) => ({ + userId: admin.user.id, + orgId, + type: NotificationType.RELAY_HEALTH_ALERT, + title: "Relay Health Alert", + body, + link: "/organization/networking" + })) + ); + + await smtpService.sendMail({ + recipients: admins.map((admin) => admin.user.email).filter((v): v is string => !!v), + subjectLine: "Relay Health Alert", + substitutions: { + type: "relay", + names: relayNames + }, + template: SmtpTemplates.HealthAlert + }); + } + + await Promise.all(relays.map((r) => relayDAL.updateById(r.id, { healthAlertedAt: new Date() }))); + } catch (error) { + logger.error(error, `Failed to send relay health notifications for organization [orgId=${orgId}]`); + } + } + }; + + const initializeHealthcheckNotify = async () => { + logger.info("Setting up background notification process for relay health-checks"); + + await $healthcheckNotify(); + + // run every 5 minutes + const job = new CronJob("*/5 * * * *", $healthcheckNotify); + job.start(); + + return job; + }; + return { registerRelay, getCredentialsForGateway, getCredentialsForClient, getRelays, - deleteRelay + deleteRelay, + heartbeat, + initializeHealthcheckNotify }; }; diff --git a/backend/src/lib/gateway-v2/gateway-v2.ts b/backend/src/lib/gateway-v2/gateway-v2.ts index e6e873f11..5ae0e5b1d 100644 --- a/backend/src/lib/gateway-v2/gateway-v2.ts +++ b/backend/src/lib/gateway-v2/gateway-v2.ts @@ -18,7 +18,7 @@ interface IGatewayRelayServer { getRelayError: () => string; } -const createRelayConnection = async ({ +export const createRelayConnection = async ({ relayHost, clientCertificate, clientPrivateKey, diff --git a/backend/src/server/routes/index.ts b/backend/src/server/routes/index.ts index 824f5384c..01de9d559 100644 --- a/backend/src/server/routes/index.ts +++ b/backend/src/server/routes/index.ts @@ -1160,7 +1160,11 @@ export const registerRoutes = async ( relayDAL, kmsService, licenseService, - permissionService + permissionService, + orgDAL, + notificationService, + smtpService, + userDAL }); const gatewayV2Service = gatewayV2ServiceFactory({ @@ -1170,7 +1174,10 @@ export const registerRoutes = async ( orgGatewayConfigV2DAL, gatewayV2DAL, relayDAL, - permissionService + permissionService, + orgDAL, + notificationService, + smtpService }); const secretSyncQueue = secretSyncQueueFactory({ @@ -2363,6 +2370,16 @@ export const registerRoutes = async ( cronJobs.push(configSyncJob); } + const gatewayHealthcheckNotifyJob = await gatewayV2Service.initializeHealthcheckNotify(); + if (gatewayHealthcheckNotifyJob) { + cronJobs.push(gatewayHealthcheckNotifyJob); + } + + const relayHealthcheckNotifyJob = await relayService.initializeHealthcheckNotify(); + if (relayHealthcheckNotifyJob) { + cronJobs.push(relayHealthcheckNotifyJob); + } + const oauthConfigSyncJob = await initializeOauthConfigSync(); if (oauthConfigSyncJob) { cronJobs.push(oauthConfigSyncJob); diff --git a/backend/src/services/notification/notification-types.ts b/backend/src/services/notification/notification-types.ts index a3657c680..84cf35a50 100644 --- a/backend/src/services/notification/notification-types.ts +++ b/backend/src/services/notification/notification-types.ts @@ -15,7 +15,9 @@ export enum NotificationType { DIRECT_PROJECT_ACCESS_ISSUED_TO_ADMIN = "direct-project-access-issued-to-admin", PROJECT_ACCESS_REQUEST = "project-access-request", PROJECT_INVITATION = "project-invitation", - SECRET_SYNC_FAILED = "secret-sync-failed" + SECRET_SYNC_FAILED = "secret-sync-failed", + GATEWAY_HEALTH_ALERT = "gateway-health-alert", + RELAY_HEALTH_ALERT = "relay-health-alert" } export interface TCreateUserNotificationDTO { diff --git a/backend/src/services/smtp/emails/HealthAlertTemplate.tsx b/backend/src/services/smtp/emails/HealthAlertTemplate.tsx new file mode 100644 index 000000000..46b919f11 --- /dev/null +++ b/backend/src/services/smtp/emails/HealthAlertTemplate.tsx @@ -0,0 +1,47 @@ +import { Heading, Section, Text } from "@react-email/components"; + +import { BaseEmailWrapper, BaseEmailWrapperProps } from "./BaseEmailWrapper"; +import { BaseLink } from "./BaseLink"; + +interface HealthAlertTemplateProps extends Omit { + type: "gateway" | "relay" | "instance-relay"; + names: string; +} + +export const HealthAlertTemplate = ({ siteUrl, names, type }: HealthAlertTemplateProps) => { + return ( + + + {type === "gateway" ? "Gateway" : "Relay"} Health Alert + +
+ + The following {type}(s) in your organization may be offline as they haven't reported a + heartbeat in over an hour: {names}. + + + {type === "instance-relay" && ( + <> + If the issue persists, you can contact the Infisical team at{" "} + support@infisical.com. + + )} + {type === "relay" && <>Please contact your relay administrators.} + {type === "gateway" && <>Please contact your gateway administrators.} + +
+ + ); +}; + +export default HealthAlertTemplate; + +HealthAlertTemplate.PreviewProps = { + type: "gateway", + names: '"gateway1", "gateway2"', + siteUrl: "https://infisical.com" +} as HealthAlertTemplateProps; diff --git a/backend/src/services/smtp/emails/index.ts b/backend/src/services/smtp/emails/index.ts index d7204085c..06ac31ab6 100644 --- a/backend/src/services/smtp/emails/index.ts +++ b/backend/src/services/smtp/emails/index.ts @@ -6,6 +6,7 @@ export * from "./EmailVerificationTemplate"; export * from "./ExternalImportFailedTemplate"; export * from "./ExternalImportStartedTemplate"; export * from "./ExternalImportSucceededTemplate"; +export * from "./HealthAlertTemplate"; export * from "./IntegrationSyncFailedTemplate"; export * from "./NewDeviceLoginTemplate"; export * from "./OAuthPasswordResetTemplate"; diff --git a/backend/src/services/smtp/smtp-service.ts b/backend/src/services/smtp/smtp-service.ts index 63645cf67..652f56567 100644 --- a/backend/src/services/smtp/smtp-service.ts +++ b/backend/src/services/smtp/smtp-service.ts @@ -15,6 +15,7 @@ import { ExternalImportFailedTemplate, ExternalImportStartedTemplate, ExternalImportSucceededTemplate, + HealthAlertTemplate, IntegrationSyncFailedTemplate, NewDeviceLoginTemplate, OAuthPasswordResetTemplate, @@ -85,7 +86,8 @@ export enum SmtpTemplates { ServiceTokenExpired = "serviceTokenExpired", SecretScanningV2ScanFailed = "secretScanningV2ScanFailed", SecretScanningV2SecretsDetected = "secretScanningV2SecretsDetected", - AccountDeletionConfirmation = "accountDeletionConfirmation" + AccountDeletionConfirmation = "accountDeletionConfirmation", + HealthAlert = "healthAlert" } export enum SmtpHost { @@ -131,7 +133,8 @@ const EmailTemplateMap: Record> = { [SmtpTemplates.PkiExpirationAlert]: PkiExpirationAlertTemplate, [SmtpTemplates.SecretScanningV2ScanFailed]: SecretScanningScanFailedTemplate, [SmtpTemplates.SecretScanningV2SecretsDetected]: SecretScanningSecretsDetectedTemplate, - [SmtpTemplates.AccountDeletionConfirmation]: AccountDeletionConfirmationTemplate + [SmtpTemplates.AccountDeletionConfirmation]: AccountDeletionConfirmationTemplate, + [SmtpTemplates.HealthAlert]: HealthAlertTemplate }; export const smtpServiceFactory = (cfg: TSmtpConfig) => { diff --git a/frontend/src/hooks/api/relays/types.ts b/frontend/src/hooks/api/relays/types.ts index 621fd52db..1d40cb9f0 100644 --- a/frontend/src/hooks/api/relays/types.ts +++ b/frontend/src/hooks/api/relays/types.ts @@ -6,6 +6,7 @@ export type TRelay = { identityId: string | null; name: string; host: string; + heartbeat: string; }; export type TDeleteRelayDTO = { diff --git a/frontend/src/pages/organization/NetworkingPage/components/RelayTab/RelayTab.tsx b/frontend/src/pages/organization/NetworkingPage/components/RelayTab/RelayTab.tsx index 89b7bf4c9..e525b765d 100644 --- a/frontend/src/pages/organization/NetworkingPage/components/RelayTab/RelayTab.tsx +++ b/frontend/src/pages/organization/NetworkingPage/components/RelayTab/RelayTab.tsx @@ -5,6 +5,7 @@ import { faCopy, faDoorClosed, faEllipsisV, + faInfoCircle, faMagnifyingGlass, faSearch, faTrash @@ -41,6 +42,25 @@ import { withPermission } from "@app/hoc"; import { usePopUp } from "@app/hooks"; import { useDeleteRelayById, useGetRelays } from "@app/hooks/api/relays"; +const RelayHealthStatus = ({ heartbeat }: { heartbeat?: string }) => { + const heartbeatDate = heartbeat ? new Date(heartbeat) : null; + const now = new Date(); + const oneHourAgo = new Date(now.getTime() - 60 * 60 * 1000); + + const isHealthy = !heartbeatDate || heartbeatDate >= oneHourAgo; + const tooltipContent = heartbeatDate + ? `Last heartbeat: ${heartbeatDate.toLocaleString()}` + : "No heartbeat data available"; + + return ( + + + {isHealthy ? "Healthy" : "Unreachable"} + + + ); +}; + export const RelayTab = withPermission( () => { const [search, setSearch] = useState(""); @@ -106,6 +126,16 @@ export const RelayTab = withPermission( Name Host Created + + Health Check + + + + @@ -129,6 +159,9 @@ export const RelayTab = withPermission( {el.host} {formatRelative(new Date(el.createdAt), new Date())} + + +