From 12b59fc2c0b7006098ac545f695a0e47910366cc Mon Sep 17 00:00:00 2001 From: x032205 Date: Tue, 7 Oct 2025 21:17:25 -0400 Subject: [PATCH 1/6] feat(relays): healthcheck --- .../20251008003912_relay-heartbeat.ts | 19 +++++ backend/src/db/schemas/relays.ts | 3 +- backend/src/ee/routes/v1/relay-router.ts | 71 +++++++++++++++++ .../src/ee/services/relay/relay-service.ts | 78 ++++++++++++++++++- backend/src/lib/gateway-v2/gateway-v2.ts | 2 +- .../SecretSyncReviewFields.tsx | 5 +- frontend/src/hooks/api/relays/types.ts | 1 + .../components/GatewayTab/GatewayTab.tsx | 2 +- .../components/RelayTab/RelayTab.tsx | 16 ++++ 9 files changed, 191 insertions(+), 6 deletions(-) create mode 100644 backend/src/db/migrations/20251008003912_relay-heartbeat.ts diff --git a/backend/src/db/migrations/20251008003912_relay-heartbeat.ts b/backend/src/db/migrations/20251008003912_relay-heartbeat.ts new file mode 100644 index 000000000..f19c540cb --- /dev/null +++ b/backend/src/db/migrations/20251008003912_relay-heartbeat.ts @@ -0,0 +1,19 @@ +import { Knex } from "knex"; + +import { TableName } from "../schemas"; + +export async function up(knex: Knex): Promise { + if (!(await knex.schema.hasColumn(TableName.Relay, "heartbeat"))) { + await knex.schema.alterTable(TableName.Relay, (t) => { + t.datetime("heartbeat"); + }); + } +} + +export async function down(knex: Knex): Promise { + if (await knex.schema.hasColumn(TableName.Relay, "heartbeat")) { + await knex.schema.alterTable(TableName.Relay, (t) => { + t.dropColumn("heartbeat"); + }); + } +} diff --git a/backend/src/db/schemas/relays.ts b/backend/src/db/schemas/relays.ts index 4bb615e96..476e537c8 100644 --- a/backend/src/db/schemas/relays.ts +++ b/backend/src/db/schemas/relays.ts @@ -14,7 +14,8 @@ export const RelaysSchema = z.object({ orgId: z.string().uuid().nullable().optional(), identityId: z.string().uuid().nullable().optional(), name: z.string(), - host: z.string() + host: z.string(), + heartbeat: z.date().nullable().optional() }); export type TRelays = z.infer; diff --git a/backend/src/ee/routes/v1/relay-router.ts b/backend/src/ee/routes/v1/relay-router.ts index f3d006b10..5db5c0323 100644 --- a/backend/src/ee/routes/v1/relay-router.ts +++ b/backend/src/ee/routes/v1/relay-router.ts @@ -146,4 +146,75 @@ export const registerRelayRouter = async (server: FastifyZodProvider) => { }); } }); + + server.route({ + method: "POST", + url: "/heartbeat-instance-relay", + config: { + rateLimit: writeLimit + }, + schema: { + body: z.object({ + name: slugSchema({ min: 1, max: 32, field: "name" }) + }), + response: { + 200: z.object({ + message: z.string() + }) + } + }, + onRequest: (req, _, next) => { + const authHeader = req.headers.authorization; + + if (appCfg.RELAY_AUTH_SECRET && authHeader) { + const expectedHeader = `Bearer ${appCfg.RELAY_AUTH_SECRET}`; + if ( + authHeader.length === expectedHeader.length && + crypto.nativeCrypto.timingSafeEqual(Buffer.from(authHeader), Buffer.from(expectedHeader)) + ) { + return next(); + } + } + + throw new UnauthorizedError({ + message: "Invalid relay auth secret" + }); + }, + handler: async (req) => { + await server.services.relay.heartbeat({ + name: req.body.name + }); + + return { message: "Successfully triggered heartbeat" }; + } + }); + + server.route({ + method: "POST", + url: "/heartbeat-org-relay", + config: { + rateLimit: writeLimit + }, + schema: { + body: z.object({ + name: slugSchema({ min: 1, max: 32, field: "name" }) + }), + response: { + 200: z.object({ + message: z.string() + }) + } + }, + onRequest: verifyAuth([AuthMode.IDENTITY_ACCESS_TOKEN]), + handler: async (req) => { + await server.services.relay.heartbeat({ + name: req.body.name, + identityId: req.permission.id, + orgId: req.permission.orgId, + actorAuthMethod: req.permission.authMethod + }); + + return { message: "Successfully triggered heartbeat" }; + } + }); }; diff --git a/backend/src/ee/services/relay/relay-service.ts b/backend/src/ee/services/relay/relay-service.ts index 696bd1f4f..43e7cd1de 100644 --- a/backend/src/ee/services/relay/relay-service.ts +++ b/backend/src/ee/services/relay/relay-service.ts @@ -6,7 +6,8 @@ import * as x509 from "@peculiar/x509"; import { TRelays } from "@app/db/schemas"; import { PgSqlLock } from "@app/keystore/keystore"; import { crypto } from "@app/lib/crypto"; -import { BadRequestError, NotFoundError } from "@app/lib/errors"; +import { BadRequestError, ForbiddenRequestError, NotFoundError } from "@app/lib/errors"; +import { createRelayConnection } from "@app/lib/gateway-v2/gateway-v2"; import { ActorAuthMethod, ActorType } from "@app/services/auth/auth-type"; import { constructPemChainFromCerts, prependCertToPemChain } from "@app/services/certificate/certificate-fns"; import { CertExtendedKeyUsage, CertKeyAlgorithm, CertKeyUsage } from "@app/services/certificate/certificate-types"; @@ -1056,6 +1057,78 @@ export const relayServiceFactory = ({ }); }; + const heartbeat = async ({ + name, + identityId, + actorAuthMethod, + orgId + }: { + name: string; + identityId?: string; + actorAuthMethod?: ActorAuthMethod; + orgId?: string; + }) => { + const relay = await relayDAL.findOne({ + name, + orgId: orgId ?? null + }); + + if (!relay) { + throw new NotFoundError({ message: `Relay with name ${name} not found.` }); + } + + let clientOrgId: string; + let clientOrgName: string; + + if (relay.orgId) { + if (!identityId || !orgId || relay.orgId !== orgId) { + throw new ForbiddenRequestError({ + message: "You do not have permission to perform this action on this relay." + }); + } + + const { permission } = await permissionService.getOrgPermission( + ActorType.IDENTITY, + identityId, + orgId, + actorAuthMethod!, + orgId + ); + ForbiddenError.from(permission).throwUnlessCan( + OrgPermissionRelayActions.CreateRelays, + OrgPermissionSubjects.Relay + ); + clientOrgId = orgId; + clientOrgName = orgId; + } else { + clientOrgId = "00000000-0000-0000-0000-000000000000"; + clientOrgName = "heartbeat"; + } + + const relayClientCredentials = await getCredentialsForClient({ + relayId: relay.id, + orgId: clientOrgId, + orgName: clientOrgName, + gatewayId: "00000000-0000-0000-0000-000000000000", + gatewayName: "heartbeat", + duration: 60 * 1000 // 1 minute + }); + + try { + await createRelayConnection({ + relayHost: relayClientCredentials.relayHost, + clientCertificate: relayClientCredentials.clientCertificate, + clientPrivateKey: relayClientCredentials.clientPrivateKey, + serverCertificateChain: relayClientCredentials.serverCertificateChain + }); + + await relayDAL.updateById(relay.id, { heartbeat: new Date() }); + } catch (err) { + const error = err as Error; + throw new BadRequestError({ message: `Relay ${name} is not reachable: ${error.message}` }); + } + }; + const getRelays = async ({ actorId, actor, @@ -1125,6 +1198,7 @@ export const relayServiceFactory = ({ getCredentialsForGateway, getCredentialsForClient, getRelays, - deleteRelay + deleteRelay, + heartbeat }; }; diff --git a/backend/src/lib/gateway-v2/gateway-v2.ts b/backend/src/lib/gateway-v2/gateway-v2.ts index e6e873f11..5ae0e5b1d 100644 --- a/backend/src/lib/gateway-v2/gateway-v2.ts +++ b/backend/src/lib/gateway-v2/gateway-v2.ts @@ -18,7 +18,7 @@ interface IGatewayRelayServer { getRelayError: () => string; } -const createRelayConnection = async ({ +export const createRelayConnection = async ({ relayHost, clientCertificate, clientPrivateKey, diff --git a/frontend/src/components/secret-syncs/forms/SecretSyncReviewFields/SecretSyncReviewFields.tsx b/frontend/src/components/secret-syncs/forms/SecretSyncReviewFields/SecretSyncReviewFields.tsx index 84801b000..0969e446d 100644 --- a/frontend/src/components/secret-syncs/forms/SecretSyncReviewFields/SecretSyncReviewFields.tsx +++ b/frontend/src/components/secret-syncs/forms/SecretSyncReviewFields/SecretSyncReviewFields.tsx @@ -199,7 +199,10 @@ export const SecretSyncReviewFields = () => {

{duplicateProjectId && (

- Duplicate found in project ID: {duplicateProjectId} + Duplicate found in project ID:{" "} + + {duplicateProjectId} +

)} diff --git a/frontend/src/hooks/api/relays/types.ts b/frontend/src/hooks/api/relays/types.ts index 621fd52db..1d40cb9f0 100644 --- a/frontend/src/hooks/api/relays/types.ts +++ b/frontend/src/hooks/api/relays/types.ts @@ -6,6 +6,7 @@ export type TRelay = { identityId: string | null; name: string; host: string; + heartbeat: string; }; export type TDeleteRelayDTO = { diff --git a/frontend/src/pages/organization/NetworkingPage/components/GatewayTab/GatewayTab.tsx b/frontend/src/pages/organization/NetworkingPage/components/GatewayTab/GatewayTab.tsx index 0e3069aee..152222b29 100644 --- a/frontend/src/pages/organization/NetworkingPage/components/GatewayTab/GatewayTab.tsx +++ b/frontend/src/pages/organization/NetworkingPage/components/GatewayTab/GatewayTab.tsx @@ -48,7 +48,7 @@ import { useDeleteGatewayV2ById } from "@app/hooks/api/gateways-v2"; import { EditGatewayDetailsModal } from "./components/EditGatewayDetailsModal"; -const GatewayHealthStatus = ({ heartbeat }: { heartbeat?: string }) => { +export const GatewayHealthStatus = ({ heartbeat }: { heartbeat?: string }) => { const heartbeatDate = heartbeat ? new Date(heartbeat) : null; const now = new Date(); const oneHourAgo = new Date(now.getTime() - 60 * 60 * 1000); diff --git a/frontend/src/pages/organization/NetworkingPage/components/RelayTab/RelayTab.tsx b/frontend/src/pages/organization/NetworkingPage/components/RelayTab/RelayTab.tsx index 4eda269d5..51accf6b6 100644 --- a/frontend/src/pages/organization/NetworkingPage/components/RelayTab/RelayTab.tsx +++ b/frontend/src/pages/organization/NetworkingPage/components/RelayTab/RelayTab.tsx @@ -5,6 +5,7 @@ import { faCopy, faDoorClosed, faEllipsisV, + faInfoCircle, faMagnifyingGlass, faSearch, faTrash @@ -41,6 +42,8 @@ import { withPermission } from "@app/hoc"; import { usePopUp } from "@app/hooks"; import { useDeleteRelayById, useGetRelays } from "@app/hooks/api/relays"; +import { GatewayHealthStatus } from "../GatewayTab/GatewayTab"; + export const RelayTab = withPermission( () => { const [search, setSearch] = useState(""); @@ -106,6 +109,16 @@ export const RelayTab = withPermission( Name Host Created + + Health Check + + + + @@ -129,6 +142,9 @@ export const RelayTab = withPermission( {el.host} {formatRelative(new Date(el.createdAt), new Date())} + + + From c6696d4fdec6cdbbe7e38abc80404c3889138907 Mon Sep 17 00:00:00 2001 From: x032205 Date: Wed, 8 Oct 2025 23:46:47 -0400 Subject: [PATCH 2/6] alerting for gateways and relays --- ...251008220303_relay-gateway-health-alarm.ts | 29 +++++ backend/src/db/schemas/gateways-v2.ts | 3 +- backend/src/db/schemas/relays.ts | 3 +- .../ee/services/gateway-v2/gateway-v2-dal.ts | 24 +++- .../services/gateway-v2/gateway-v2-service.ts | 91 +++++++++++++- backend/src/ee/services/relay/relay-dal.ts | 42 ++++++- .../src/ee/services/relay/relay-service.ts | 117 +++++++++++++++++- backend/src/server/routes/index.ts | 21 +++- .../notification/notification-types.ts | 4 +- .../smtp/emails/HealthAlertTemplate.tsx | 41 ++++++ backend/src/services/smtp/emails/index.ts | 1 + backend/src/services/smtp/smtp-service.ts | 7 +- 12 files changed, 365 insertions(+), 18 deletions(-) create mode 100644 backend/src/db/migrations/20251008220303_relay-gateway-health-alarm.ts create mode 100644 backend/src/services/smtp/emails/HealthAlertTemplate.tsx diff --git a/backend/src/db/migrations/20251008220303_relay-gateway-health-alarm.ts b/backend/src/db/migrations/20251008220303_relay-gateway-health-alarm.ts new file mode 100644 index 000000000..2abc1cf69 --- /dev/null +++ b/backend/src/db/migrations/20251008220303_relay-gateway-health-alarm.ts @@ -0,0 +1,29 @@ +import { Knex } from "knex"; + +import { TableName } from "../schemas"; + +export async function up(knex: Knex): Promise { + if (!(await knex.schema.hasColumn(TableName.Relay, "healthAlertedAt"))) { + await knex.schema.alterTable(TableName.Relay, (t) => { + t.datetime("healthAlertedAt"); + }); + } + if (!(await knex.schema.hasColumn(TableName.GatewayV2, "healthAlertedAt"))) { + await knex.schema.alterTable(TableName.GatewayV2, (t) => { + t.datetime("healthAlertedAt"); + }); + } +} + +export async function down(knex: Knex): Promise { + if (await knex.schema.hasColumn(TableName.GatewayV2, "healthAlertedAt")) { + await knex.schema.alterTable(TableName.GatewayV2, (t) => { + t.dropColumn("healthAlertedAt"); + }); + } + if (await knex.schema.hasColumn(TableName.Relay, "healthAlertedAt")) { + await knex.schema.alterTable(TableName.Relay, (t) => { + t.dropColumn("healthAlertedAt"); + }); + } +} diff --git a/backend/src/db/schemas/gateways-v2.ts b/backend/src/db/schemas/gateways-v2.ts index 1362793f6..4a48e3ce5 100644 --- a/backend/src/db/schemas/gateways-v2.ts +++ b/backend/src/db/schemas/gateways-v2.ts @@ -18,7 +18,8 @@ export const GatewaysV2Schema = z.object({ relayId: z.string().uuid().nullable().optional(), name: z.string(), heartbeat: z.date().nullable().optional(), - encryptedPamSessionKey: zodBuffer.nullable().optional() + encryptedPamSessionKey: zodBuffer.nullable().optional(), + healthAlertedAt: z.date().nullable().optional() }); export type TGatewaysV2 = z.infer; diff --git a/backend/src/db/schemas/relays.ts b/backend/src/db/schemas/relays.ts index 476e537c8..82a5fa830 100644 --- a/backend/src/db/schemas/relays.ts +++ b/backend/src/db/schemas/relays.ts @@ -15,7 +15,8 @@ export const RelaysSchema = z.object({ identityId: z.string().uuid().nullable().optional(), name: z.string(), host: z.string(), - heartbeat: z.date().nullable().optional() + heartbeat: z.date().nullable().optional(), + healthAlertedAt: z.date().nullable().optional() }); export type TRelays = z.infer; diff --git a/backend/src/ee/services/gateway-v2/gateway-v2-dal.ts b/backend/src/ee/services/gateway-v2/gateway-v2-dal.ts index da9d3c1ef..36feb3809 100644 --- a/backend/src/ee/services/gateway-v2/gateway-v2-dal.ts +++ b/backend/src/ee/services/gateway-v2/gateway-v2-dal.ts @@ -10,11 +10,16 @@ export type TGatewayV2DALFactory = ReturnType; export const gatewayV2DalFactory = (db: TDbClient) => { const orm = ormify(db, TableName.GatewayV2); - const find = async (filter: TFindFilter, { offset, limit, sort, tx }: TFindOpt = {}) => { + const find = async ( + filter: TFindFilter & { isHeartbeatStale?: boolean }, + { offset, limit, sort, tx }: TFindOpt = {} + ) => { try { + const { isHeartbeatStale, ...regularFilter } = filter; + const query = (tx || db.replicaNode())(TableName.GatewayV2) // eslint-disable-next-line @typescript-eslint/no-misused-promises - .where(buildFindFilter(filter, TableName.GatewayV2)) + .where(buildFindFilter(regularFilter, TableName.GatewayV2)) .join(TableName.Identity, `${TableName.Identity}.id`, `${TableName.GatewayV2}.identityId`) .join( TableName.IdentityOrgMembership, @@ -24,6 +29,21 @@ export const gatewayV2DalFactory = (db: TDbClient) => { .select(selectAllTableCols(TableName.GatewayV2)) .select(db.ref("name").withSchema(TableName.Identity).as("identityName")); + if (isHeartbeatStale) { + const oneHourAgo = new Date(); + oneHourAgo.setHours(oneHourAgo.getHours() - 1); + void query.where(`${TableName.GatewayV2}.heartbeat`, "<", oneHourAgo); + void query.where((v) => { + void v + .whereNull(`${TableName.GatewayV2}.healthAlertedAt`) + .orWhere( + `${TableName.GatewayV2}.healthAlertedAt`, + "<", + db.ref("heartbeat").withSchema(TableName.GatewayV2) + ); + }); + } + if (limit) void query.limit(limit); if (offset) void query.offset(offset); if (sort) { diff --git a/backend/src/ee/services/gateway-v2/gateway-v2-service.ts b/backend/src/ee/services/gateway-v2/gateway-v2-service.ts index 4bf6b1aef..128f9a651 100644 --- a/backend/src/ee/services/gateway-v2/gateway-v2-service.ts +++ b/backend/src/ee/services/gateway-v2/gateway-v2-service.ts @@ -2,14 +2,16 @@ import net from "node:net"; import { ForbiddenError } from "@casl/ability"; import * as x509 from "@peculiar/x509"; +import { CronJob } from "cron"; -import { TRelays } from "@app/db/schemas"; +import { OrgMembershipRole, TRelays } from "@app/db/schemas"; import { PgSqlLock } from "@app/keystore/keystore"; import { crypto } from "@app/lib/crypto"; import { DatabaseErrorCode } from "@app/lib/error-codes"; import { BadRequestError, DatabaseError, NotFoundError } from "@app/lib/errors"; import { GatewayProxyProtocol } from "@app/lib/gateway/types"; import { withGatewayV2Proxy } from "@app/lib/gateway-v2/gateway-v2"; +import { logger } from "@app/lib/logger"; import { OrgServiceActor } from "@app/lib/types"; import { ActorAuthMethod, ActorType } from "@app/services/auth/auth-type"; import { constructPemChainFromCerts } from "@app/services/certificate/certificate-fns"; @@ -20,6 +22,10 @@ import { } from "@app/services/certificate-authority/certificate-authority-fns"; import { TKmsServiceFactory } from "@app/services/kms/kms-service"; import { KmsDataKey } from "@app/services/kms/kms-types"; +import { TNotificationServiceFactory } from "@app/services/notification/notification-service"; +import { NotificationType } from "@app/services/notification/notification-types"; +import { TOrgDALFactory } from "@app/services/org/org-dal"; +import { SmtpTemplates, TSmtpService } from "@app/services/smtp/smtp-service"; import { TLicenseServiceFactory } from "../license/license-service"; import { PamResource } from "../pam-resource/pam-resource-enums"; @@ -39,6 +45,9 @@ type TGatewayV2ServiceFactoryDep = { gatewayV2DAL: TGatewayV2DALFactory; relayDAL: TRelayDALFactory; permissionService: TPermissionServiceFactory; + orgDAL: Pick; + notificationService: Pick; + smtpService: Pick; }; export type TGatewayV2ServiceFactory = ReturnType; @@ -50,7 +59,10 @@ export const gatewayV2ServiceFactory = ({ relayService, gatewayV2DAL, relayDAL, - permissionService + permissionService, + orgDAL, + notificationService, + smtpService }: TGatewayV2ServiceFactoryDep) => { const $validateIdentityAccessToGateway = async (orgId: string, actorId: string, actorAuthMethod: ActorAuthMethod) => { const orgLicensePlan = await licenseService.getPlan(orgId); @@ -878,6 +890,78 @@ export const gatewayV2ServiceFactory = ({ }); }; + const $healthcheckNotify = async () => { + const unhealthyGateways = await gatewayV2DAL.find({ + isHeartbeatStale: true + }); + + if (unhealthyGateways.length === 0) return; + + logger.warn( + { gatewayIds: unhealthyGateways.map((g) => g.id) }, + "Found gateways with last heartbeat over an hour ago. Sending notifications." + ); + + await Promise.all(unhealthyGateways.map((gw) => gatewayV2DAL.updateById(gw.id, { healthAlertedAt: new Date() }))); + + const gatewaysByOrg = unhealthyGateways.reduce>((acc, gw) => { + if (!acc[gw.orgId]) { + acc[gw.orgId] = []; + } + acc[gw.orgId].push(gw); + return acc; + }, {}); + + for await (const [orgId, gateways] of Object.entries(gatewaysByOrg)) { + try { + const admins = await orgDAL.findOrgMembersByRole(orgId, OrgMembershipRole.Admin); + if (admins.length === 0) { + logger.warn({ orgId }, "Organization has no admins to notify about unhealthy gateway."); + // eslint-disable-next-line no-continue + continue; + } + + const gatewayNames = gateways.map((g) => `"${g.name}"`).join(", "); + const body = `The following gateway(s) in your organization may be offline as they haven't reported a heartbeat in over an hour: ${gatewayNames}. Please check their status.`; + + await notificationService.createUserNotifications( + admins.map((admin) => ({ + userId: admin.user.id, + orgId, + type: NotificationType.GATEWAY_HEALTH_ALERT, + title: "Gateway Health Alert", + body, + link: "/organization/networking" + })) + ); + + await smtpService.sendMail({ + recipients: admins.map((admin) => admin.user.email).filter((v): v is string => !!v), + subjectLine: "Gateway Health Alert", + substitutions: { + type: "gateway", + names: gatewayNames + }, + template: SmtpTemplates.HealthAlert + }); + } catch (error) { + logger.error(error, `Failed to send gateway health notifications for organization [orgId=${orgId}]`); + } + } + }; + + const initializeHealthcheckNotify = async () => { + logger.info("Setting up background notification process for gateway v2 health-checks"); + + await $healthcheckNotify(); + + // run every 5 minutes + const job = new CronJob("*/5 * * * *", $healthcheckNotify); + job.start(); + + return job; + }; + return { listGateways, registerGateway, @@ -885,6 +969,7 @@ export const gatewayV2ServiceFactory = ({ getPAMConnectionDetails, deleteGatewayById, heartbeat, - getPamSessionKey + getPamSessionKey, + initializeHealthcheckNotify }; }; diff --git a/backend/src/ee/services/relay/relay-dal.ts b/backend/src/ee/services/relay/relay-dal.ts index 9107e0807..cef5e643a 100644 --- a/backend/src/ee/services/relay/relay-dal.ts +++ b/backend/src/ee/services/relay/relay-dal.ts @@ -1,11 +1,47 @@ import { TDbClient } from "@app/db"; -import { TableName } from "@app/db/schemas"; -import { ormify } from "@app/lib/knex"; +import { TableName, TRelays } from "@app/db/schemas"; +import { DatabaseError } from "@app/lib/errors"; +import { buildFindFilter, ormify, TFindFilter, TFindOpt } from "@app/lib/knex"; export type TRelayDALFactory = ReturnType; export const relayDalFactory = (db: TDbClient) => { const orm = ormify(db, TableName.Relay); - return orm; + const find = async ( + filter: TFindFilter & { isHeartbeatStale?: boolean }, + { offset, limit, sort, tx }: TFindOpt = {} + ) => { + try { + const { isHeartbeatStale, ...regularFilter } = filter; + + const query = (tx || db.replicaNode())(TableName.Relay) + // eslint-disable-next-line @typescript-eslint/no-misused-promises + .where(buildFindFilter(regularFilter, TableName.Relay)); + + if (isHeartbeatStale) { + const oneHourAgo = new Date(); + oneHourAgo.setHours(oneHourAgo.getHours() - 1); + void query.where(`${TableName.Relay}.heartbeat`, "<", oneHourAgo); + void query.where((v) => { + void v + .whereNull(`${TableName.Relay}.healthAlertedAt`) + .orWhere(`${TableName.Relay}.healthAlertedAt`, "<", db.ref("heartbeat").withSchema(TableName.Relay)); + }); + } + + if (limit) void query.limit(limit); + if (offset) void query.offset(offset); + if (sort) { + void query.orderBy(sort.map(([column, order, nulls]) => ({ column: column as string, order, nulls }))); + } + + const docs = await query; + return docs; + } catch (error) { + throw new DatabaseError({ error, name: `${TableName.Relay}: Find` }); + } + }; + + return { ...orm, find }; }; diff --git a/backend/src/ee/services/relay/relay-service.ts b/backend/src/ee/services/relay/relay-service.ts index 43e7cd1de..661a3d304 100644 --- a/backend/src/ee/services/relay/relay-service.ts +++ b/backend/src/ee/services/relay/relay-service.ts @@ -2,12 +2,14 @@ import { isIP } from "node:net"; import { ForbiddenError } from "@casl/ability"; import * as x509 from "@peculiar/x509"; +import { CronJob } from "cron"; -import { TRelays } from "@app/db/schemas"; +import { OrgMembershipRole, TRelays } from "@app/db/schemas"; import { PgSqlLock } from "@app/keystore/keystore"; import { crypto } from "@app/lib/crypto"; import { BadRequestError, ForbiddenRequestError, NotFoundError } from "@app/lib/errors"; import { createRelayConnection } from "@app/lib/gateway-v2/gateway-v2"; +import { logger } from "@app/lib/logger"; import { ActorAuthMethod, ActorType } from "@app/services/auth/auth-type"; import { constructPemChainFromCerts, prependCertToPemChain } from "@app/services/certificate/certificate-fns"; import { CertExtendedKeyUsage, CertKeyAlgorithm, CertKeyUsage } from "@app/services/certificate/certificate-types"; @@ -17,6 +19,11 @@ import { } from "@app/services/certificate-authority/certificate-authority-fns"; import { TKmsServiceFactory } from "@app/services/kms/kms-service"; import { KmsDataKey } from "@app/services/kms/kms-types"; +import { TNotificationServiceFactory } from "@app/services/notification/notification-service"; +import { NotificationType } from "@app/services/notification/notification-types"; +import { TOrgDALFactory } from "@app/services/org/org-dal"; +import { SmtpTemplates, TSmtpService } from "@app/services/smtp/smtp-service"; +import { TUserDALFactory } from "@app/services/user/user-dal"; import { verifyHostInputValidity } from "../dynamic-secret/dynamic-secret-fns"; import { TLicenseServiceFactory } from "../license/license-service"; @@ -40,7 +47,11 @@ export const relayServiceFactory = ({ relayDAL, kmsService, licenseService, - permissionService + permissionService, + orgDAL, + notificationService, + smtpService, + userDAL }: { instanceRelayConfigDAL: TInstanceRelayConfigDALFactory; orgRelayConfigDAL: TOrgRelayConfigDALFactory; @@ -48,6 +59,10 @@ export const relayServiceFactory = ({ kmsService: TKmsServiceFactory; licenseService: TLicenseServiceFactory; permissionService: TPermissionServiceFactory; + orgDAL: Pick; + notificationService: Pick; + smtpService: Pick; + userDAL: Pick; }) => { const $getInstanceCAs = async () => { const instanceConfig = await instanceRelayConfigDAL.transaction(async (tx) => { @@ -1193,12 +1208,108 @@ export const relayServiceFactory = ({ return deletedRelay; }; + const $healthcheckNotify = async () => { + const oneHourAgo = new Date(); + oneHourAgo.setHours(oneHourAgo.getHours() - 1); + const unhealthyRelays = await relayDAL.find({ + isHeartbeatStale: true + }); + + if (unhealthyRelays.length === 0) return; + + logger.warn( + { relayIds: unhealthyRelays.map((g) => g.id) }, + "Found relays with last heartbeat over an hour ago. Sending notifications." + ); + + await Promise.all(unhealthyRelays.map((r) => relayDAL.updateById(r.id, { healthAlertedAt: new Date() }))); + + const relaysByOrg = unhealthyRelays.reduce>((acc, r) => { + const key = r.orgId ?? "instance"; + if (!acc[key]) { + acc[key] = []; + } + acc[key].push(r); + return acc; + }, {}); + + for await (const [orgId, relays] of Object.entries(relaysByOrg)) { + try { + if (orgId === "instance") { + const superAdmins = await userDAL.find({ + superAdmin: true + }); + + const recipients = superAdmins.map((admin) => admin.email).filter((v): v is string => !!v); + + if (recipients.length > 0) { + const relayNames = relays.map((r) => `"${r.name}"`).join(", "); + await smtpService.sendMail({ + recipients, + subjectLine: "Relay Health Alert", + substitutions: { + type: "relay", + names: relayNames + }, + template: SmtpTemplates.HealthAlert + }); + } + } else { + const admins = await orgDAL.findOrgMembersByRole(orgId, OrgMembershipRole.Admin); + if (admins.length === 0) { + // eslint-disable-next-line no-continue + continue; + } + + const relayNames = relays.map((r) => `"${r.name}"`).join(", "); + const body = `The following relay(s) in your organization may be offline as they haven't reported a heartbeat in over an hour: ${relayNames}. Please check their status.`; + + await notificationService.createUserNotifications( + admins.map((admin) => ({ + userId: admin.user.id, + orgId, + type: NotificationType.RELAY_HEALTH_ALERT, + title: "Relay Health Alert", + body, + link: "/organization/networking" + })) + ); + + await smtpService.sendMail({ + recipients: admins.map((admin) => admin.user.email).filter((v): v is string => !!v), + subjectLine: "Relay Health Alert", + substitutions: { + type: "relay", + names: relayNames + }, + template: SmtpTemplates.HealthAlert + }); + } + } catch (error) { + logger.error(error, `Failed to send relay health notifications for organization [orgId=${orgId}]`); + } + } + }; + + const initializeHealthcheckNotify = async () => { + logger.info("Setting up background notification process for relay health-checks"); + + await $healthcheckNotify(); + + // run every 5 minutes + const job = new CronJob("*/5 * * * *", $healthcheckNotify); + job.start(); + + return job; + }; + return { registerRelay, getCredentialsForGateway, getCredentialsForClient, getRelays, deleteRelay, - heartbeat + heartbeat, + initializeHealthcheckNotify }; }; diff --git a/backend/src/server/routes/index.ts b/backend/src/server/routes/index.ts index 406daa63d..d16ee4034 100644 --- a/backend/src/server/routes/index.ts +++ b/backend/src/server/routes/index.ts @@ -1121,7 +1121,11 @@ export const registerRoutes = async ( relayDAL, kmsService, licenseService, - permissionService + permissionService, + orgDAL, + notificationService, + smtpService, + userDAL }); const gatewayV2Service = gatewayV2ServiceFactory({ @@ -1131,7 +1135,10 @@ export const registerRoutes = async ( orgGatewayConfigV2DAL, gatewayV2DAL, relayDAL, - permissionService + permissionService, + orgDAL, + notificationService, + smtpService }); const secretSyncQueue = secretSyncQueueFactory({ @@ -2330,6 +2337,16 @@ export const registerRoutes = async ( cronJobs.push(configSyncJob); } + const gatewayHealthcheckNotifyJob = await gatewayV2Service.initializeHealthcheckNotify(); + if (gatewayHealthcheckNotifyJob) { + cronJobs.push(gatewayHealthcheckNotifyJob); + } + + const relayHealthcheckNotifyJob = await relayService.initializeHealthcheckNotify(); + if (relayHealthcheckNotifyJob) { + cronJobs.push(relayHealthcheckNotifyJob); + } + const oauthConfigSyncJob = await initializeOauthConfigSync(); if (oauthConfigSyncJob) { cronJobs.push(oauthConfigSyncJob); diff --git a/backend/src/services/notification/notification-types.ts b/backend/src/services/notification/notification-types.ts index a3657c680..84cf35a50 100644 --- a/backend/src/services/notification/notification-types.ts +++ b/backend/src/services/notification/notification-types.ts @@ -15,7 +15,9 @@ export enum NotificationType { DIRECT_PROJECT_ACCESS_ISSUED_TO_ADMIN = "direct-project-access-issued-to-admin", PROJECT_ACCESS_REQUEST = "project-access-request", PROJECT_INVITATION = "project-invitation", - SECRET_SYNC_FAILED = "secret-sync-failed" + SECRET_SYNC_FAILED = "secret-sync-failed", + GATEWAY_HEALTH_ALERT = "gateway-health-alert", + RELAY_HEALTH_ALERT = "relay-health-alert" } export interface TCreateUserNotificationDTO { diff --git a/backend/src/services/smtp/emails/HealthAlertTemplate.tsx b/backend/src/services/smtp/emails/HealthAlertTemplate.tsx new file mode 100644 index 000000000..9b9a1f591 --- /dev/null +++ b/backend/src/services/smtp/emails/HealthAlertTemplate.tsx @@ -0,0 +1,41 @@ +import { Heading, Section, Text } from "@react-email/components"; + +import { BaseEmailWrapper, BaseEmailWrapperProps } from "./BaseEmailWrapper"; +import { BaseLink } from "./BaseLink"; + +interface HealthAlertTemplateProps extends Omit { + type: "gateway" | "relay"; + names: string; +} + +export const HealthAlertTemplate = ({ siteUrl, names, type }: HealthAlertTemplateProps) => { + return ( + + + {type === "gateway" ? "Gateway" : "Relay"} Health Alert + +
+ + The following {type}(s) in your organization may be offline as they haven't reported a + heartbeat in over an hour: {names}. + + + If your issue persists, you can contact the Infisical team at{" "} + support@infisical.com. + +
+ + ); +}; + +export default HealthAlertTemplate; + +HealthAlertTemplate.PreviewProps = { + type: "gateway", + names: '"gateway1", "gateway2"', + siteUrl: "https://infisical.com" +} as HealthAlertTemplateProps; diff --git a/backend/src/services/smtp/emails/index.ts b/backend/src/services/smtp/emails/index.ts index d7204085c..06ac31ab6 100644 --- a/backend/src/services/smtp/emails/index.ts +++ b/backend/src/services/smtp/emails/index.ts @@ -6,6 +6,7 @@ export * from "./EmailVerificationTemplate"; export * from "./ExternalImportFailedTemplate"; export * from "./ExternalImportStartedTemplate"; export * from "./ExternalImportSucceededTemplate"; +export * from "./HealthAlertTemplate"; export * from "./IntegrationSyncFailedTemplate"; export * from "./NewDeviceLoginTemplate"; export * from "./OAuthPasswordResetTemplate"; diff --git a/backend/src/services/smtp/smtp-service.ts b/backend/src/services/smtp/smtp-service.ts index 63645cf67..652f56567 100644 --- a/backend/src/services/smtp/smtp-service.ts +++ b/backend/src/services/smtp/smtp-service.ts @@ -15,6 +15,7 @@ import { ExternalImportFailedTemplate, ExternalImportStartedTemplate, ExternalImportSucceededTemplate, + HealthAlertTemplate, IntegrationSyncFailedTemplate, NewDeviceLoginTemplate, OAuthPasswordResetTemplate, @@ -85,7 +86,8 @@ export enum SmtpTemplates { ServiceTokenExpired = "serviceTokenExpired", SecretScanningV2ScanFailed = "secretScanningV2ScanFailed", SecretScanningV2SecretsDetected = "secretScanningV2SecretsDetected", - AccountDeletionConfirmation = "accountDeletionConfirmation" + AccountDeletionConfirmation = "accountDeletionConfirmation", + HealthAlert = "healthAlert" } export enum SmtpHost { @@ -131,7 +133,8 @@ const EmailTemplateMap: Record> = { [SmtpTemplates.PkiExpirationAlert]: PkiExpirationAlertTemplate, [SmtpTemplates.SecretScanningV2ScanFailed]: SecretScanningScanFailedTemplate, [SmtpTemplates.SecretScanningV2SecretsDetected]: SecretScanningSecretsDetectedTemplate, - [SmtpTemplates.AccountDeletionConfirmation]: AccountDeletionConfirmationTemplate + [SmtpTemplates.AccountDeletionConfirmation]: AccountDeletionConfirmationTemplate, + [SmtpTemplates.HealthAlert]: HealthAlertTemplate }; export const smtpServiceFactory = (cfg: TSmtpConfig) => { From e752391dbfe53aacd3132bd399efd1c654f2ef2e Mon Sep 17 00:00:00 2001 From: x032205 Date: Wed, 8 Oct 2025 23:58:38 -0400 Subject: [PATCH 3/6] greptile review fixes --- backend/src/ee/routes/v1/relay-router.ts | 26 +++++++++++++------ .../ee/services/gateway-v2/gateway-v2-dal.ts | 3 +-- .../services/gateway-v2/gateway-v2-service.ts | 4 +-- backend/src/ee/services/relay/relay-dal.ts | 3 +-- .../src/ee/services/relay/relay-service.ts | 6 ++--- 5 files changed, 24 insertions(+), 18 deletions(-) diff --git a/backend/src/ee/routes/v1/relay-router.ts b/backend/src/ee/routes/v1/relay-router.ts index 5db5c0323..766025c21 100644 --- a/backend/src/ee/routes/v1/relay-router.ts +++ b/backend/src/ee/routes/v1/relay-router.ts @@ -166,14 +166,24 @@ export const registerRelayRouter = async (server: FastifyZodProvider) => { onRequest: (req, _, next) => { const authHeader = req.headers.authorization; - if (appCfg.RELAY_AUTH_SECRET && authHeader) { - const expectedHeader = `Bearer ${appCfg.RELAY_AUTH_SECRET}`; - if ( - authHeader.length === expectedHeader.length && - crypto.nativeCrypto.timingSafeEqual(Buffer.from(authHeader), Buffer.from(expectedHeader)) - ) { - return next(); - } + if (!appCfg.RELAY_AUTH_SECRET) { + throw new UnauthorizedError({ + message: "Relay authentication not configured" + }); + } + + if (!authHeader) { + throw new UnauthorizedError({ + message: "Missing authorization header" + }); + } + + const expectedHeader = `Bearer ${appCfg.RELAY_AUTH_SECRET}`; + if ( + authHeader.length === expectedHeader.length && + crypto.nativeCrypto.timingSafeEqual(Buffer.from(authHeader), Buffer.from(expectedHeader)) + ) { + return next(); } throw new UnauthorizedError({ diff --git a/backend/src/ee/services/gateway-v2/gateway-v2-dal.ts b/backend/src/ee/services/gateway-v2/gateway-v2-dal.ts index 36feb3809..4bb4f58ba 100644 --- a/backend/src/ee/services/gateway-v2/gateway-v2-dal.ts +++ b/backend/src/ee/services/gateway-v2/gateway-v2-dal.ts @@ -30,8 +30,7 @@ export const gatewayV2DalFactory = (db: TDbClient) => { .select(db.ref("name").withSchema(TableName.Identity).as("identityName")); if (isHeartbeatStale) { - const oneHourAgo = new Date(); - oneHourAgo.setHours(oneHourAgo.getHours() - 1); + const oneHourAgo = new Date(Date.now() - 60 * 60 * 1000); void query.where(`${TableName.GatewayV2}.heartbeat`, "<", oneHourAgo); void query.where((v) => { void v diff --git a/backend/src/ee/services/gateway-v2/gateway-v2-service.ts b/backend/src/ee/services/gateway-v2/gateway-v2-service.ts index 128f9a651..78a62bcb8 100644 --- a/backend/src/ee/services/gateway-v2/gateway-v2-service.ts +++ b/backend/src/ee/services/gateway-v2/gateway-v2-service.ts @@ -902,8 +902,6 @@ export const gatewayV2ServiceFactory = ({ "Found gateways with last heartbeat over an hour ago. Sending notifications." ); - await Promise.all(unhealthyGateways.map((gw) => gatewayV2DAL.updateById(gw.id, { healthAlertedAt: new Date() }))); - const gatewaysByOrg = unhealthyGateways.reduce>((acc, gw) => { if (!acc[gw.orgId]) { acc[gw.orgId] = []; @@ -944,6 +942,8 @@ export const gatewayV2ServiceFactory = ({ }, template: SmtpTemplates.HealthAlert }); + + await Promise.all(gateways.map((gw) => gatewayV2DAL.updateById(gw.id, { healthAlertedAt: new Date() }))); } catch (error) { logger.error(error, `Failed to send gateway health notifications for organization [orgId=${orgId}]`); } diff --git a/backend/src/ee/services/relay/relay-dal.ts b/backend/src/ee/services/relay/relay-dal.ts index cef5e643a..687b2d0e8 100644 --- a/backend/src/ee/services/relay/relay-dal.ts +++ b/backend/src/ee/services/relay/relay-dal.ts @@ -20,8 +20,7 @@ export const relayDalFactory = (db: TDbClient) => { .where(buildFindFilter(regularFilter, TableName.Relay)); if (isHeartbeatStale) { - const oneHourAgo = new Date(); - oneHourAgo.setHours(oneHourAgo.getHours() - 1); + const oneHourAgo = new Date(Date.now() - 60 * 60 * 1000); void query.where(`${TableName.Relay}.heartbeat`, "<", oneHourAgo); void query.where((v) => { void v diff --git a/backend/src/ee/services/relay/relay-service.ts b/backend/src/ee/services/relay/relay-service.ts index 661a3d304..6b56908f5 100644 --- a/backend/src/ee/services/relay/relay-service.ts +++ b/backend/src/ee/services/relay/relay-service.ts @@ -1209,8 +1209,6 @@ export const relayServiceFactory = ({ }; const $healthcheckNotify = async () => { - const oneHourAgo = new Date(); - oneHourAgo.setHours(oneHourAgo.getHours() - 1); const unhealthyRelays = await relayDAL.find({ isHeartbeatStale: true }); @@ -1222,8 +1220,6 @@ export const relayServiceFactory = ({ "Found relays with last heartbeat over an hour ago. Sending notifications." ); - await Promise.all(unhealthyRelays.map((r) => relayDAL.updateById(r.id, { healthAlertedAt: new Date() }))); - const relaysByOrg = unhealthyRelays.reduce>((acc, r) => { const key = r.orgId ?? "instance"; if (!acc[key]) { @@ -1285,6 +1281,8 @@ export const relayServiceFactory = ({ template: SmtpTemplates.HealthAlert }); } + + await Promise.all(relays.map((r) => relayDAL.updateById(r.id, { healthAlertedAt: new Date() }))); } catch (error) { logger.error(error, `Failed to send relay health notifications for organization [orgId=${orgId}]`); } From 2f01d096c6eba6c61d64ddf425ccffbc802f3ba1 Mon Sep 17 00:00:00 2001 From: x032205 Date: Fri, 10 Oct 2025 14:37:52 -0400 Subject: [PATCH 4/6] review fixes --- .../ee/services/gateway-v2/gateway-v2-dal.ts | 1 + .../services/gateway-v2/gateway-v2-service.ts | 9 ++------ backend/src/ee/services/relay/relay-dal.ts | 1 + .../src/ee/services/relay/relay-service.ts | 12 +++-------- .../smtp/emails/HealthAlertTemplate.tsx | 12 ++++++++--- .../components/GatewayTab/GatewayTab.tsx | 2 +- .../components/RelayTab/RelayTab.tsx | 21 +++++++++++++++++-- 7 files changed, 36 insertions(+), 22 deletions(-) diff --git a/backend/src/ee/services/gateway-v2/gateway-v2-dal.ts b/backend/src/ee/services/gateway-v2/gateway-v2-dal.ts index 4bb4f58ba..e19dbe394 100644 --- a/backend/src/ee/services/gateway-v2/gateway-v2-dal.ts +++ b/backend/src/ee/services/gateway-v2/gateway-v2-dal.ts @@ -31,6 +31,7 @@ export const gatewayV2DalFactory = (db: TDbClient) => { if (isHeartbeatStale) { const oneHourAgo = new Date(Date.now() - 60 * 60 * 1000); + void query.whereNotNull(`${TableName.GatewayV2}.heartbeat`); void query.where(`${TableName.GatewayV2}.heartbeat`, "<", oneHourAgo); void query.where((v) => { void v diff --git a/backend/src/ee/services/gateway-v2/gateway-v2-service.ts b/backend/src/ee/services/gateway-v2/gateway-v2-service.ts index 78a62bcb8..dce5bc299 100644 --- a/backend/src/ee/services/gateway-v2/gateway-v2-service.ts +++ b/backend/src/ee/services/gateway-v2/gateway-v2-service.ts @@ -9,6 +9,7 @@ import { PgSqlLock } from "@app/keystore/keystore"; import { crypto } from "@app/lib/crypto"; import { DatabaseErrorCode } from "@app/lib/error-codes"; import { BadRequestError, DatabaseError, NotFoundError } from "@app/lib/errors"; +import { groupBy } from "@app/lib/fn"; import { GatewayProxyProtocol } from "@app/lib/gateway/types"; import { withGatewayV2Proxy } from "@app/lib/gateway-v2/gateway-v2"; import { logger } from "@app/lib/logger"; @@ -902,13 +903,7 @@ export const gatewayV2ServiceFactory = ({ "Found gateways with last heartbeat over an hour ago. Sending notifications." ); - const gatewaysByOrg = unhealthyGateways.reduce>((acc, gw) => { - if (!acc[gw.orgId]) { - acc[gw.orgId] = []; - } - acc[gw.orgId].push(gw); - return acc; - }, {}); + const gatewaysByOrg = groupBy(unhealthyGateways, (gw) => gw.orgId); for await (const [orgId, gateways] of Object.entries(gatewaysByOrg)) { try { diff --git a/backend/src/ee/services/relay/relay-dal.ts b/backend/src/ee/services/relay/relay-dal.ts index 687b2d0e8..8e7eac8de 100644 --- a/backend/src/ee/services/relay/relay-dal.ts +++ b/backend/src/ee/services/relay/relay-dal.ts @@ -21,6 +21,7 @@ export const relayDalFactory = (db: TDbClient) => { if (isHeartbeatStale) { const oneHourAgo = new Date(Date.now() - 60 * 60 * 1000); + void query.whereNotNull(`${TableName.Relay}.heartbeat`); void query.where(`${TableName.Relay}.heartbeat`, "<", oneHourAgo); void query.where((v) => { void v diff --git a/backend/src/ee/services/relay/relay-service.ts b/backend/src/ee/services/relay/relay-service.ts index 6b56908f5..41fe91bfc 100644 --- a/backend/src/ee/services/relay/relay-service.ts +++ b/backend/src/ee/services/relay/relay-service.ts @@ -8,6 +8,7 @@ import { OrgMembershipRole, TRelays } from "@app/db/schemas"; import { PgSqlLock } from "@app/keystore/keystore"; import { crypto } from "@app/lib/crypto"; import { BadRequestError, ForbiddenRequestError, NotFoundError } from "@app/lib/errors"; +import { groupBy } from "@app/lib/fn"; import { createRelayConnection } from "@app/lib/gateway-v2/gateway-v2"; import { logger } from "@app/lib/logger"; import { ActorAuthMethod, ActorType } from "@app/services/auth/auth-type"; @@ -1220,14 +1221,7 @@ export const relayServiceFactory = ({ "Found relays with last heartbeat over an hour ago. Sending notifications." ); - const relaysByOrg = unhealthyRelays.reduce>((acc, r) => { - const key = r.orgId ?? "instance"; - if (!acc[key]) { - acc[key] = []; - } - acc[key].push(r); - return acc; - }, {}); + const relaysByOrg = groupBy(unhealthyRelays, (r) => r.orgId ?? "instance"); for await (const [orgId, relays] of Object.entries(relaysByOrg)) { try { @@ -1244,7 +1238,7 @@ export const relayServiceFactory = ({ recipients, subjectLine: "Relay Health Alert", substitutions: { - type: "relay", + type: "instance-relay", names: relayNames }, template: SmtpTemplates.HealthAlert diff --git a/backend/src/services/smtp/emails/HealthAlertTemplate.tsx b/backend/src/services/smtp/emails/HealthAlertTemplate.tsx index 9b9a1f591..cc90ae3cb 100644 --- a/backend/src/services/smtp/emails/HealthAlertTemplate.tsx +++ b/backend/src/services/smtp/emails/HealthAlertTemplate.tsx @@ -4,7 +4,7 @@ import { BaseEmailWrapper, BaseEmailWrapperProps } from "./BaseEmailWrapper"; import { BaseLink } from "./BaseLink"; interface HealthAlertTemplateProps extends Omit { - type: "gateway" | "relay"; + type: "gateway" | "relay" | "instance-relay"; names: string; } @@ -24,8 +24,14 @@ export const HealthAlertTemplate = ({ siteUrl, names, type }: HealthAlertTemplat heartbeat in over an hour: {names}. - If your issue persists, you can contact the Infisical team at{" "} - support@infisical.com. + {type === "instance-relay" && ( + <> + If your issue persists, you can contact the Infisical team at{" "} + support@infisical.com. + + )} + {type === "relay" && <>Please contact your relay administrators.} + {type === "gateway" && <>Please contact your gateway administrators.} diff --git a/frontend/src/pages/organization/NetworkingPage/components/GatewayTab/GatewayTab.tsx b/frontend/src/pages/organization/NetworkingPage/components/GatewayTab/GatewayTab.tsx index 152222b29..0e3069aee 100644 --- a/frontend/src/pages/organization/NetworkingPage/components/GatewayTab/GatewayTab.tsx +++ b/frontend/src/pages/organization/NetworkingPage/components/GatewayTab/GatewayTab.tsx @@ -48,7 +48,7 @@ import { useDeleteGatewayV2ById } from "@app/hooks/api/gateways-v2"; import { EditGatewayDetailsModal } from "./components/EditGatewayDetailsModal"; -export const GatewayHealthStatus = ({ heartbeat }: { heartbeat?: string }) => { +const GatewayHealthStatus = ({ heartbeat }: { heartbeat?: string }) => { const heartbeatDate = heartbeat ? new Date(heartbeat) : null; const now = new Date(); const oneHourAgo = new Date(now.getTime() - 60 * 60 * 1000); diff --git a/frontend/src/pages/organization/NetworkingPage/components/RelayTab/RelayTab.tsx b/frontend/src/pages/organization/NetworkingPage/components/RelayTab/RelayTab.tsx index 51accf6b6..24992a58e 100644 --- a/frontend/src/pages/organization/NetworkingPage/components/RelayTab/RelayTab.tsx +++ b/frontend/src/pages/organization/NetworkingPage/components/RelayTab/RelayTab.tsx @@ -42,7 +42,24 @@ import { withPermission } from "@app/hoc"; import { usePopUp } from "@app/hooks"; import { useDeleteRelayById, useGetRelays } from "@app/hooks/api/relays"; -import { GatewayHealthStatus } from "../GatewayTab/GatewayTab"; +const RelayHealthStatus = ({ heartbeat }: { heartbeat?: string }) => { + const heartbeatDate = heartbeat ? new Date(heartbeat) : null; + const now = new Date(); + const oneHourAgo = new Date(now.getTime() - 60 * 60 * 1000); + + const isHealthy = heartbeatDate && heartbeatDate >= oneHourAgo; + const tooltipContent = heartbeatDate + ? `Last heartbeat: ${heartbeatDate.toLocaleString()}` + : "No heartbeat data available"; + + return ( + + + {isHealthy ? "Healthy" : "Unreachable"} + + + ); +}; export const RelayTab = withPermission( () => { @@ -143,7 +160,7 @@ export const RelayTab = withPermission( {el.host} {formatRelative(new Date(el.createdAt), new Date())} - + From 425a883662c3d16dbb913ebeea3a3c3f219dcc22 Mon Sep 17 00:00:00 2001 From: x032205 Date: Fri, 10 Oct 2025 14:50:16 -0400 Subject: [PATCH 5/6] small review fixes --- backend/src/ee/services/gateway-v2/gateway-v2-dal.ts | 1 - backend/src/services/smtp/emails/HealthAlertTemplate.tsx | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/backend/src/ee/services/gateway-v2/gateway-v2-dal.ts b/backend/src/ee/services/gateway-v2/gateway-v2-dal.ts index 36dd45b17..1896192cd 100644 --- a/backend/src/ee/services/gateway-v2/gateway-v2-dal.ts +++ b/backend/src/ee/services/gateway-v2/gateway-v2-dal.ts @@ -26,7 +26,6 @@ export const gatewayV2DalFactory = (db: TDbClient) => { if (isHeartbeatStale) { const oneHourAgo = new Date(Date.now() - 60 * 60 * 1000); - void query.whereNotNull(`${TableName.GatewayV2}.heartbeat`); void query.where(`${TableName.GatewayV2}.heartbeat`, "<", oneHourAgo); void query.where((v) => { void v diff --git a/backend/src/services/smtp/emails/HealthAlertTemplate.tsx b/backend/src/services/smtp/emails/HealthAlertTemplate.tsx index cc90ae3cb..46b919f11 100644 --- a/backend/src/services/smtp/emails/HealthAlertTemplate.tsx +++ b/backend/src/services/smtp/emails/HealthAlertTemplate.tsx @@ -26,7 +26,7 @@ export const HealthAlertTemplate = ({ siteUrl, names, type }: HealthAlertTemplat {type === "instance-relay" && ( <> - If your issue persists, you can contact the Infisical team at{" "} + If the issue persists, you can contact the Infisical team at{" "} support@infisical.com. )} From bfbd1d1f914e3d8a3b9417e9fd7ec51c3ec153aa Mon Sep 17 00:00:00 2001 From: x032205 Date: Fri, 10 Oct 2025 14:57:09 -0400 Subject: [PATCH 6/6] show relays as healthy if heartbeat is null --- .../NetworkingPage/components/RelayTab/RelayTab.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/pages/organization/NetworkingPage/components/RelayTab/RelayTab.tsx b/frontend/src/pages/organization/NetworkingPage/components/RelayTab/RelayTab.tsx index 46356a312..e525b765d 100644 --- a/frontend/src/pages/organization/NetworkingPage/components/RelayTab/RelayTab.tsx +++ b/frontend/src/pages/organization/NetworkingPage/components/RelayTab/RelayTab.tsx @@ -47,7 +47,7 @@ const RelayHealthStatus = ({ heartbeat }: { heartbeat?: string }) => { const now = new Date(); const oneHourAgo = new Date(now.getTime() - 60 * 60 * 1000); - const isHealthy = heartbeatDate && heartbeatDate >= oneHourAgo; + const isHealthy = !heartbeatDate || heartbeatDate >= oneHourAgo; const tooltipContent = heartbeatDate ? `Last heartbeat: ${heartbeatDate.toLocaleString()}` : "No heartbeat data available";