update relay health check

This commit is contained in:
Maidul Islam
2025-03-05 23:50:11 -05:00
parent dcd5f20325
commit fc4e3f1f72

View File

@@ -176,7 +176,7 @@ func (g *Gateway) Listen(ctx context.Context) error {
KeepAlivePeriod: 2 * time.Second, KeepAlivePeriod: 2 * time.Second,
} }
g.registerRelayIsActive(ctx, relayUdpConnection.LocalAddr().String(), errCh) g.registerRelayIsActive(ctx, errCh)
quicListener, err := quic.Listen(relayUdpConnection, tlsConfig, quicConfig) quicListener, err := quic.Listen(relayUdpConnection, tlsConfig, quicConfig)
if err != nil { if err != nil {
return fmt.Errorf("Failed to listen for QUIC: %w", err) return fmt.Errorf("Failed to listen for QUIC: %w", err)
@@ -320,39 +320,49 @@ func (g *Gateway) createPermissionForStaticIps(staticIps string) error {
return nil return nil
} }
func (g *Gateway) registerRelayIsActive(ctx context.Context, relayAddress string, errCh chan error) error { func (g *Gateway) registerRelayIsActive(ctx context.Context, errCh chan error) error {
ticker := time.NewTicker(10 * time.Second) ticker := time.NewTicker(15 * time.Second)
maxFailures := 3 maxFailures := 3
failures := 0 failures := 0
log.Info().Msg("Starting relay connection health check")
go func() { go func() {
time.Sleep(2 * time.Second) time.Sleep(5 * time.Second)
for { for {
select { select {
case <-ctx.Done(): case <-ctx.Done():
log.Info().Msg("Stopping relay connection health check")
return return
case <-ticker.C: case <-ticker.C:
// Configure TLS to skip verification
tlsConfig := &tls.Config{
InsecureSkipVerify: true,
NextProtos: []string{"infisical-gateway"},
}
quicConfig := &quic.Config{
EnableDatagrams: true,
}
func() { func() {
checkCtx, cancel := context.WithTimeout(ctx, 3*time.Second) log.Debug().Msg("Performing relay connection health check")
defer cancel()
conn, err := quic.DialAddr(checkCtx, relayAddress, tlsConfig, quicConfig) if g.client == nil {
if err != nil {
failures++ failures++
log.Warn().Err(err).Int("failures", failures).Msg("Relay connection check failed") log.Warn().Int("failures", failures).Msg("TURN client is nil")
if failures >= maxFailures {
errCh <- fmt.Errorf("relay connection check failed: TURN client is nil")
}
return
}
// we try to refresh permissions - this is a lightweight operation
// that will fail immediately if the UDP connection is broken. good for health check
log.Debug().Msg("Refreshing TURN permissions to verify connection")
if err := g.createPermissionForStaticIps(g.config.InfisicalStaticIp); err != nil {
failures++
log.Warn().Err(err).Int("failures", failures).Msg("Failed to refresh TURN permissions")
if failures >= maxFailures { if failures >= maxFailures {
errCh <- fmt.Errorf("relay connection check failed: %w", err) errCh <- fmt.Errorf("relay connection check failed: %w", err)
} }
return
} }
if conn != nil {
conn.CloseWithError(0, "closed") log.Debug().Msg("Successfully refreshed TURN permissions - connection is healthy")
if failures > 0 {
log.Info().Int("previous_failures", failures).Msg("Relay connection restored")
failures = 0
} }
}() }()
} }