feat: added sentinel suppor for backend

backend/e2e-test/vitest-environment-knex.ts

@@ -55,8 +55,8 @@ export default {
       });
 
       const smtp = mockSmtpServer();
-      const queue = queueServiceFactory(envConfig.REDIS_URL, { dbConnectionUrl: envConfig.DB_CONNECTION_URI });
+      const queue = queueServiceFactory(envConfig, { dbConnectionUrl: envConfig.DB_CONNECTION_URI });
-      const keyStore = keyStoreFactory(envConfig.REDIS_URL);
+      const keyStore = keyStoreFactory(envConfig);
 
       const hsmModule = initializeHsmModule(envConfig);
       hsmModule.initialize();

backend/src/keystore/keystore.ts

@@ -1,5 +1,4 @@
-import { Redis } from "ioredis";
+import { buildRedisFromConfig, TRedisConfigKeys } from "@app/lib/config/redis";
-
 import { pgAdvisoryLockHashText } from "@app/lib/crypto/hashtext";
 import { applyJitter } from "@app/lib/dates";
 import { delay as delayMs } from "@app/lib/delay";
@@ -66,8 +65,8 @@ type TWaitTillReady = {
   jitter?: number;
 };
 
-export const keyStoreFactory = (redisUrl: string) => {
+export const keyStoreFactory = (redisConfigKeys: TRedisConfigKeys) => {
-  const redis = new Redis(redisUrl);
+  const redis = buildRedisFromConfig(redisConfigKeys);
   const redisLock = new Redlock([redis], { retryCount: 2, retryDelay: 200 });
 
   const setItem = async (key: string, value: string | number | Buffer, prefix?: string) =>

backend/src/lib/config/env.ts

@@ -30,7 +30,19 @@ const envSchema = z
       .enum(["true", "false"])
       .default("false")
       .transform((el) => el === "true"),
-    REDIS_URL: zpStr(z.string()),
+    REDIS_URL: zpStr(z.string().optional()),
+    REDIS_SENTINEL_HOSTS: zpStr(
+      z
+        .string()
+        .optional()
+        .describe("Comma-separated list of Sentinel host:port pairs. Eg: 192.168.65.254:26379,192.168.65.254:26380")
+    ),
+    REDIS_SENTINEL_MASTER_NAME: zpStr(
+      z.string().optional().default("mymaster").describe("The name of the Redis master set monitored by Sentinel")
+    ),
+    REDIS_SENTINEL_ENABLE_TLS: zodStrBool.optional().describe(" Whether to use TLS/SSL for Redis Sentinel connection"),
+    REDIS_SENTINEL_USERNAME: zpStr(z.string().optional().describe("Authentication username for Redis Sentinel")),
+    REDIS_SENTINEL_PASSWORD: zpStr(z.string().optional().describe("Authentication password for Redis Sentinel")),
     HOST: zpStr(z.string().default("localhost")),
     DB_CONNECTION_URI: zpStr(z.string().describe("Postgres database connection string")).default(
       `postgresql://${process.env.DB_USER}:${process.env.DB_PASSWORD}@${process.env.DB_HOST}:${process.env.DB_PORT}/${process.env.DB_NAME}`
@@ -256,26 +268,32 @@ const envSchema = z
     (data) => Boolean(data.ENCRYPTION_KEY) || Boolean(data.ROOT_ENCRYPTION_KEY),
     "Either ENCRYPTION_KEY or ROOT_ENCRYPTION_KEY must be defined."
   )
+  .refine(
+    (data) => Boolean(data.REDIS_URL) || Boolean(data.REDIS_SENTINEL_HOSTS),
+    "Either REDIS_URL or REDIS_SENTINEL_HOSTS must be defined."
+  )
   .transform((data) => ({
     ...data,
-
     DB_READ_REPLICAS: data.DB_READ_REPLICAS
       ? databaseReadReplicaSchema.parse(JSON.parse(data.DB_READ_REPLICAS))
       : undefined,
     isCloud: Boolean(data.LICENSE_SERVER_KEY),
     isSmtpConfigured: Boolean(data.SMTP_HOST),
-    isRedisConfigured: Boolean(data.REDIS_URL),
+    isRedisConfigured: Boolean(data.REDIS_URL || data.REDIS_SENTINEL_HOSTS),
     isDevelopmentMode: data.NODE_ENV === "development",
     isRotationDevelopmentMode: data.NODE_ENV === "development" && data.ROTATION_DEVELOPMENT_MODE,
     isProductionMode: data.NODE_ENV === "production" || IS_PACKAGED,
-
+    isRedisSentinelMode: Boolean(data.REDIS_SENTINEL_HOSTS),
+    REDIS_SENTINEL_HOSTS: data.REDIS_SENTINEL_HOSTS?.split(",").map((el) => {
+      const [host, port] = el.split(":");
+      return { host: host.trim(), port: Number(port.trim()) };
+    }),
     isSecretScanningConfigured:
       Boolean(data.SECRET_SCANNING_GIT_APP_ID) &&
       Boolean(data.SECRET_SCANNING_PRIVATE_KEY) &&
       Boolean(data.SECRET_SCANNING_WEBHOOK_SECRET),
     isHsmConfigured:
       Boolean(data.HSM_LIB_PATH) && Boolean(data.HSM_PIN) && Boolean(data.HSM_KEY_LABEL) && data.HSM_SLOT !== undefined,
-
     samlDefaultOrgSlug: data.DEFAULT_SAML_ORG_SLUG,
     SECRET_SCANNING_ORG_WHITELIST: data.SECRET_SCANNING_ORG_WHITELIST?.split(",")
   }));

backend/src/lib/config/redis.ts

@@ -0,0 +1,24 @@
+import { Redis } from "ioredis";
+
+export type TRedisConfigKeys = Partial<{
+  REDIS_URL: string;
+  REDIS_SENTINEL_HOSTS: { host: string; port: number }[];
+  REDIS_SENTINEL_MASTER_NAME: string;
+  REDIS_SENTINEL_ENABLE_TLS: boolean;
+  REDIS_SENTINEL_USERNAME: string;
+  REDIS_SENTINEL_PASSWORD: string;
+}>;
+
+export const buildRedisFromConfig = (cfg: TRedisConfigKeys) => {
+  if (cfg.REDIS_URL) return new Redis(cfg.REDIS_URL, { maxRetriesPerRequest: null });
+
+  return new Redis({
+    // refine at tope will catch this case
+    sentinels: cfg.REDIS_SENTINEL_HOSTS!,
+    name: cfg.REDIS_SENTINEL_MASTER_NAME!,
+    maxRetriesPerRequest: null,
+    sentinelUsername: cfg.REDIS_SENTINEL_USERNAME,
+    sentinelPassword: cfg.REDIS_SENTINEL_PASSWORD,
+    enableTLSForSentinelMode: cfg.REDIS_SENTINEL_ENABLE_TLS
+  });
+};

backend/src/main.ts

@@ -1,7 +1,6 @@
 import "./lib/telemetry/instrumentation";
 
 import dotenv from "dotenv";
-import { Redis } from "ioredis";
 
 import { initializeHsmModule } from "@app/ee/services/hsm/hsm-fns";
 
@@ -9,6 +8,7 @@ import { runMigrations } from "./auto-start-migrations";
 import { initAuditLogDbConnection, initDbConnection } from "./db";
 import { keyStoreFactory } from "./keystore/keystore";
 import { formatSmtpConfig, initEnvConfig } from "./lib/config/env";
+import { buildRedisFromConfig } from "./lib/config/redis";
 import { removeTemporaryBaseDirectory } from "./lib/files";
 import { initLogger } from "./lib/logger";
 import { queueServiceFactory } from "./queue";
@@ -44,15 +44,15 @@ const run = async () => {
 
   const smtp = smtpServiceFactory(formatSmtpConfig());
 
-  const queue = queueServiceFactory(envConfig.REDIS_URL, {
+  const queue = queueServiceFactory(envConfig, {
     dbConnectionUrl: envConfig.DB_CONNECTION_URI,
     dbRootCert: envConfig.DB_ROOT_CERT
   });
 
   await queue.initialize();
 
-  const keyStore = keyStoreFactory(envConfig.REDIS_URL);
+  const keyStore = keyStoreFactory(envConfig);
-  const redis = new Redis(envConfig.REDIS_URL);
+  const redis = buildRedisFromConfig(envConfig);
 
   const hsmModule = initializeHsmModule(envConfig);
   hsmModule.initialize();

backend/src/queue/queue-service.ts

@@ -1,5 +1,4 @@
 import { Job, JobsOptions, Queue, QueueOptions, RepeatOptions, Worker, WorkerListener } from "bullmq";
-import Redis from "ioredis";
 import PgBoss, { WorkOptions } from "pg-boss";
 
 import { SecretEncryptionAlgo, SecretKeyEncoding } from "@app/db/schemas";
@@ -13,6 +12,7 @@ import {
   TScanPushEventPayload
 } from "@app/ee/services/secret-scanning/secret-scanning-queue/secret-scanning-queue-types";
 import { getConfig } from "@app/lib/config/env";
+import { buildRedisFromConfig, TRedisConfigKeys } from "@app/lib/config/redis";
 import { logger } from "@app/lib/logger";
 import {
   TFailedIntegrationSyncEmailsPayload,
@@ -249,10 +249,10 @@ export type TQueueJobTypes = {
 
 export type TQueueServiceFactory = ReturnType<typeof queueServiceFactory>;
 export const queueServiceFactory = (
-  redisUrl: string,
+  redisCfg: TRedisConfigKeys,
   { dbConnectionUrl, dbRootCert }: { dbConnectionUrl: string; dbRootCert?: string }
 ) => {
-  const connection = new Redis(redisUrl, { maxRetriesPerRequest: null });
+  const connection = buildRedisFromConfig(redisCfg);
   const queueContainer = {} as Record<
     QueueName,
     Queue<TQueueJobTypes[QueueName]["payload"], void, TQueueJobTypes[QueueName]["name"]>

backend/src/server/boot-strap-check.ts

@@ -1,9 +1,9 @@
 /* eslint-disable no-console */
-import { Redis } from "ioredis";
 import { Knex } from "knex";
 import { createTransport } from "nodemailer";
 
 import { formatSmtpConfig, getConfig } from "@app/lib/config/env";
+import { buildRedisFromConfig } from "@app/lib/config/redis";
 import { logger } from "@app/lib/logger";
 import { getServerCfg } from "@app/services/super-admin/super-admin-service";
 
@@ -65,12 +65,15 @@ export const bootstrapCheck = async ({ db }: BootstrapOpt) => {
     });
 
   console.log("Testing redis connection");
-  const redis = new Redis(appCfg.REDIS_URL);
+  const redis = buildRedisFromConfig(appCfg);
   const redisPing = await redis?.ping();
   if (!redisPing) {
     console.error("Redis - Failed to connect");
   } else {
     console.error("Redis successfully connected");
+    if (appCfg.isRedisSentinelMode) {
+      console.log("Redis Sentinel Mode");
+    }
     redis.disconnect();
   }
 

backend/src/server/config/rateLimiter.ts

@@ -1,14 +1,12 @@
 import type { RateLimitOptions, RateLimitPluginOptions } from "@fastify/rate-limit";
-import { Redis } from "ioredis";
 
 import { getConfig } from "@app/lib/config/env";
+import { buildRedisFromConfig } from "@app/lib/config/redis";
 import { RateLimitError } from "@app/lib/errors";
 
 export const globalRateLimiterCfg = (): RateLimitPluginOptions => {
   const appCfg = getConfig();
-  const redis = appCfg.isRedisConfigured
+  const redis = appCfg.isRedisConfigured ? buildRedisFromConfig(appCfg) : null;
-    ? new Redis(appCfg.REDIS_URL, { connectTimeout: 500, maxRetriesPerRequest: 1 })
-    : null;
 
   return {
     errorResponseBuilder: (_, context) => {

backend/src/services/certificate/certificate-service.ts

@@ -8,6 +8,7 @@ import {
   ProjectPermissionCertificateActions,
   ProjectPermissionSub
 } from "@app/ee/services/permission/project-permission";
+import { NotFoundError } from "@app/lib/errors";
 import { TCertificateBodyDALFactory } from "@app/services/certificate/certificate-body-dal";
 import { TCertificateDALFactory } from "@app/services/certificate/certificate-dal";
 import { TCertificateAuthorityCertDALFactory } from "@app/services/certificate-authority/certificate-authority-cert-dal";
@@ -29,7 +30,6 @@ import {
   TGetCertPrivateKeyDTO,
   TRevokeCertDTO
 } from "./certificate-types";
-import { NotFoundError } from "@app/lib/errors";
 
 type TCertificateServiceFactoryDep = {
   certificateDAL: Pick<TCertificateDALFactory, "findOne" | "deleteById" | "update" | "find">;