Merge pull request #4464 from Infisical/feat/redis-cluster-support

Redis cluster support

backend/src/@types/fastify.d.ts

@@ -1,6 +1,6 @@
 import "fastify";
 
-import { Redis } from "ioredis";
+import { Cluster, Redis } from "ioredis";
 
 import { TUsers } from "@app/db/schemas";
 import { TAccessApprovalPolicyServiceFactory } from "@app/ee/services/access-approval-policy/access-approval-policy-types";
@@ -196,7 +196,7 @@ declare module "fastify" {
   }
 
   interface FastifyInstance {
-    redis: Redis;
+    redis: Redis | Cluster;
     services: {
       login: TAuthLoginFactory;
       password: TAuthPasswordFactory;

backend/src/ee/services/event/event-bus-service.ts

@@ -1,11 +1,11 @@
-import Redis from "ioredis";
+import { Cluster, Redis } from "ioredis";
 import { z } from "zod";
 
 import { logger } from "@app/lib/logger";
 
 import { BusEventSchema, TopicName } from "./types";
 
-export const eventBusFactory = (redis: Redis) => {
+export const eventBusFactory = (redis: Redis | Cluster) => {
   const publisher = redis.duplicate();
   // Duplicate the publisher to create a subscriber.
   // This is necessary because Redis does not allow a single connection to both publish and subscribe.

backend/src/ee/services/event/event-sse-service.ts

@@ -1,6 +1,6 @@
 /* eslint-disable no-continue */
 import { subject } from "@casl/ability";
-import Redis from "ioredis";
+import { Cluster, Redis } from "ioredis";
 
 import { KeyStorePrefixes } from "@app/keystore/keystore";
 import { logger } from "@app/lib/logger";
@@ -12,7 +12,7 @@ import { BusEvent, RegisteredEvent } from "./types";
 const AUTH_REFRESH_INTERVAL = 60 * 1000;
 const HEART_BEAT_INTERVAL = 15 * 1000;
 
-export const sseServiceFactory = (bus: TEventBusService, redis: Redis) => {
+export const sseServiceFactory = (bus: TEventBusService, redis: Redis | Cluster) => {
   const clients = new Set<EventStreamClient>();
 
   const heartbeatInterval = setInterval(() => {

backend/src/ee/services/event/event-sse-stream.ts

@@ -3,7 +3,7 @@ import { Readable } from "node:stream";
 
 import { MongoAbility, PureAbility } from "@casl/ability";
 import { MongoQuery } from "@ucast/mongo2js";
-import Redis from "ioredis";
+import { Cluster, Redis } from "ioredis";
 import { nanoid } from "nanoid";
 
 import { ProjectType } from "@app/db/schemas";
@@ -65,7 +65,7 @@ export type EventStreamClient = {
   matcher: PureAbility;
 };
 
-export function createEventStreamClient(redis: Redis, options: IEventStreamClientOpts): EventStreamClient {
+export function createEventStreamClient(redis: Redis | Cluster, options: IEventStreamClientOpts): EventStreamClient {
   const rules = options.registered.map((r) => {
     const secretPath = r.conditions?.secretPath;
     const hasConditions = r.conditions?.environmentSlug || r.conditions?.secretPath;

backend/src/lib/config/env.ts

@@ -37,6 +37,8 @@ const envSchema = z
       .default("false")
       .transform((el) => el === "true"),
     REDIS_URL: zpStr(z.string().optional()),
+    REDIS_USERNAME: zpStr(z.string().optional()),
+    REDIS_PASSWORD: zpStr(z.string().optional()),
     REDIS_SENTINEL_HOSTS: zpStr(
       z
         .string()
@@ -49,6 +51,12 @@ const envSchema = z
     REDIS_SENTINEL_ENABLE_TLS: zodStrBool.optional().describe("Whether to use TLS/SSL for Redis Sentinel connection"),
     REDIS_SENTINEL_USERNAME: zpStr(z.string().optional().describe("Authentication username for Redis Sentinel")),
     REDIS_SENTINEL_PASSWORD: zpStr(z.string().optional().describe("Authentication password for Redis Sentinel")),
+    REDIS_CLUSTER_HOSTS: zpStr(
+      z
+        .string()
+        .optional()
+        .describe("Comma-separated list of Redis Cluster host:port pairs. Eg: 192.168.65.254:6379,192.168.65.254:6380")
+    ),
     HOST: zpStr(z.string().default("localhost")),
     DB_CONNECTION_URI: zpStr(z.string().describe("Postgres database connection string")).default(
       `postgresql://${process.env.DB_USER}:${process.env.DB_PASSWORD}@${process.env.DB_HOST}:${process.env.DB_PORT}/${process.env.DB_NAME}`
@@ -337,8 +345,8 @@ const envSchema = z
     "Either ENCRYPTION_KEY or ROOT_ENCRYPTION_KEY must be defined."
   )
   .refine(
-    (data) => Boolean(data.REDIS_URL) || Boolean(data.REDIS_SENTINEL_HOSTS),
+    (data) => Boolean(data.REDIS_URL) || Boolean(data.REDIS_SENTINEL_HOSTS) || Boolean(data.REDIS_CLUSTER_HOSTS),
-    "Either REDIS_URL or REDIS_SENTINEL_HOSTS must be defined."
+    "Either REDIS_URL, REDIS_SENTINEL_HOSTS or REDIS_CLUSTER_HOSTS  must be defined."
   )
   .transform((data) => ({
     ...data,
@@ -348,7 +356,7 @@ const envSchema = z
       : undefined,
     isCloud: Boolean(data.LICENSE_SERVER_KEY),
     isSmtpConfigured: Boolean(data.SMTP_HOST),
-    isRedisConfigured: Boolean(data.REDIS_URL || data.REDIS_SENTINEL_HOSTS),
+    isRedisConfigured: Boolean(data.REDIS_URL || data.REDIS_SENTINEL_HOSTS || data.REDIS_CLUSTER_HOSTS),
     isDevelopmentMode: data.NODE_ENV === "development",
     isTestMode: data.NODE_ENV === "test",
     isRotationDevelopmentMode:
@@ -363,6 +371,12 @@ const envSchema = z
         const [host, port] = el.trim().split(":");
         return { host: host.trim(), port: Number(port.trim()) };
       }),
+    REDIS_CLUSTER_HOSTS: data.REDIS_CLUSTER_HOSTS?.trim()
+      ?.split(",")
+      .map((el) => {
+        const [host, port] = el.trim().split(":");
+        return { host: host.trim(), port: Number(port.trim()) };
+      }),
     isSecretScanningConfigured:
       Boolean(data.SECRET_SCANNING_GIT_APP_ID) &&
       Boolean(data.SECRET_SCANNING_PRIVATE_KEY) &&

backend/src/lib/config/redis.ts

@@ -2,6 +2,11 @@ import { Redis } from "ioredis";
 
 export type TRedisConfigKeys = Partial<{
   REDIS_URL: string;
+  REDIS_USERNAME: string;
+  REDIS_PASSWORD: string;
+
+  REDIS_CLUSTER_HOSTS: { host: string; port: number }[];
+
   REDIS_SENTINEL_HOSTS: { host: string; port: number }[];
   REDIS_SENTINEL_MASTER_NAME: string;
   REDIS_SENTINEL_ENABLE_TLS: boolean;
@@ -12,6 +17,15 @@ export type TRedisConfigKeys = Partial<{
 export const buildRedisFromConfig = (cfg: TRedisConfigKeys) => {
   if (cfg.REDIS_URL) return new Redis(cfg.REDIS_URL, { maxRetriesPerRequest: null });
 
+  if (cfg.REDIS_CLUSTER_HOSTS) {
+    return new Redis.Cluster(cfg.REDIS_CLUSTER_HOSTS, {
+      redisOptions: {
+        username: cfg.REDIS_USERNAME,
+        password: cfg.REDIS_PASSWORD
+      }
+    });
+  }
+
   return new Redis({
     // refine at tope will catch this case
     sentinels: cfg.REDIS_SENTINEL_HOSTS!,
@@ -19,6 +33,8 @@ export const buildRedisFromConfig = (cfg: TRedisConfigKeys) => {
     maxRetriesPerRequest: null,
     sentinelUsername: cfg.REDIS_SENTINEL_USERNAME,
     sentinelPassword: cfg.REDIS_SENTINEL_PASSWORD,
-    enableTLSForSentinelMode: cfg.REDIS_SENTINEL_ENABLE_TLS
+    enableTLSForSentinelMode: cfg.REDIS_SENTINEL_ENABLE_TLS,
+    username: cfg.REDIS_USERNAME,
+    password: cfg.REDIS_PASSWORD
   });
 };

backend/src/queue/queue-service.ts

@@ -415,6 +415,7 @@ export const queueServiceFactory = (
   redisCfg: TRedisConfigKeys,
   { dbConnectionUrl, dbRootCert }: { dbConnectionUrl: string; dbRootCert?: string }
 ): TQueueServiceFactory => {
+  const isClusterMode = Boolean(redisCfg?.REDIS_CLUSTER_HOSTS);
   const connection = buildRedisFromConfig(redisCfg);
   const queueContainer = {} as Record<
     QueueName,
@@ -457,6 +458,8 @@ export const queueServiceFactory = (
     }
 
     queueContainer[name] = new Queue(name as string, {
+      // ref: docs.bullmq.io/bull/patterns/redis-cluster
+      prefix: isClusterMode ? `{${name}}` : undefined,
       ...queueSettings,
       ...(crypto.isFipsModeEnabled()
         ? {
@@ -472,6 +475,7 @@ export const queueServiceFactory = (
     const appCfg = getConfig();
     if (appCfg.QUEUE_WORKERS_ENABLED && isQueueEnabled(name)) {
       workerContainer[name] = new Worker(name, jobFn, {
+        prefix: isClusterMode ? `{${name}}` : undefined,
         ...queueSettings,
         ...(crypto.isFipsModeEnabled()
           ? {

backend/src/server/app.ts

@@ -12,7 +12,7 @@ import type { FastifyRateLimitOptions } from "@fastify/rate-limit";
 import ratelimiter from "@fastify/rate-limit";
 import { fastifyRequestContext } from "@fastify/request-context";
 import fastify from "fastify";
-import { Redis } from "ioredis";
+import { Cluster, Redis } from "ioredis";
 import { Knex } from "knex";
 
 import { HsmModule } from "@app/ee/services/hsm/hsm-types";
@@ -43,7 +43,7 @@ type TMain = {
   queue: TQueueServiceFactory;
   keyStore: TKeyStoreFactory;
   hsmModule: HsmModule;
-  redis: Redis;
+  redis: Redis | Cluster;
   envConfig: TEnvConfig;
   superAdminDAL: TSuperAdminDALFactory;
 };
@@ -76,6 +76,7 @@ export const main = async ({
   server.setValidatorCompiler(validatorCompiler);
   server.setSerializerCompiler(serializerCompiler);
 
+  // @ts-expect-error akhilmhdh: even on setting it fastify as Redis | Cluster it's throwing error
   server.decorate("redis", redis);
   server.addContentTypeParser("application/scim+json", { parseAs: "string" }, (_, body, done) => {
     try {

docs/self-hosting/configuration/envars.mdx

@@ -142,7 +142,7 @@ DB_READ_REPLICAS=[{"DB_CONNECTION_URI":""}]
 Redis is used for caching and background tasks. You can use either a standalone Redis instance or a Redis Sentinel setup.
 
 <Tabs>
-  <Tab title="Redis standalone">
+  <Tab title="Redis Standalone">
     <ParamField query="REDIS_URL" type="string" default="none" required>
       Redis connection string.
     </ParamField>
@@ -173,6 +173,29 @@ Redis is used for caching and background tasks. You can use either a standalone
     <ParamField query="REDIS_SENTINEL_PASSWORD" type="string" default="none">
       Authentication password for Redis Sentinel
     </ParamField>
+    <ParamField query="REDIS_USERNAME" type="string" default="none">
+      Authentication username for Redis Node
+    </ParamField>
+    <ParamField query="REDIS_PASSWORD" type="string" default="none">
+      Authentication password for Redis Node
+    </ParamField>
+  </Tab>
+  <Tab title="Redis Cluster">
+    <ParamField
+      query="REDIS_CLUSTER_HOSTS"
+      type="string"
+      default="none"
+      required
+    >
+      Comma-separated list of Redis Cluster host:port pairs. ```
+      192.168.65.254:26379,192.168.65.254:26380 ```
+    </ParamField>
+    <ParamField query="REDIS_USERNAME" type="string" default="none">
+      Authentication username for Redis Node
+    </ParamField>
+    <ParamField query="REDIS_PASSWORD" type="string" default="none">
+      Authentication password for Redis Node
+    </ParamField>
   </Tab>
 </Tabs>
 

sink/redis-cluster/README.md

@@ -0,0 +1,42 @@
+# Redis Cluster Setup
+
+## Quick Start
+
+1. **Update IP Address**: Replace `192.168.1.33` with your system's IP address in `docker-compose.yml`:
+   ```bash
+   # Find your IP
+   ifconfig | grep "inet " | grep -v 127.0.0.1 | awk '{print $2}' | head -1
+   ```
+
+2. **Start Cluster**:
+   ```bash
+   docker compose up -d
+   ```
+
+3. **Verify Cluster**:
+   ```bash
+   docker exec redis-node-1 redis-cli -p 7001 cluster info
+   ```
+
+## Connection Details
+
+- **Redis Cluster**: `YOUR_IP:7001`, `YOUR_IP:7002`, `YOUR_IP:7003`
+- **RedisInsight UI**: `localhost:5540`
+
+## Clean Restart
+
+To completely reset the cluster:
+```bash
+docker compose down -v
+# Update IP in docker-compose.yml if needed
+docker compose up -d
+```
+
+## External Docker Compose Usage
+
+```yaml
+environment:
+  - REDIS_CLUSTER_URLS=redis://YOUR_IP:7001,redis://YOUR_IP:7002,redis://YOUR_IP:7003
+```
+
+**Important**: Always replace `YOUR_IP` with your actual system IP address.

sink/redis-cluster/docker-compose.yml

@@ -0,0 +1,87 @@
+version: "3.8"
+
+services:
+  redis-node-1:
+    image: redis:7
+    container_name: redis-node-1
+    ports:
+      - "7001:7001"
+      - "17001:17001"
+    volumes:
+      - redis-node-1-data:/data
+    command: >
+      redis-server
+      --port 7001
+      --cluster-enabled yes
+      --cluster-config-file nodes.conf
+      --cluster-node-timeout 5000
+      --appendonly yes
+      --bind 0.0.0.0
+      --cluster-announce-ip 192.168.1.33
+      --cluster-announce-port 7001
+      --cluster-announce-bus-port 17001
+
+  redis-node-2:
+    image: redis:7
+    container_name: redis-node-2
+    ports:
+      - "7002:7002"
+      - "17002:17002"
+    volumes:
+      - redis-node-2-data:/data
+    command: >
+      redis-server
+      --port 7002
+      --cluster-enabled yes
+      --cluster-config-file nodes.conf
+      --cluster-node-timeout 5000
+      --appendonly yes
+      --bind 0.0.0.0
+      --cluster-announce-ip 192.168.1.33
+      --cluster-announce-port 7002
+      --cluster-announce-bus-port 17002
+
+  redis-node-3:
+    image: redis:7
+    container_name: redis-node-3
+    ports:
+      - "7003:7003"
+      - "17003:17003"
+    volumes:
+      - redis-node-3-data:/data
+    command: >
+      redis-server
+      --port 7003
+      --cluster-enabled yes
+      --cluster-config-file nodes.conf
+      --cluster-node-timeout 5000
+      --appendonly yes
+      --bind 0.0.0.0
+      --cluster-announce-ip 192.168.1.33
+      --cluster-announce-port 7003
+      --cluster-announce-bus-port 17003
+
+  redis-insight:
+    container_name: redis-insight
+    image: redis/redisinsight
+    ports:
+      - "5540:5540"
+
+  redis-cluster-init:
+    image: redis:7
+    depends_on:
+      - redis-node-1
+      - redis-node-2
+      - redis-node-3
+    restart: "no"
+    command: >
+      sh -c "
+        sleep 10
+        redis-cli --cluster create 192.168.1.33:7001 192.168.1.33:7002 192.168.1.33:7003 --cluster-yes
+        echo 'Cluster initialized'
+      "
+
+volumes:
+  redis-node-1-data:
+  redis-node-2-data:
+  redis-node-3-data: