diff --git a/backend/src/@types/fastify.d.ts b/backend/src/@types/fastify.d.ts index 60bcf7348..4289818eb 100644 --- a/backend/src/@types/fastify.d.ts +++ b/backend/src/@types/fastify.d.ts @@ -1,6 +1,6 @@ import "fastify"; -import { Redis } from "ioredis"; +import { Cluster, Redis } from "ioredis"; import { TUsers } from "@app/db/schemas"; import { TAccessApprovalPolicyServiceFactory } from "@app/ee/services/access-approval-policy/access-approval-policy-types"; @@ -196,7 +196,7 @@ declare module "fastify" { } interface FastifyInstance { - redis: Redis; + redis: Redis | Cluster; services: { login: TAuthLoginFactory; password: TAuthPasswordFactory; diff --git a/backend/src/ee/services/event/event-bus-service.ts b/backend/src/ee/services/event/event-bus-service.ts index bb102c721..00fd14e9f 100644 --- a/backend/src/ee/services/event/event-bus-service.ts +++ b/backend/src/ee/services/event/event-bus-service.ts @@ -1,11 +1,11 @@ -import Redis from "ioredis"; +import { Cluster, Redis } from "ioredis"; import { z } from "zod"; import { logger } from "@app/lib/logger"; import { BusEventSchema, TopicName } from "./types"; -export const eventBusFactory = (redis: Redis) => { +export const eventBusFactory = (redis: Redis | Cluster) => { const publisher = redis.duplicate(); // Duplicate the publisher to create a subscriber. // This is necessary because Redis does not allow a single connection to both publish and subscribe. diff --git a/backend/src/ee/services/event/event-sse-service.ts b/backend/src/ee/services/event/event-sse-service.ts index dc52cc14c..147af8e3c 100644 --- a/backend/src/ee/services/event/event-sse-service.ts +++ b/backend/src/ee/services/event/event-sse-service.ts @@ -1,6 +1,6 @@ /* eslint-disable no-continue */ import { subject } from "@casl/ability"; -import Redis from "ioredis"; +import { Cluster, Redis } from "ioredis"; import { KeyStorePrefixes } from "@app/keystore/keystore"; import { logger } from "@app/lib/logger"; @@ -12,7 +12,7 @@ import { BusEvent, RegisteredEvent } from "./types"; const AUTH_REFRESH_INTERVAL = 60 * 1000; const HEART_BEAT_INTERVAL = 15 * 1000; -export const sseServiceFactory = (bus: TEventBusService, redis: Redis) => { +export const sseServiceFactory = (bus: TEventBusService, redis: Redis | Cluster) => { const clients = new Set(); const heartbeatInterval = setInterval(() => { diff --git a/backend/src/ee/services/event/event-sse-stream.ts b/backend/src/ee/services/event/event-sse-stream.ts index 13e18374f..9ed5e677c 100644 --- a/backend/src/ee/services/event/event-sse-stream.ts +++ b/backend/src/ee/services/event/event-sse-stream.ts @@ -3,7 +3,7 @@ import { Readable } from "node:stream"; import { MongoAbility, PureAbility } from "@casl/ability"; import { MongoQuery } from "@ucast/mongo2js"; -import Redis from "ioredis"; +import { Cluster, Redis } from "ioredis"; import { nanoid } from "nanoid"; import { ProjectType } from "@app/db/schemas"; @@ -65,7 +65,7 @@ export type EventStreamClient = { matcher: PureAbility; }; -export function createEventStreamClient(redis: Redis, options: IEventStreamClientOpts): EventStreamClient { +export function createEventStreamClient(redis: Redis | Cluster, options: IEventStreamClientOpts): EventStreamClient { const rules = options.registered.map((r) => { const secretPath = r.conditions?.secretPath; const hasConditions = r.conditions?.environmentSlug || r.conditions?.secretPath; diff --git a/backend/src/lib/config/env.ts b/backend/src/lib/config/env.ts index 7416baa59..922fdbcdf 100644 --- a/backend/src/lib/config/env.ts +++ b/backend/src/lib/config/env.ts @@ -37,6 +37,8 @@ const envSchema = z .default("false") .transform((el) => el === "true"), REDIS_URL: zpStr(z.string().optional()), + REDIS_USERNAME: zpStr(z.string().optional()), + REDIS_PASSWORD: zpStr(z.string().optional()), REDIS_SENTINEL_HOSTS: zpStr( z .string() @@ -49,6 +51,12 @@ const envSchema = z REDIS_SENTINEL_ENABLE_TLS: zodStrBool.optional().describe("Whether to use TLS/SSL for Redis Sentinel connection"), REDIS_SENTINEL_USERNAME: zpStr(z.string().optional().describe("Authentication username for Redis Sentinel")), REDIS_SENTINEL_PASSWORD: zpStr(z.string().optional().describe("Authentication password for Redis Sentinel")), + REDIS_CLUSTER_HOSTS: zpStr( + z + .string() + .optional() + .describe("Comma-separated list of Redis Cluster host:port pairs. Eg: 192.168.65.254:6379,192.168.65.254:6380") + ), HOST: zpStr(z.string().default("localhost")), DB_CONNECTION_URI: zpStr(z.string().describe("Postgres database connection string")).default( `postgresql://${process.env.DB_USER}:${process.env.DB_PASSWORD}@${process.env.DB_HOST}:${process.env.DB_PORT}/${process.env.DB_NAME}` @@ -337,8 +345,8 @@ const envSchema = z "Either ENCRYPTION_KEY or ROOT_ENCRYPTION_KEY must be defined." ) .refine( - (data) => Boolean(data.REDIS_URL) || Boolean(data.REDIS_SENTINEL_HOSTS), - "Either REDIS_URL or REDIS_SENTINEL_HOSTS must be defined." + (data) => Boolean(data.REDIS_URL) || Boolean(data.REDIS_SENTINEL_HOSTS) || Boolean(data.REDIS_CLUSTER_HOSTS), + "Either REDIS_URL, REDIS_SENTINEL_HOSTS or REDIS_CLUSTER_HOSTS must be defined." ) .transform((data) => ({ ...data, @@ -348,7 +356,7 @@ const envSchema = z : undefined, isCloud: Boolean(data.LICENSE_SERVER_KEY), isSmtpConfigured: Boolean(data.SMTP_HOST), - isRedisConfigured: Boolean(data.REDIS_URL || data.REDIS_SENTINEL_HOSTS), + isRedisConfigured: Boolean(data.REDIS_URL || data.REDIS_SENTINEL_HOSTS || data.REDIS_CLUSTER_HOSTS), isDevelopmentMode: data.NODE_ENV === "development", isTestMode: data.NODE_ENV === "test", isRotationDevelopmentMode: @@ -363,6 +371,12 @@ const envSchema = z const [host, port] = el.trim().split(":"); return { host: host.trim(), port: Number(port.trim()) }; }), + REDIS_CLUSTER_HOSTS: data.REDIS_CLUSTER_HOSTS?.trim() + ?.split(",") + .map((el) => { + const [host, port] = el.trim().split(":"); + return { host: host.trim(), port: Number(port.trim()) }; + }), isSecretScanningConfigured: Boolean(data.SECRET_SCANNING_GIT_APP_ID) && Boolean(data.SECRET_SCANNING_PRIVATE_KEY) && diff --git a/backend/src/lib/config/redis.ts b/backend/src/lib/config/redis.ts index 987518dd5..620f0f936 100644 --- a/backend/src/lib/config/redis.ts +++ b/backend/src/lib/config/redis.ts @@ -2,6 +2,11 @@ import { Redis } from "ioredis"; export type TRedisConfigKeys = Partial<{ REDIS_URL: string; + REDIS_USERNAME: string; + REDIS_PASSWORD: string; + + REDIS_CLUSTER_HOSTS: { host: string; port: number }[]; + REDIS_SENTINEL_HOSTS: { host: string; port: number }[]; REDIS_SENTINEL_MASTER_NAME: string; REDIS_SENTINEL_ENABLE_TLS: boolean; @@ -12,6 +17,15 @@ export type TRedisConfigKeys = Partial<{ export const buildRedisFromConfig = (cfg: TRedisConfigKeys) => { if (cfg.REDIS_URL) return new Redis(cfg.REDIS_URL, { maxRetriesPerRequest: null }); + if (cfg.REDIS_CLUSTER_HOSTS) { + return new Redis.Cluster(cfg.REDIS_CLUSTER_HOSTS, { + redisOptions: { + username: cfg.REDIS_USERNAME, + password: cfg.REDIS_PASSWORD + } + }); + } + return new Redis({ // refine at tope will catch this case sentinels: cfg.REDIS_SENTINEL_HOSTS!, @@ -19,6 +33,8 @@ export const buildRedisFromConfig = (cfg: TRedisConfigKeys) => { maxRetriesPerRequest: null, sentinelUsername: cfg.REDIS_SENTINEL_USERNAME, sentinelPassword: cfg.REDIS_SENTINEL_PASSWORD, - enableTLSForSentinelMode: cfg.REDIS_SENTINEL_ENABLE_TLS + enableTLSForSentinelMode: cfg.REDIS_SENTINEL_ENABLE_TLS, + username: cfg.REDIS_USERNAME, + password: cfg.REDIS_PASSWORD }); }; diff --git a/backend/src/queue/queue-service.ts b/backend/src/queue/queue-service.ts index f4c49d15a..9ca9a09d5 100644 --- a/backend/src/queue/queue-service.ts +++ b/backend/src/queue/queue-service.ts @@ -415,6 +415,7 @@ export const queueServiceFactory = ( redisCfg: TRedisConfigKeys, { dbConnectionUrl, dbRootCert }: { dbConnectionUrl: string; dbRootCert?: string } ): TQueueServiceFactory => { + const isClusterMode = Boolean(redisCfg?.REDIS_CLUSTER_HOSTS); const connection = buildRedisFromConfig(redisCfg); const queueContainer = {} as Record< QueueName, @@ -457,6 +458,8 @@ export const queueServiceFactory = ( } queueContainer[name] = new Queue(name as string, { + // ref: docs.bullmq.io/bull/patterns/redis-cluster + prefix: isClusterMode ? `{${name}}` : undefined, ...queueSettings, ...(crypto.isFipsModeEnabled() ? { @@ -472,6 +475,7 @@ export const queueServiceFactory = ( const appCfg = getConfig(); if (appCfg.QUEUE_WORKERS_ENABLED && isQueueEnabled(name)) { workerContainer[name] = new Worker(name, jobFn, { + prefix: isClusterMode ? `{${name}}` : undefined, ...queueSettings, ...(crypto.isFipsModeEnabled() ? { diff --git a/backend/src/server/app.ts b/backend/src/server/app.ts index 321a3656e..8cf23f703 100644 --- a/backend/src/server/app.ts +++ b/backend/src/server/app.ts @@ -12,7 +12,7 @@ import type { FastifyRateLimitOptions } from "@fastify/rate-limit"; import ratelimiter from "@fastify/rate-limit"; import { fastifyRequestContext } from "@fastify/request-context"; import fastify from "fastify"; -import { Redis } from "ioredis"; +import { Cluster, Redis } from "ioredis"; import { Knex } from "knex"; import { HsmModule } from "@app/ee/services/hsm/hsm-types"; @@ -43,7 +43,7 @@ type TMain = { queue: TQueueServiceFactory; keyStore: TKeyStoreFactory; hsmModule: HsmModule; - redis: Redis; + redis: Redis | Cluster; envConfig: TEnvConfig; superAdminDAL: TSuperAdminDALFactory; }; @@ -76,6 +76,7 @@ export const main = async ({ server.setValidatorCompiler(validatorCompiler); server.setSerializerCompiler(serializerCompiler); + // @ts-expect-error akhilmhdh: even on setting it fastify as Redis | Cluster it's throwing error server.decorate("redis", redis); server.addContentTypeParser("application/scim+json", { parseAs: "string" }, (_, body, done) => { try { diff --git a/docs/self-hosting/configuration/envars.mdx b/docs/self-hosting/configuration/envars.mdx index 53adf95a4..6f3f01933 100644 --- a/docs/self-hosting/configuration/envars.mdx +++ b/docs/self-hosting/configuration/envars.mdx @@ -142,7 +142,7 @@ DB_READ_REPLICAS=[{"DB_CONNECTION_URI":""}] Redis is used for caching and background tasks. You can use either a standalone Redis instance or a Redis Sentinel setup. - + Redis connection string. @@ -173,6 +173,29 @@ Redis is used for caching and background tasks. You can use either a standalone Authentication password for Redis Sentinel + + Authentication username for Redis Node + + + Authentication password for Redis Node + + + + + Comma-separated list of Redis Cluster host:port pairs. ``` + 192.168.65.254:26379,192.168.65.254:26380 ``` + + + Authentication username for Redis Node + + + Authentication password for Redis Node + diff --git a/sink/redis-cluster/README.md b/sink/redis-cluster/README.md new file mode 100644 index 000000000..6f4a40a80 --- /dev/null +++ b/sink/redis-cluster/README.md @@ -0,0 +1,42 @@ +# Redis Cluster Setup + +## Quick Start + +1. **Update IP Address**: Replace `192.168.1.33` with your system's IP address in `docker-compose.yml`: + ```bash + # Find your IP + ifconfig | grep "inet " | grep -v 127.0.0.1 | awk '{print $2}' | head -1 + ``` + +2. **Start Cluster**: + ```bash + docker compose up -d + ``` + +3. **Verify Cluster**: + ```bash + docker exec redis-node-1 redis-cli -p 7001 cluster info + ``` + +## Connection Details + +- **Redis Cluster**: `YOUR_IP:7001`, `YOUR_IP:7002`, `YOUR_IP:7003` +- **RedisInsight UI**: `localhost:5540` + +## Clean Restart + +To completely reset the cluster: +```bash +docker compose down -v +# Update IP in docker-compose.yml if needed +docker compose up -d +``` + +## External Docker Compose Usage + +```yaml +environment: + - REDIS_CLUSTER_URLS=redis://YOUR_IP:7001,redis://YOUR_IP:7002,redis://YOUR_IP:7003 +``` + +**Important**: Always replace `YOUR_IP` with your actual system IP address. \ No newline at end of file diff --git a/sink/redis-cluster/docker-compose.yml b/sink/redis-cluster/docker-compose.yml new file mode 100644 index 000000000..05c7f7cc7 --- /dev/null +++ b/sink/redis-cluster/docker-compose.yml @@ -0,0 +1,87 @@ +version: "3.8" + +services: + redis-node-1: + image: redis:7 + container_name: redis-node-1 + ports: + - "7001:7001" + - "17001:17001" + volumes: + - redis-node-1-data:/data + command: > + redis-server + --port 7001 + --cluster-enabled yes + --cluster-config-file nodes.conf + --cluster-node-timeout 5000 + --appendonly yes + --bind 0.0.0.0 + --cluster-announce-ip 192.168.1.33 + --cluster-announce-port 7001 + --cluster-announce-bus-port 17001 + + redis-node-2: + image: redis:7 + container_name: redis-node-2 + ports: + - "7002:7002" + - "17002:17002" + volumes: + - redis-node-2-data:/data + command: > + redis-server + --port 7002 + --cluster-enabled yes + --cluster-config-file nodes.conf + --cluster-node-timeout 5000 + --appendonly yes + --bind 0.0.0.0 + --cluster-announce-ip 192.168.1.33 + --cluster-announce-port 7002 + --cluster-announce-bus-port 17002 + + redis-node-3: + image: redis:7 + container_name: redis-node-3 + ports: + - "7003:7003" + - "17003:17003" + volumes: + - redis-node-3-data:/data + command: > + redis-server + --port 7003 + --cluster-enabled yes + --cluster-config-file nodes.conf + --cluster-node-timeout 5000 + --appendonly yes + --bind 0.0.0.0 + --cluster-announce-ip 192.168.1.33 + --cluster-announce-port 7003 + --cluster-announce-bus-port 17003 + + redis-insight: + container_name: redis-insight + image: redis/redisinsight + ports: + - "5540:5540" + + redis-cluster-init: + image: redis:7 + depends_on: + - redis-node-1 + - redis-node-2 + - redis-node-3 + restart: "no" + command: > + sh -c " + sleep 10 + redis-cli --cluster create 192.168.1.33:7001 192.168.1.33:7002 192.168.1.33:7003 --cluster-yes + echo 'Cluster initialized' + " + +volumes: + redis-node-1-data: + redis-node-2-data: + redis-node-3-data: \ No newline at end of file