awatertrevi/infisical
1
0
Fork 0
mirror of https://github.com/awatertrevi/infisical.git synced 2026-10-19 22:20:36 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: added support for redis read replica and updated doc

Browse Source
This commit is contained in:
=
2025-09-07 22:45:41 +05:30
parent 354ca31559
commit 0502dc60a6
4 changed files with 95 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/src/keystore/keystore.ts
+43 -13
View File
@@ -3,6 +3,7 @@ import { pgAdvisoryLockHashText } from "@app/lib/crypto/hashtext";
import { applyJitter } from "@app/lib/dates";
import { delay as delayMs } from "@app/lib/delay";
import { ExecutionResult, Redlock, Settings } from "@app/lib/red-lock";
import { Redis, Cluster } from "ioredis";
export const PgSqlLock = {
BootUpMigration: 2023,
@@ -102,30 +103,53 @@ export type TKeyStoreFactory = {
getKeysByPattern: (pattern: string, limit?: number) => Promise<string[]>;
};
const pickPrimaryOrSecondaryRedis = (primary: Redis | Cluster, secondaries?: Array<Redis | Cluster>) => {
if (!secondaries || !secondaries.length) return primary;
const selectedReplica = secondaries[Math.floor(Math.random() * secondaries.length)];
return selectedReplica;
};
export const keyStoreFactory = (redisConfigKeys: TRedisConfigKeys): TKeyStoreFactory => {
const redis = buildRedisFromConfig(redisConfigKeys);
const redisLock = new Redlock([redis], { retryCount: 2, retryDelay: 200 });
const primaryRedis = buildRedisFromConfig(redisConfigKeys);
const redisReadReplicas = redisConfigKeys.REDIS_READ_REPLICAS?.map((el) => {
if (redisConfigKeys.REDIS_URL) {
const primaryNode = new URL(redisConfigKeys?.REDIS_URL);
primaryNode.hostname = el.host;
primaryNode.port = String(el.port);
return buildRedisFromConfig({ ...redisConfigKeys, REDIS_URL: primaryNode.toString() });
}
if (redisConfigKeys.REDIS_SENTINEL_HOSTS) {
return buildRedisFromConfig({ ...redisConfigKeys, REDIS_SENTINEL_HOSTS: [el] });
}
return buildRedisFromConfig({ ...redisConfigKeys, REDIS_CLUSTER_HOSTS: [el] });
});
const redisLock = new Redlock([primaryRedis], { retryCount: 2, retryDelay: 200 });
const setItem = async (key: string, value: string | number | Buffer, prefix?: string) =>
redis.set(prefix ? `${prefix}:${key}` : key, value);
primaryRedis.set(prefix ? `${prefix}:${key}` : key, value);
const getItem = async (key: string, prefix?: string) => redis.get(prefix ? `${prefix}:${key}` : key);
const getItem = async (key: string, prefix?: string) =>
pickPrimaryOrSecondaryRedis(primaryRedis, redisReadReplicas).get(prefix ? `${prefix}:${key}` : key);
const getItems = async (keys: string[], prefix?: string) =>
redis.mget(keys.map((key) => (prefix ? `${prefix}:${key}` : key)));
pickPrimaryOrSecondaryRedis(primaryRedis, redisReadReplicas).mget(
keys.map((key) => (prefix ? `${prefix}:${key}` : key))
);
const setItemWithExpiry = async (
key: string,
expiryInSeconds: number | string,
value: string | number | Buffer,
prefix?: string
) => redis.set(prefix ? `${prefix}:${key}` : key, value, "EX", expiryInSeconds);
) => primaryRedis.set(prefix ? `${prefix}:${key}` : key, value, "EX", expiryInSeconds);
const deleteItem = async (key: string) => redis.del(key);
const deleteItem = async (key: string) => primaryRedis.del(key);
const deleteItemsByKeyIn = async (keys: string[]) => {
if (keys.length === 0) return 0;
return redis.del(keys);
return primaryRedis.del(keys);
};
const deleteItems = async ({ pattern, batchSize = 500, delay = 1500, jitter = 200 }: TDeleteItems) => {
@@ -135,12 +159,12 @@ export const keyStoreFactory = (redisConfigKeys: TRedisConfigKeys): TKeyStoreFac
do {
// Await in loop is needed so that Redis is not overwhelmed
// eslint-disable-next-line no-await-in-loop
const [nextCursor, keys] = await redis.scan(cursor, "MATCH", pattern, "COUNT", 1000); // Count should be 1000 - 5000 for prod loads
const [nextCursor, keys] = await primaryRedis.scan(cursor, "MATCH", pattern, "COUNT", 1000); // Count should be 1000 - 5000 for prod loads
cursor = nextCursor;
for (let i = 0; i < keys.length; i += batchSize) {
const batch = keys.slice(i, i + batchSize);
const pipeline = redis.pipeline();
const pipeline = primaryRedis.pipeline();
for (const key of batch) {
pipeline.unlink(key);
}
@@ -156,9 +180,9 @@ export const keyStoreFactory = (redisConfigKeys: TRedisConfigKeys): TKeyStoreFac
return totalDeleted;
};
const incrementBy = async (key: string, value: number) => redis.incrby(key, value);
const incrementBy = async (key: string, value: number) => primaryRedis.incrby(key, value);
const setExpiry = async (key: string, expiryInSeconds: number) => redis.expire(key, expiryInSeconds);
const setExpiry = async (key: string, expiryInSeconds: number) => primaryRedis.expire(key, expiryInSeconds);
const waitTillReady = async ({
key,
@@ -189,7 +213,13 @@ export const keyStoreFactory = (redisConfigKeys: TRedisConfigKeys): TKeyStoreFac
do {
// eslint-disable-next-line no-await-in-loop
const [nextCursor, keys] = await redis.scan(cursor, "MATCH", pattern, "COUNT", 1000);
const [nextCursor, keys] = await pickPrimaryOrSecondaryRedis(primaryRedis, redisReadReplicas).scan(
cursor,
"MATCH",
pattern,
"COUNT",
1000
);
cursor = nextCursor;
allKeys.push(...keys);