awatertrevi/infisical
1
0
Fork 0
mirror of https://github.com/awatertrevi/infisical.git synced 2026-10-15 14:27:23 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #3394 from akhilmhdh/feat/secret-cache

Browse Source 
Added caching for secret dal
This commit is contained in:
Maidul Islam
2025-04-11 16:11:30 -04:00
committed by GitHub
parent fe58508136 9565ef29d0
commit 32641cfc3a
19 changed files with 305 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/src/services/integration-auth/integration-delete-secret.ts
+2 -1
View File
@@ -50,7 +50,7 @@ const getIntegrationSecretsV2 = async (
}
// process secrets in current folder
const secrets = await secretV2BridgeDAL.findByFolderId(dto.folderId);
const secrets = await secretV2BridgeDAL.findByFolderId({ folderId: dto.folderId, projectId: dto.projectId });
secrets.forEach((secret) => {
const secretKey = secret.key;
@@ -63,6 +63,7 @@ const getIntegrationSecretsV2 = async (
// if no imports then return secrets in the current folder
if (!secretImports.length) return content;
const importedSecrets = await fnSecretsV2FromImports({
projectId: dto.projectId,
decryptor: dto.decryptor,
folderDAL,
secretDAL: secretV2BridgeDAL,
backend/src/services/secret-import/secret-import-fns.ts
+5 -2
View File
@@ -159,7 +159,8 @@ export const fnSecretsV2FromImports = async ({
decryptor,
expandSecretReferences,
hasSecretAccess,
viewSecretValue
viewSecretValue,
projectId
}: {
secretImports: (Omit<TSecretImports, "importEnv"> & {
importEnv: { id: string; slug: string; name: string };
@@ -176,6 +177,7 @@ export const fnSecretsV2FromImports = async ({
environment: string;
}) => Promise<string | undefined>;
hasSecretAccess: (environment: string, secretPath: string, secretName: string, secretTagSlugs: string[]) => boolean;
projectId: string;
}) => {
const cyclicDetector = new Set();
const stack: {
@@ -216,7 +218,8 @@ export const fnSecretsV2FromImports = async ({
type: SecretType.Shared
},
{
sort: [["id", "asc"]]
sort: [["id", "asc"]],
useCache: { projectId }
}
);
const importedSecretsGroupByFolderId = groupBy(importedSecrets, (i) => i.folderId);
backend/src/services/secret-import/secret-import-service.ts
+6 -1
View File
@@ -43,7 +43,7 @@ type TSecretImportServiceFactoryDep = {
secretImportDAL: TSecretImportDALFactory;
folderDAL: TSecretFolderDALFactory;
secretDAL: Pick<TSecretDALFactory, "find">;
secretV2BridgeDAL: Pick<TSecretV2BridgeDALFactory, "find">;
secretV2BridgeDAL: Pick<TSecretV2BridgeDALFactory, "find" | "invalidateSecretCacheByProjectId">;
projectBotService: Pick<TProjectBotServiceFactory, "getBotKey">;
projectDAL: Pick<TProjectDALFactory, "checkProjectUpgradeStatus">;
projectEnvDAL: TProjectEnvDALFactory;
@@ -184,6 +184,7 @@ export const secretImportServiceFactory = ({
});
}
await secretV2BridgeDAL.invalidateSecretCacheByProjectId(projectId);
return { ...secImport, importEnv };
};
@@ -281,6 +282,8 @@ export const secretImportServiceFactory = ({
);
return doc;
});
await secretV2BridgeDAL.invalidateSecretCacheByProjectId(projectId);
return { ...updatedSecImport, importEnv: importedEnv };
};
@@ -355,6 +358,7 @@ export const secretImportServiceFactory = ({
actorId
});
await secretV2BridgeDAL.invalidateSecretCacheByProjectId(projectId);
return secImport;
};
@@ -693,6 +697,7 @@ export const secretImportServiceFactory = ({
projectId
});
const importedSecrets = await fnSecretsV2FromImports({
projectId,
secretImports,
folderDAL,
viewSecretValue: true,
backend/src/services/secret-sync/secret-sync-queue.ts
+2 -1
View File
@@ -213,7 +213,7 @@ export const secretSyncQueueFactory = ({
canExpandValue: () => true
});
const secrets = await secretV2BridgeDAL.findByFolderId(folderId);
const secrets = await secretV2BridgeDAL.findByFolderId({ folderId, projectId });
await Promise.allSettled(
secrets.map(async (secret) => {
@@ -243,6 +243,7 @@ export const secretSyncQueueFactory = ({
if (secretImports.length) {
const importedSecrets = await fnSecretsV2FromImports({
projectId,
decryptor: decryptSecretValue,
folderDAL,
secretDAL: secretV2BridgeDAL,
backend/src/services/secret-v2-bridge/secret-v2-bridge-dal.ts
+199 -14
View File
@@ -2,7 +2,10 @@ import { Knex } from "knex";
import { validate as uuidValidate } from "uuid";
import { TDbClient } from "@app/db";
import { SecretsV2Schema, SecretType, TableName, TSecretsV2, TSecretsV2Update } from "@app/db/schemas";
import { ProjectType, SecretsV2Schema, SecretType, TableName, TSecretsV2, TSecretsV2Update } from "@app/db/schemas";
import { TKeyStoreFactory } from "@app/keystore/keystore";
import { getConfig } from "@app/lib/config/env";
import { generateCacheKeyFromData } from "@app/lib/crypto/cache";
import { BadRequestError, DatabaseError, NotFoundError } from "@app/lib/errors";
import {
buildFindFilter,
@@ -12,15 +15,67 @@ import {
TFindFilter,
TFindOpt
} from "@app/lib/knex";
import { OrderByDirection } from "@app/lib/types";
import { BufferKeysToString, OrderByDirection } from "@app/lib/types";
import { SecretsOrderBy } from "@app/services/secret/secret-types";
import { TFindSecretsByFolderIdsFilter } from "@app/services/secret-v2-bridge/secret-v2-bridge-types";
import type { TFindSecretsByFolderIdsFilter } from "@app/services/secret-v2-bridge/secret-v2-bridge-types";
export const SecretDalCacheKeys = {
get productKey() {
const { INFISICAL_PLATFORM_VERSION } = getConfig();
return `${ProjectType.SecretManager}:${INFISICAL_PLATFORM_VERSION || 0}`;
},
getSecretDalVersion: (projectId: string) => {
return `${SecretDalCacheKeys.productKey}:${projectId}:${TableName.SecretV2}-dal-version`;
},
findByFolderIds: (
projectId: string,
version: number,
{ useCache, tx, ...cacheKey }: Parameters<TSecretV2BridgeDALFactory["findByFolderIds"]>[0]
) => {
return `${SecretDalCacheKeys.productKey}:${projectId}:${
TableName.SecretV2
}-dal:v${version}:find-by-folder-ids:${generateCacheKeyFromData(cacheKey)}`;
},
findByFolderId: (
projectId: string,
version: number,
{ useCache, tx, ...cacheKey }: Parameters<TSecretV2BridgeDALFactory["findByFolderId"]>[0]
) => {
return `${SecretDalCacheKeys.productKey}:${projectId}:${
TableName.SecretV2
}-dal:v${version}:find-by-folder-id:${generateCacheKeyFromData(cacheKey)}`;
},
find: (projectId: string, version: number, ...args: Parameters<TSecretV2BridgeDALFactory["find"]>) => {
const [filter, opts] = args;
delete opts?.tx;
delete opts?.useCache;
return `${SecretDalCacheKeys.productKey}:${projectId}:${
TableName.SecretV2
}-dal:v${version}:find:${generateCacheKeyFromData({
filter,
opts
})}`;
}
};
export type TSecretV2BridgeDALFactory = ReturnType<typeof secretV2BridgeDALFactory>;
interface TSecretV2DalArg {
db: TDbClient;
keyStore: TKeyStoreFactory;
}
export const secretV2BridgeDALFactory = (db: TDbClient) => {
const SECRET_DAL_TTL = 5 * 60;
const SECRET_DAL_VERSION_TTL = 15 * 60;
const MAX_SECRET_CACHE_BYTES = 25 * 1024 * 1024;
export const secretV2BridgeDALFactory = ({ db, keyStore }: TSecretV2DalArg) => {
const secretOrm = ormify(db, TableName.SecretV2);
const invalidateSecretCacheByProjectId = async (projectId: string) => {
const secretDalVersionKey = SecretDalCacheKeys.getSecretDalVersion(projectId);
await keyStore.incrementBy(secretDalVersionKey, 1);
await keyStore.setExpiry(secretDalVersionKey, SECRET_DAL_VERSION_TTL);
};
const findOne = async (filter: Partial<TSecretsV2>, tx?: Knex) => {
try {
const docs = await (tx || db)(TableName.SecretV2)
@@ -73,8 +128,35 @@ export const secretV2BridgeDALFactory = (db: TDbClient) => {
}
};
const find = async (filter: TFindFilter<TSecretsV2>, { offset, limit, sort, tx }: TFindOpt<TSecretsV2> = {}) => {
const find = async (
filter: TFindFilter<TSecretsV2>,
opts: TFindOpt<TSecretsV2> & { useCache?: { projectId: string } } = {}
) => {
const { offset, limit, sort, tx, useCache } = opts;
try {
let secretDalVersion = 0;
if (useCache) {
const cachedSecretDalVersion = await keyStore.getItem(
SecretDalCacheKeys.getSecretDalVersion(useCache.projectId)
);
secretDalVersion = Number(cachedSecretDalVersion || 0);
const cacheKey = SecretDalCacheKeys.find(useCache.projectId, secretDalVersion, filter, opts);
const cachedSecrets = await keyStore.getItem(cacheKey);
if (cachedSecrets) {
await keyStore.setExpiry(cacheKey, SECRET_DAL_TTL);
const unsanitizedSecrets = JSON.parse(cachedSecrets) as BufferKeysToString<(typeof data)[number]>[];
const sanitizedSecrets = unsanitizedSecrets.map((el) => {
const encryptedValue = el.encryptedValue ? Buffer.from(el.encryptedValue, "base64") : null;
const encryptedComment = el.encryptedComment ? Buffer.from(el.encryptedComment, "base64") : null;
const createdAt = new Date(el.createdAt);
const updatedAt = new Date(el.updatedAt);
return { ...el, encryptedComment, encryptedValue, createdAt, updatedAt };
});
return sanitizedSecrets;
}
}
const query = (tx || db)(TableName.SecretV2)
// eslint-disable-next-line @typescript-eslint/no-misused-promises
.where(buildFindFilter(filter))
@@ -142,6 +224,23 @@ export const secretV2BridgeDALFactory = (db: TDbClient) => {
}
]
});
if (useCache) {
const cachedSecrets = data.map((el) => {
const encryptedValue = el.encryptedValue ? el.encryptedValue.toString("base64") : null;
const encryptedComment = el.encryptedComment ? el.encryptedComment.toString("base64") : null;
return { ...el, encryptedValue, encryptedComment };
});
const cache = JSON.stringify(cachedSecrets);
if (Buffer.byteLength(cache, "utf8") < MAX_SECRET_CACHE_BYTES) {
await keyStore.setItemWithExpiry(
SecretDalCacheKeys.find(useCache.projectId, secretDalVersion, filter, opts),
SECRET_DAL_TTL,
cache
);
}
}
return data;
} catch (error) {
throw new DatabaseError({ error, name: `${TableName.SecretV2}: Find` });
@@ -246,14 +345,43 @@ export const secretV2BridgeDALFactory = (db: TDbClient) => {
}
};
const findByFolderId = async (folderId: string, userId?: string, tx?: Knex) => {
const findByFolderId = async (dto: {
folderId: string;
userId?: string;
tx?: Knex;
projectId: string;
useCache?: boolean;
}) => {
try {
// check if not uui then userId id is null (corner case because service token's ID is not UUI in effort to keep backwards compatibility from mongo)
const { folderId, tx, projectId } = dto;
let { userId } = dto;
// check if not uui then userId id is null (corner case because service token's ID is not UUI in effort to keep backwards compatibility from mongo
if (userId && !uuidValidate(userId)) {
// eslint-disable-next-line
userId = undefined;
}
const cachedSecretDalVersion = await keyStore.getItem(SecretDalCacheKeys.getSecretDalVersion(projectId));
const secretDalVersion = Number(cachedSecretDalVersion || 0);
if (dto.useCache) {
const cacheKey = SecretDalCacheKeys.findByFolderId(projectId, secretDalVersion, dto);
const cachedSecrets = await keyStore.getItem(cacheKey);
if (cachedSecrets) {
await keyStore.setExpiry(cacheKey, SECRET_DAL_TTL);
const unsanitizedSecrets = JSON.parse(cachedSecrets) as BufferKeysToString<(typeof data)[number]>[];
const sanitizedSecrets = unsanitizedSecrets.map((el) => {
const encryptedValue = el.encryptedValue ? Buffer.from(el.encryptedValue, "base64") : null;
const encryptedComment = el.encryptedComment ? Buffer.from(el.encryptedComment, "base64") : null;
const createdAt = new Date(el.createdAt);
const updatedAt = new Date(el.updatedAt);
return { ...el, encryptedComment, encryptedValue, createdAt, updatedAt };
});
return sanitizedSecrets;
}
}
const secs = await (tx || db.replicaNode())(TableName.SecretV2)
.where({ folderId })
.where((bd) => {
@@ -309,6 +437,22 @@ export const secretV2BridgeDALFactory = (db: TDbClient) => {
}
]
});
if (dto.useCache) {
const newCachedSecrets = data.map((el) => {
const encryptedValue = el.encryptedValue ? el.encryptedValue.toString("base64") : null;
const encryptedComment = el.encryptedComment ? el.encryptedComment.toString("base64") : null;
return { ...el, encryptedValue, encryptedComment };
});
const cache = JSON.stringify(newCachedSecrets);
if (Buffer.byteLength(cache, "utf8") < MAX_SECRET_CACHE_BYTES) {
await keyStore.setItemWithExpiry(
SecretDalCacheKeys.findByFolderId(projectId, secretDalVersion, dto),
SECRET_DAL_TTL,
cache
);
}
}
return data;
} catch (error) {
throw new DatabaseError({ error, name: "get all secret" });
@@ -394,12 +538,16 @@ export const secretV2BridgeDALFactory = (db: TDbClient) => {
}
};
const findByFolderIds = async (
folderIds: string[],
userId?: string,
tx?: Knex,
filters?: TFindSecretsByFolderIdsFilter
) => {
const findByFolderIds = async (dto: {
folderIds: string[];
userId?: string;
tx?: Knex;
projectId: string;
filters?: TFindSecretsByFolderIdsFilter;
useCache?: boolean;
}) => {
const { folderIds, tx, filters, useCache, projectId } = dto;
let { userId } = dto;
try {
// check if not uui then userId id is null (corner case because service token's ID is not UUI in effort to keep backwards compatibility from mongo)
if (userId && !uuidValidate(userId)) {
@@ -407,6 +555,26 @@ export const secretV2BridgeDALFactory = (db: TDbClient) => {
userId = undefined;
}
const cachedSecretDalVersion = await keyStore.getItem(SecretDalCacheKeys.getSecretDalVersion(projectId));
const secretDalVersion = Number(cachedSecretDalVersion || 0);
if (useCache) {
const cacheKey = SecretDalCacheKeys.findByFolderIds(projectId, secretDalVersion, dto);
const cachedSecrets = await keyStore.getItem(cacheKey);
if (cachedSecrets) {
await keyStore.setExpiry(cacheKey, SECRET_DAL_TTL);
const unsanitizedSecrets = JSON.parse(cachedSecrets) as BufferKeysToString<(typeof data)[number]>[];
const sanitizedSecrets = unsanitizedSecrets.map((el) => {
const encryptedValue = el.encryptedValue ? Buffer.from(el.encryptedValue, "base64") : null;
const encryptedComment = el.encryptedComment ? Buffer.from(el.encryptedComment, "base64") : null;
const createdAt = new Date(el.createdAt);
const updatedAt = new Date(el.updatedAt);
return { ...el, encryptedComment, encryptedValue, createdAt, updatedAt };
});
return sanitizedSecrets;
}
}
const query = (tx || db.replicaNode())(TableName.SecretV2)
.whereIn(`${TableName.SecretV2}.folderId`, folderIds)
.where((bd) => {
@@ -532,6 +700,22 @@ export const secretV2BridgeDALFactory = (db: TDbClient) => {
}
]
});
if (useCache) {
const cachedSecrets = data.map((el) => {
const encryptedValue = el.encryptedValue ? el.encryptedValue.toString("base64") : null;
const encryptedComment = el.encryptedComment ? el.encryptedComment.toString("base64") : null;
return { ...el, encryptedValue, encryptedComment };
});
const cache = JSON.stringify(cachedSecrets);
if (Buffer.byteLength(cache, "utf8") < MAX_SECRET_CACHE_BYTES) {
await keyStore.setItemWithExpiry(
SecretDalCacheKeys.findByFolderIds(projectId, secretDalVersion, dto),
SECRET_DAL_TTL,
cache
);
}
}
return data;
} catch (error) {
@@ -724,6 +908,7 @@ export const secretV2BridgeDALFactory = (db: TDbClient) => {
findAllProjectSecretValues,
countByFolderIds,
findOne,
find
find,
invalidateSecretCacheByProjectId
};
};
backend/src/services/secret-v2-bridge/secret-v2-bridge-fns.ts
+1 -1
View File
@@ -501,7 +501,7 @@ export const expandSecretReferencesFactory = ({
const folder = await folderDAL.findBySecretPath(projectId, environment, secretPath);
if (!folder) return { value: "", tags: [] };
const secrets = await secretDAL.findByFolderId(folder.id);
const secrets = await secretDAL.findByFolderId({ folderId: folder.id, projectId, useCache: true });
const decryptedSecret = secrets.reduce<Record<string, { value: string; tags: string[] }>>((prev, secret) => {
// eslint-disable-next-line no-param-reassign
backend/src/services/secret-v2-bridge/secret-v2-bridge-service.ts
+27 -13
View File
@@ -331,6 +331,7 @@ export const secretV2BridgeServiceFactory = ({
return createdSecret;
});
await secretDAL.invalidateSecretCacheByProjectId(projectId);
if (inputSecret.type === SecretType.Shared) {
await snapshotService.performSnapshot(folderId);
await secretQueueService.syncSecrets({
@@ -539,6 +540,7 @@ export const secretV2BridgeServiceFactory = ({
projectId
});
await secretDAL.invalidateSecretCacheByProjectId(projectId);
if (inputSecret.type === SecretType.Shared) {
await snapshotService.performSnapshot(folderId);
await secretQueueService.syncSecrets({
@@ -647,6 +649,7 @@ export const secretV2BridgeServiceFactory = ({
})
);
await secretDAL.invalidateSecretCacheByProjectId(projectId);
if (inputSecret.type === SecretType.Shared) {
await snapshotService.performSnapshot(folderId);
await secretQueueService.syncSecrets({
@@ -796,12 +799,14 @@ export const secretV2BridgeServiceFactory = ({
) => {
const groupedFolderMappings = groupBy(folderMappings, (folderMapping) => folderMapping.folderId);
const secrets = await secretDAL.findByFolderIds(
folderMappings.map((folderMapping) => folderMapping.folderId),
const secrets = await secretDAL.findByFolderIds({
projectId,
folderIds: folderMappings.map((folderMapping) => folderMapping.folderId),
userId,
undefined,
filters
);
tx: undefined,
filters,
useCache: true
});
const { decryptor: secretManagerDecryptor } = await kmsService.createCipherPairWithDataKey({
type: KmsDataKey.SecretManager,
@@ -952,12 +957,14 @@ export const secretV2BridgeServiceFactory = ({
const groupedPaths = groupBy(paths, (p) => p.folderId);
const secrets = await secretDAL.findByFolderIds(
paths.map((p) => p.folderId),
actorId,
undefined,
params
);
const secrets = await secretDAL.findByFolderIds({
projectId,
folderIds: paths.map((p) => p.folderId),
userId: actorId,
tx: undefined,
filters: params,
useCache: true
});
const { decryptor: secretManagerDecryptor } = await kmsService.createCipherPairWithDataKey({
type: KmsDataKey.SecretManager,
@@ -1087,6 +1094,7 @@ export const secretV2BridgeServiceFactory = ({
const secretImports = await secretImportDAL.findByFolderIds(paths.map((p) => p.folderId));
const allowedImports = secretImports.filter(({ isReplication }) => !isReplication);
const importedSecrets = await fnSecretsV2FromImports({
projectId,
viewSecretValue,
secretImports: allowedImports,
secretDAL,
@@ -1304,6 +1312,7 @@ export const secretV2BridgeServiceFactory = ({
if (!secret && includeImports) {
const secretImports = await secretImportDAL.find({ folderId, isReplication: false });
const importedSecrets = await fnSecretsV2FromImports({
projectId,
secretImports,
viewSecretValue,
secretDAL,
@@ -1543,7 +1552,7 @@ export const secretV2BridgeServiceFactory = ({
tx
})
);
await secretDAL.invalidateSecretCacheByProjectId(projectId);
await snapshotService.performSnapshot(folderId);
await secretQueueService.syncSecrets({
actor,
@@ -1883,6 +1892,7 @@ export const secretV2BridgeServiceFactory = ({
}
});
await secretDAL.invalidateSecretCacheByProjectId(projectId);
await Promise.allSettled(folders.map((el) => (el?.id ? snapshotService.performSnapshot(el.id) : undefined)));
await Promise.allSettled(
folders.map((el) =>
@@ -2014,6 +2024,7 @@ export const secretV2BridgeServiceFactory = ({
})
);
await secretDAL.invalidateSecretCacheByProjectId(projectId);
await snapshotService.performSnapshot(folderId);
await secretQueueService.syncSecrets({
actor,
@@ -2537,6 +2548,9 @@ export const secretV2BridgeServiceFactory = ({
}
});
if (isDestinationUpdated || isSourceUpdated) {
await secretDAL.invalidateSecretCacheByProjectId(projectId);
}
if (isDestinationUpdated) {
await snapshotService.performSnapshot(destinationFolder.id);
await secretQueueService.syncSecrets({
@@ -2715,7 +2729,7 @@ export const secretV2BridgeServiceFactory = ({
generatePaths(folderMap).map(({ folderId, path }) => [folderId, path === "/" ? path : path.substring(1)])
);
const secrets = await secretDAL.findByFolderIds(folders.map((f) => f.id));
const secrets = await secretDAL.findByFolderIds({ folderIds: folders.map((f) => f.id), projectId, useCache: true });
const { decryptor: secretManagerDecryptor } = await kmsService.createCipherPairWithDataKey({
type: KmsDataKey.SecretManager,
backend/src/services/secret-v2-bridge/secret-v2-bridge-types.ts
+7
View File
@@ -279,6 +279,13 @@ export type TUpdateManySecretsFnFactory = {
folderDAL: TSecretFolderDALFactory;
};
export type TFindByFolderIdDALDTO = {
folderId: string;
userId?: string;
tx?: Knex;
projectId: string;
};
export type TUpdateManySecretsFn = {
projectId: string;
environment: string;
backend/src/services/secret/secret-queue.ts
+2 -1
View File
@@ -367,7 +367,7 @@ export const secretQueueFactory = ({
canExpandValue: () => true
});
// process secrets in current folder
const secrets = await secretV2BridgeDAL.findByFolderId(dto.folderId);
const secrets = await secretV2BridgeDAL.findByFolderId({ folderId: dto.folderId, projectId: dto.projectId });
await Promise.allSettled(
secrets.map(async (secret) => {
@@ -397,6 +397,7 @@ export const secretQueueFactory = ({
// if no imports then return secrets in the current folder
if (!secretImports.length) return content;
const importedSecrets = await fnSecretsV2FromImports({
projectId: dto.projectId,
decryptor: dto.decryptor,
folderDAL,
secretDAL: secretV2BridgeDAL,