awatertrevi/infisical
1
0
Fork 0
mirror of https://github.com/awatertrevi/infisical.git synced 2026-10-16 12:04:03 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: added caching for secret dal\

Browse Source
This commit is contained in:
=
2025-04-12 00:36:41 +05:30
parent 27fdf68e42
commit 5f6db870a6
18 changed files with 288 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/src/services/secret-v2-bridge/secret-v2-bridge-dal.ts
+185 -14
View File
@@ -2,7 +2,10 @@ import { Knex } from "knex";
import { validate as uuidValidate } from "uuid";
import { TDbClient } from "@app/db";
import { SecretsV2Schema, SecretType, TableName, TSecretsV2, TSecretsV2Update } from "@app/db/schemas";
import { ProjectType, SecretsV2Schema, SecretType, TableName, TSecretsV2, TSecretsV2Update } from "@app/db/schemas";
import { TKeyStoreFactory } from "@app/keystore/keystore";
import { getConfig } from "@app/lib/config/env";
import { getCacheKey } from "@app/lib/crypto/cache";
import { BadRequestError, DatabaseError, NotFoundError } from "@app/lib/errors";
import {
buildFindFilter,
@@ -12,15 +15,64 @@ import {
TFindFilter,
TFindOpt
} from "@app/lib/knex";
import { OrderByDirection } from "@app/lib/types";
import { BufferKeysToString, OrderByDirection } from "@app/lib/types";
import { SecretsOrderBy } from "@app/services/secret/secret-types";
import { TFindSecretsByFolderIdsFilter } from "@app/services/secret-v2-bridge/secret-v2-bridge-types";
import type { TFindSecretsByFolderIdsFilter } from "@app/services/secret-v2-bridge/secret-v2-bridge-types";
export const SecretDalCacheKeys = {
get productKey() {
const { INFISICAL_PLATFORM_VERSION } = getConfig();
return `${ProjectType.SecretManager}:${INFISICAL_PLATFORM_VERSION || 0}`;
},
getSecretDalVersion: (projectId: string) => {
return `${SecretDalCacheKeys.productKey}:${projectId}:${TableName.SecretV2}-dal-version`;
},
findByFolderIds: (
projectId: string,
version: number,
{ useCache, tx, ...cacheKey }: Parameters<TSecretV2BridgeDALFactory["findByFolderIds"]>[0]
) => {
return `${SecretDalCacheKeys.productKey}:${projectId}:${
TableName.SecretV2
}-dal:v${version}:find-by-folder-ids:${getCacheKey(cacheKey)}`;
},
findByFolderId: (
projectId: string,
version: number,
{ useCache, tx, ...cacheKey }: Parameters<TSecretV2BridgeDALFactory["findByFolderId"]>[0]
) => {
return `${SecretDalCacheKeys.productKey}:${projectId}:${
TableName.SecretV2
}-dal:v${version}:find-by-folder-id:${getCacheKey(cacheKey)}`;
},
find: (projectId: string, version: number, ...args: Parameters<TSecretV2BridgeDALFactory["find"]>) => {
const [filter, opts] = args;
delete opts?.tx;
delete opts?.useCache;
return `${SecretDalCacheKeys.productKey}:${projectId}:${TableName.SecretV2}-dal:v${version}:find:${getCacheKey({
filter,
opts
})}`;
}
};
export type TSecretV2BridgeDALFactory = ReturnType<typeof secretV2BridgeDALFactory>;
interface TSecretV2DalArg {
db: TDbClient;
keyStore: TKeyStoreFactory;
}
export const secretV2BridgeDALFactory = (db: TDbClient) => {
const SECRET_DAL_TTL = 5 * 60;
const SECRET_DAL_VERSION_TTL = 15 * 60;
export const secretV2BridgeDALFactory = ({ db, keyStore }: TSecretV2DalArg) => {
const secretOrm = ormify(db, TableName.SecretV2);
const cacheInvalidateSecretByProjectId = async (projectId: string) => {
const secretDalVersionKey = SecretDalCacheKeys.getSecretDalVersion(projectId);
await keyStore.incrementBy(secretDalVersionKey, 1);
await keyStore.setExpiry(secretDalVersionKey, SECRET_DAL_VERSION_TTL);
};
const findOne = async (filter: Partial<TSecretsV2>, tx?: Knex) => {
try {
const docs = await (tx || db)(TableName.SecretV2)
@@ -73,8 +125,35 @@ export const secretV2BridgeDALFactory = (db: TDbClient) => {
}
};
const find = async (filter: TFindFilter<TSecretsV2>, { offset, limit, sort, tx }: TFindOpt<TSecretsV2> = {}) => {
const find = async (
filter: TFindFilter<TSecretsV2>,
opts: TFindOpt<TSecretsV2> & { useCache?: { projectId: string } } = {}
) => {
const { offset, limit, sort, tx, useCache } = opts;
try {
let secretDalVersion = 0;
if (useCache) {
const cachedSecretDalVersion = await keyStore.getItem(
SecretDalCacheKeys.getSecretDalVersion(useCache.projectId)
);
secretDalVersion = Number(cachedSecretDalVersion || 0);
const cacheKey = SecretDalCacheKeys.find(useCache.projectId, secretDalVersion, filter, opts);
const cachedSecrets = await keyStore.getItem(cacheKey);
if (cachedSecrets) {
await keyStore.setExpiry(cacheKey, SECRET_DAL_TTL);
const unsanitizedSecrets = JSON.parse(cachedSecrets) as BufferKeysToString<(typeof data)[number]>[];
const sanitizedSecrets = unsanitizedSecrets.map((el) => {
const encryptedValue = el.encryptedValue ? Buffer.from(el.encryptedValue, "base64") : null;
const encryptedComment = el.encryptedComment ? Buffer.from(el.encryptedComment, "base64") : null;
const createdAt = new Date(el.createdAt);
const updatedAt = new Date(el.updatedAt);
return { ...el, encryptedComment, encryptedValue, createdAt, updatedAt };
});
return sanitizedSecrets;
}
}
const query = (tx || db)(TableName.SecretV2)
// eslint-disable-next-line @typescript-eslint/no-misused-promises
.where(buildFindFilter(filter))
@@ -142,6 +221,20 @@ export const secretV2BridgeDALFactory = (db: TDbClient) => {
}
]
});
if (useCache) {
const cachedSecrets = data.map((el) => {
const encryptedValue = el.encryptedValue ? el.encryptedValue.toString("base64") : null;
const encryptedComment = el.encryptedComment ? el.encryptedComment.toString("base64") : null;
return { ...el, encryptedValue, encryptedComment };
});
await keyStore.setItemWithExpiry(
SecretDalCacheKeys.find(useCache.projectId, secretDalVersion, filter, opts),
SECRET_DAL_TTL,
JSON.stringify(cachedSecrets)
);
}
return data;
} catch (error) {
throw new DatabaseError({ error, name: `${TableName.SecretV2}: Find` });
@@ -246,14 +339,43 @@ export const secretV2BridgeDALFactory = (db: TDbClient) => {
}
};
const findByFolderId = async (folderId: string, userId?: string, tx?: Knex) => {
const findByFolderId = async (dto: {
folderId: string;
userId?: string;
tx?: Knex;
projectId: string;
useCache?: boolean;
}) => {
try {
// check if not uui then userId id is null (corner case because service token's ID is not UUI in effort to keep backwards compatibility from mongo)
const { folderId, tx, projectId } = dto;
let { userId } = dto;
// check if not uui then userId id is null (corner case because service token's ID is not UUI in effort to keep backwards compatibility from mongo
if (userId && !uuidValidate(userId)) {
// eslint-disable-next-line
userId = undefined;
}
const cachedSecretDalVersion = await keyStore.getItem(SecretDalCacheKeys.getSecretDalVersion(projectId));
const secretDalVersion = Number(cachedSecretDalVersion || 0);
if (dto.useCache) {
const cacheKey = SecretDalCacheKeys.findByFolderId(projectId, secretDalVersion, dto);
const cachedSecrets = await keyStore.getItem(cacheKey);
if (cachedSecrets) {
await keyStore.setExpiry(cacheKey, SECRET_DAL_TTL);
const unsanitizedSecrets = JSON.parse(cachedSecrets) as BufferKeysToString<(typeof data)[number]>[];
const sanitizedSecrets = unsanitizedSecrets.map((el) => {
const encryptedValue = el.encryptedValue ? Buffer.from(el.encryptedValue, "base64") : null;
const encryptedComment = el.encryptedComment ? Buffer.from(el.encryptedComment, "base64") : null;
const createdAt = new Date(el.createdAt);
const updatedAt = new Date(el.updatedAt);
return { ...el, encryptedComment, encryptedValue, createdAt, updatedAt };
});
return sanitizedSecrets;
}
}
const secs = await (tx || db.replicaNode())(TableName.SecretV2)
.where({ folderId })
.where((bd) => {
@@ -309,6 +431,18 @@ export const secretV2BridgeDALFactory = (db: TDbClient) => {
}
]
});
if (dto.useCache) {
const newCachedSecrets = data.map((el) => {
const encryptedValue = el.encryptedValue ? el.encryptedValue.toString("base64") : null;
const encryptedComment = el.encryptedComment ? el.encryptedComment.toString("base64") : null;
return { ...el, encryptedValue, encryptedComment };
});
await keyStore.setItemWithExpiry(
SecretDalCacheKeys.findByFolderId(projectId, secretDalVersion, dto),
SECRET_DAL_TTL,
JSON.stringify(newCachedSecrets)
);
}
return data;
} catch (error) {
throw new DatabaseError({ error, name: "get all secret" });
@@ -394,12 +528,16 @@ export const secretV2BridgeDALFactory = (db: TDbClient) => {
}
};
const findByFolderIds = async (
folderIds: string[],
userId?: string,
tx?: Knex,
filters?: TFindSecretsByFolderIdsFilter
) => {
const findByFolderIds = async (dto: {
folderIds: string[];
userId?: string;
tx?: Knex;
projectId: string;
filters?: TFindSecretsByFolderIdsFilter;
useCache?: boolean;
}) => {
const { folderIds, tx, filters, useCache, projectId } = dto;
let { userId } = dto;
try {
// check if not uui then userId id is null (corner case because service token's ID is not UUI in effort to keep backwards compatibility from mongo)
if (userId && !uuidValidate(userId)) {
@@ -407,6 +545,26 @@ export const secretV2BridgeDALFactory = (db: TDbClient) => {
userId = undefined;
}
const cachedSecretDalVersion = await keyStore.getItem(SecretDalCacheKeys.getSecretDalVersion(projectId));
const secretDalVersion = Number(cachedSecretDalVersion || 0);
if (useCache) {
const cacheKey = SecretDalCacheKeys.findByFolderIds(projectId, secretDalVersion, dto);
const cachedSecrets = await keyStore.getItem(cacheKey);
if (cachedSecrets) {
await keyStore.setExpiry(cacheKey, SECRET_DAL_TTL);
const unsanitizedSecrets = JSON.parse(cachedSecrets) as BufferKeysToString<(typeof data)[number]>[];
const sanitizedSecrets = unsanitizedSecrets.map((el) => {
const encryptedValue = el.encryptedValue ? Buffer.from(el.encryptedValue, "base64") : null;
const encryptedComment = el.encryptedComment ? Buffer.from(el.encryptedComment, "base64") : null;
const createdAt = new Date(el.createdAt);
const updatedAt = new Date(el.updatedAt);
return { ...el, encryptedComment, encryptedValue, createdAt, updatedAt };
});
return sanitizedSecrets;
}
}
const query = (tx || db.replicaNode())(TableName.SecretV2)
.whereIn(`${TableName.SecretV2}.folderId`, folderIds)
.where((bd) => {
@@ -532,6 +690,18 @@ export const secretV2BridgeDALFactory = (db: TDbClient) => {
}
]
});
if (useCache) {
const cachedSecrets = data.map((el) => {
const encryptedValue = el.encryptedValue ? el.encryptedValue.toString("base64") : null;
const encryptedComment = el.encryptedComment ? el.encryptedComment.toString("base64") : null;
return { ...el, encryptedValue, encryptedComment };
});
await keyStore.setItemWithExpiry(
SecretDalCacheKeys.findByFolderIds(projectId, secretDalVersion, dto),
SECRET_DAL_TTL,
JSON.stringify(cachedSecrets)
);
}
return data;
} catch (error) {
@@ -724,6 +894,7 @@ export const secretV2BridgeDALFactory = (db: TDbClient) => {
findAllProjectSecretValues,
countByFolderIds,
findOne,
find
find,
cacheInvalidateSecretByProjectId
};
};
backend/src/services/secret-v2-bridge/secret-v2-bridge-fns.ts
+1 -1
View File
@@ -501,7 +501,7 @@ export const expandSecretReferencesFactory = ({
const folder = await folderDAL.findBySecretPath(projectId, environment, secretPath);
if (!folder) return { value: "", tags: [] };
const secrets = await secretDAL.findByFolderId(folder.id);
const secrets = await secretDAL.findByFolderId({ folderId: folder.id, projectId, useCache: true });
const decryptedSecret = secrets.reduce<Record<string, { value: string; tags: string[] }>>((prev, secret) => {
// eslint-disable-next-line no-param-reassign
backend/src/services/secret-v2-bridge/secret-v2-bridge-service.ts
+27 -13
View File
@@ -331,6 +331,7 @@ export const secretV2BridgeServiceFactory = ({
return createdSecret;
});
await secretDAL.cacheInvalidateSecretByProjectId(projectId);
if (inputSecret.type === SecretType.Shared) {
await snapshotService.performSnapshot(folderId);
await secretQueueService.syncSecrets({
@@ -539,6 +540,7 @@ export const secretV2BridgeServiceFactory = ({
projectId
});
await secretDAL.cacheInvalidateSecretByProjectId(projectId);
if (inputSecret.type === SecretType.Shared) {
await snapshotService.performSnapshot(folderId);
await secretQueueService.syncSecrets({
@@ -647,6 +649,7 @@ export const secretV2BridgeServiceFactory = ({
})
);
await secretDAL.cacheInvalidateSecretByProjectId(projectId);
if (inputSecret.type === SecretType.Shared) {
await snapshotService.performSnapshot(folderId);
await secretQueueService.syncSecrets({
@@ -796,12 +799,14 @@ export const secretV2BridgeServiceFactory = ({
) => {
const groupedFolderMappings = groupBy(folderMappings, (folderMapping) => folderMapping.folderId);
const secrets = await secretDAL.findByFolderIds(
folderMappings.map((folderMapping) => folderMapping.folderId),
const secrets = await secretDAL.findByFolderIds({
projectId,
folderIds: folderMappings.map((folderMapping) => folderMapping.folderId),
userId,
undefined,
filters
);
tx: undefined,
filters,
useCache: true
});
const { decryptor: secretManagerDecryptor } = await kmsService.createCipherPairWithDataKey({
type: KmsDataKey.SecretManager,
@@ -952,12 +957,14 @@ export const secretV2BridgeServiceFactory = ({
const groupedPaths = groupBy(paths, (p) => p.folderId);
const secrets = await secretDAL.findByFolderIds(
paths.map((p) => p.folderId),
actorId,
undefined,
params
);
const secrets = await secretDAL.findByFolderIds({
projectId,
folderIds: paths.map((p) => p.folderId),
userId: actorId,
tx: undefined,
filters: params,
useCache: true
});
const { decryptor: secretManagerDecryptor } = await kmsService.createCipherPairWithDataKey({
type: KmsDataKey.SecretManager,
@@ -1087,6 +1094,7 @@ export const secretV2BridgeServiceFactory = ({
const secretImports = await secretImportDAL.findByFolderIds(paths.map((p) => p.folderId));
const allowedImports = secretImports.filter(({ isReplication }) => !isReplication);
const importedSecrets = await fnSecretsV2FromImports({
projectId,
viewSecretValue,
secretImports: allowedImports,
secretDAL,
@@ -1304,6 +1312,7 @@ export const secretV2BridgeServiceFactory = ({
if (!secret && includeImports) {
const secretImports = await secretImportDAL.find({ folderId, isReplication: false });
const importedSecrets = await fnSecretsV2FromImports({
projectId,
secretImports,
viewSecretValue,
secretDAL,
@@ -1543,7 +1552,7 @@ export const secretV2BridgeServiceFactory = ({
tx
})
);
await secretDAL.cacheInvalidateSecretByProjectId(projectId);
await snapshotService.performSnapshot(folderId);
await secretQueueService.syncSecrets({
actor,
@@ -1883,6 +1892,7 @@ export const secretV2BridgeServiceFactory = ({
}
});
await secretDAL.cacheInvalidateSecretByProjectId(projectId);
await Promise.allSettled(folders.map((el) => (el?.id ? snapshotService.performSnapshot(el.id) : undefined)));
await Promise.allSettled(
folders.map((el) =>
@@ -2014,6 +2024,7 @@ export const secretV2BridgeServiceFactory = ({
})
);
await secretDAL.cacheInvalidateSecretByProjectId(projectId);
await snapshotService.performSnapshot(folderId);
await secretQueueService.syncSecrets({
actor,
@@ -2537,6 +2548,9 @@ export const secretV2BridgeServiceFactory = ({
}
});
if (isDestinationUpdated || isSourceUpdated) {
await secretDAL.cacheInvalidateSecretByProjectId(projectId);
}
if (isDestinationUpdated) {
await snapshotService.performSnapshot(destinationFolder.id);
await secretQueueService.syncSecrets({
@@ -2715,7 +2729,7 @@ export const secretV2BridgeServiceFactory = ({
generatePaths(folderMap).map(({ folderId, path }) => [folderId, path === "/" ? path : path.substring(1)])
);
const secrets = await secretDAL.findByFolderIds(folders.map((f) => f.id));
const secrets = await secretDAL.findByFolderIds({ folderIds: folders.map((f) => f.id), projectId });
const { decryptor: secretManagerDecryptor } = await kmsService.createCipherPairWithDataKey({
type: KmsDataKey.SecretManager,
backend/src/services/secret-v2-bridge/secret-v2-bridge-types.ts
+7
View File
@@ -279,6 +279,13 @@ export type TUpdateManySecretsFnFactory = {
folderDAL: TSecretFolderDALFactory;
};
export type TFindByFolderIdDALDTO = {
folderId: string;
userId?: string;
tx?: Knex;
projectId: string;
};
export type TUpdateManySecretsFn = {
projectId: string;
environment: string;