feat(api): implemented secret caching version 2

This commit is contained in:
=
2025-04-23 19:15:50 +05:30
parent d7af9e84be
commit b60172f2be
10 changed files with 96 additions and 209 deletions

View File

@@ -267,7 +267,6 @@ export const secretReplicationServiceFactory = ({
const sourceLocalSecrets = await secretV2BridgeDAL.find({ folderId: folder.id, type: SecretType.Shared });
const sourceSecretImports = await secretImportDAL.find({ folderId: folder.id });
const sourceImportedSecrets = await fnSecretsV2FromImports({
projectId,
secretImports: sourceSecretImports,
secretDAL: secretV2BridgeDAL,
folderDAL,

View File

@@ -1089,7 +1089,8 @@ export const registerRoutes = async (
secretApprovalRequestSecretDAL,
kmsService,
snapshotService,
resourceMetadataDAL
resourceMetadataDAL,
keyStore
});
const secretApprovalRequestService = secretApprovalRequestServiceFactory({

View File

@@ -50,7 +50,7 @@ const getIntegrationSecretsV2 = async (
}
// process secrets in current folder
const secrets = await secretV2BridgeDAL.findByFolderId({ folderId: dto.folderId, projectId: dto.projectId });
const secrets = await secretV2BridgeDAL.findByFolderId({ folderId: dto.folderId });
secrets.forEach((secret) => {
const secretKey = secret.key;
@@ -63,7 +63,6 @@ const getIntegrationSecretsV2 = async (
// if no imports then return secrets in the current folder
if (!secretImports.length) return content;
const importedSecrets = await fnSecretsV2FromImports({
projectId: dto.projectId,
decryptor: dto.decryptor,
folderDAL,
secretDAL: secretV2BridgeDAL,

View File

@@ -159,8 +159,7 @@ export const fnSecretsV2FromImports = async ({
decryptor,
expandSecretReferences,
hasSecretAccess,
viewSecretValue,
projectId
viewSecretValue
}: {
secretImports: (Omit<TSecretImports, "importEnv"> & {
importEnv: { id: string; slug: string; name: string };
@@ -177,7 +176,6 @@ export const fnSecretsV2FromImports = async ({
environment: string;
}) => Promise<string | undefined>;
hasSecretAccess: (environment: string, secretPath: string, secretName: string, secretTagSlugs: string[]) => boolean;
projectId: string;
}) => {
const cyclicDetector = new Set();
const stack: {
@@ -218,8 +216,7 @@ export const fnSecretsV2FromImports = async ({
type: SecretType.Shared
},
{
sort: [["id", "asc"]],
useCache: { projectId }
sort: [["id", "asc"]]
}
);
const importedSecretsGroupByFolderId = groupBy(importedSecrets, (i) => i.folderId);

View File

@@ -698,7 +698,6 @@ export const secretImportServiceFactory = ({
projectId
});
const importedSecrets = await fnSecretsV2FromImports({
projectId,
secretImports,
folderDAL,
viewSecretValue: true,

View File

@@ -214,7 +214,7 @@ export const secretSyncQueueFactory = ({
canExpandValue: () => true
});
const secrets = await secretV2BridgeDAL.findByFolderId({ folderId, projectId });
const secrets = await secretV2BridgeDAL.findByFolderId({ folderId });
await Promise.allSettled(
secrets.map(async (secret) => {
@@ -244,7 +244,6 @@ export const secretSyncQueueFactory = ({
if (secretImports.length) {
const importedSecrets = await fnSecretsV2FromImports({
projectId,
decryptor: decryptSecretValue,
folderDAL,
secretDAL: secretV2BridgeDAL,

View File

@@ -15,9 +15,13 @@ import {
TFindFilter,
TFindOpt
} from "@app/lib/knex";
import { BufferKeysToString, OrderByDirection } from "@app/lib/types";
import { OrderByDirection } from "@app/lib/types";
import { SecretsOrderBy } from "@app/services/secret/secret-types";
import type { TFindSecretsByFolderIdsFilter } from "@app/services/secret-v2-bridge/secret-v2-bridge-types";
import type {
TFindSecretsByFolderIdsFilter,
TGetSecretsDTO
} from "@app/services/secret-v2-bridge/secret-v2-bridge-types";
import { MongoAbility } from "@casl/ability";
export const SecretDalCacheKeys = {
get productKey() {
@@ -27,34 +31,14 @@ export const SecretDalCacheKeys = {
getSecretDalVersion: (projectId: string) => {
return `${SecretDalCacheKeys.productKey}:${projectId}:${TableName.SecretV2}-dal-version`;
},
findByFolderIds: (
getSecretsOfServiceLayer: (
projectId: string,
version: number,
{ useCache, tx, ...cacheKey }: Parameters<TSecretV2BridgeDALFactory["findByFolderIds"]>[0]
dto: TGetSecretsDTO & { permissionRules: MongoAbility["rules"] }
) => {
return `${SecretDalCacheKeys.productKey}:${projectId}:${
TableName.SecretV2
}-dal:v${version}:find-by-folder-ids:${generateCacheKeyFromData(cacheKey)}`;
},
findByFolderId: (
projectId: string,
version: number,
{ useCache, tx, ...cacheKey }: Parameters<TSecretV2BridgeDALFactory["findByFolderId"]>[0]
) => {
return `${SecretDalCacheKeys.productKey}:${projectId}:${
TableName.SecretV2
}-dal:v${version}:find-by-folder-id:${generateCacheKeyFromData(cacheKey)}`;
},
find: (projectId: string, version: number, ...args: Parameters<TSecretV2BridgeDALFactory["find"]>) => {
const [filter, opts] = args;
delete opts?.tx;
delete opts?.useCache;
return `${SecretDalCacheKeys.productKey}:${projectId}:${
TableName.SecretV2
}-dal:v${version}:find:${generateCacheKeyFromData({
filter,
opts
})}`;
}-dal:v${version}:get-secrets-service-layer:${dto.actorId}-${generateCacheKeyFromData(dto)}`;
}
};
@@ -64,9 +48,9 @@ interface TSecretV2DalArg {
keyStore: TKeyStoreFactory;
}
const SECRET_DAL_TTL = 5 * 60;
const SECRET_DAL_VERSION_TTL = 15 * 60;
const MAX_SECRET_CACHE_BYTES = 25 * 1024 * 1024;
export const SECRET_DAL_TTL = 5 * 60;
export const SECRET_DAL_VERSION_TTL = 15 * 60;
export const MAX_SECRET_CACHE_BYTES = 25 * 1024 * 1024;
export const secretV2BridgeDALFactory = ({ db, keyStore }: TSecretV2DalArg) => {
const secretOrm = ormify(db, TableName.SecretV2);
@@ -128,35 +112,9 @@ export const secretV2BridgeDALFactory = ({ db, keyStore }: TSecretV2DalArg) => {
}
};
const find = async (
filter: TFindFilter<TSecretsV2>,
opts: TFindOpt<TSecretsV2> & { useCache?: { projectId: string } } = {}
) => {
const { offset, limit, sort, tx, useCache } = opts;
const find = async (filter: TFindFilter<TSecretsV2>, opts: TFindOpt<TSecretsV2> = {}) => {
const { offset, limit, sort, tx } = opts;
try {
let secretDalVersion = 0;
if (useCache) {
const cachedSecretDalVersion = await keyStore.getItem(
SecretDalCacheKeys.getSecretDalVersion(useCache.projectId)
);
secretDalVersion = Number(cachedSecretDalVersion || 0);
const cacheKey = SecretDalCacheKeys.find(useCache.projectId, secretDalVersion, filter, opts);
const cachedSecrets = await keyStore.getItem(cacheKey);
if (cachedSecrets) {
await keyStore.setExpiry(cacheKey, SECRET_DAL_TTL);
const unsanitizedSecrets = JSON.parse(cachedSecrets) as BufferKeysToString<(typeof data)[number]>[];
const sanitizedSecrets = unsanitizedSecrets.map((el) => {
const encryptedValue = el.encryptedValue ? Buffer.from(el.encryptedValue, "base64") : null;
const encryptedComment = el.encryptedComment ? Buffer.from(el.encryptedComment, "base64") : null;
const createdAt = new Date(el.createdAt);
const updatedAt = new Date(el.updatedAt);
return { ...el, encryptedComment, encryptedValue, createdAt, updatedAt };
});
return sanitizedSecrets;
}
}
const query = (tx || db)(TableName.SecretV2)
// eslint-disable-next-line @typescript-eslint/no-misused-promises
.where(buildFindFilter(filter))
@@ -225,22 +183,6 @@ export const secretV2BridgeDALFactory = ({ db, keyStore }: TSecretV2DalArg) => {
]
});
if (useCache) {
const cachedSecrets = data.map((el) => {
const encryptedValue = el.encryptedValue ? el.encryptedValue.toString("base64") : null;
const encryptedComment = el.encryptedComment ? el.encryptedComment.toString("base64") : null;
return { ...el, encryptedValue, encryptedComment };
});
const cache = JSON.stringify(cachedSecrets);
if (Buffer.byteLength(cache, "utf8") < MAX_SECRET_CACHE_BYTES) {
await keyStore.setItemWithExpiry(
SecretDalCacheKeys.find(useCache.projectId, secretDalVersion, filter, opts),
SECRET_DAL_TTL,
cache
);
}
}
return data;
} catch (error) {
throw new DatabaseError({ error, name: `${TableName.SecretV2}: Find` });
@@ -345,15 +287,9 @@ export const secretV2BridgeDALFactory = ({ db, keyStore }: TSecretV2DalArg) => {
}
};
const findByFolderId = async (dto: {
folderId: string;
userId?: string;
tx?: Knex;
projectId: string;
useCache?: boolean;
}) => {
const findByFolderId = async (dto: { folderId: string; userId?: string; tx?: Knex; useCache?: boolean }) => {
try {
const { folderId, tx, projectId } = dto;
const { folderId, tx } = dto;
let { userId } = dto;
// check if not uui then userId id is null (corner case because service token's ID is not UUI in effort to keep backwards compatibility from mongo
if (userId && !uuidValidate(userId)) {
@@ -361,27 +297,6 @@ export const secretV2BridgeDALFactory = ({ db, keyStore }: TSecretV2DalArg) => {
userId = undefined;
}
const cachedSecretDalVersion = await keyStore.getItem(SecretDalCacheKeys.getSecretDalVersion(projectId));
const secretDalVersion = Number(cachedSecretDalVersion || 0);
if (dto.useCache) {
const cacheKey = SecretDalCacheKeys.findByFolderId(projectId, secretDalVersion, dto);
const cachedSecrets = await keyStore.getItem(cacheKey);
if (cachedSecrets) {
await keyStore.setExpiry(cacheKey, SECRET_DAL_TTL);
const unsanitizedSecrets = JSON.parse(cachedSecrets) as BufferKeysToString<(typeof data)[number]>[];
const sanitizedSecrets = unsanitizedSecrets.map((el) => {
const encryptedValue = el.encryptedValue ? Buffer.from(el.encryptedValue, "base64") : null;
const encryptedComment = el.encryptedComment ? Buffer.from(el.encryptedComment, "base64") : null;
const createdAt = new Date(el.createdAt);
const updatedAt = new Date(el.updatedAt);
return { ...el, encryptedComment, encryptedValue, createdAt, updatedAt };
});
return sanitizedSecrets;
}
}
const secs = await (tx || db.replicaNode())(TableName.SecretV2)
.where({ folderId })
.where((bd) => {
@@ -437,22 +352,6 @@ export const secretV2BridgeDALFactory = ({ db, keyStore }: TSecretV2DalArg) => {
}
]
});
if (dto.useCache) {
const newCachedSecrets = data.map((el) => {
const encryptedValue = el.encryptedValue ? el.encryptedValue.toString("base64") : null;
const encryptedComment = el.encryptedComment ? el.encryptedComment.toString("base64") : null;
return { ...el, encryptedValue, encryptedComment };
});
const cache = JSON.stringify(newCachedSecrets);
if (Buffer.byteLength(cache, "utf8") < MAX_SECRET_CACHE_BYTES) {
await keyStore.setItemWithExpiry(
SecretDalCacheKeys.findByFolderId(projectId, secretDalVersion, dto),
SECRET_DAL_TTL,
cache
);
}
}
return data;
} catch (error) {
throw new DatabaseError({ error, name: "get all secret" });
@@ -542,11 +441,9 @@ export const secretV2BridgeDALFactory = ({ db, keyStore }: TSecretV2DalArg) => {
folderIds: string[];
userId?: string;
tx?: Knex;
projectId: string;
filters?: TFindSecretsByFolderIdsFilter;
useCache?: boolean;
}) => {
const { folderIds, tx, filters, useCache, projectId } = dto;
const { folderIds, tx, filters } = dto;
let { userId } = dto;
try {
// check if not uui then userId id is null (corner case because service token's ID is not UUI in effort to keep backwards compatibility from mongo)
@@ -555,26 +452,6 @@ export const secretV2BridgeDALFactory = ({ db, keyStore }: TSecretV2DalArg) => {
userId = undefined;
}
const cachedSecretDalVersion = await keyStore.getItem(SecretDalCacheKeys.getSecretDalVersion(projectId));
const secretDalVersion = Number(cachedSecretDalVersion || 0);
if (useCache) {
const cacheKey = SecretDalCacheKeys.findByFolderIds(projectId, secretDalVersion, dto);
const cachedSecrets = await keyStore.getItem(cacheKey);
if (cachedSecrets) {
await keyStore.setExpiry(cacheKey, SECRET_DAL_TTL);
const unsanitizedSecrets = JSON.parse(cachedSecrets) as BufferKeysToString<(typeof data)[number]>[];
const sanitizedSecrets = unsanitizedSecrets.map((el) => {
const encryptedValue = el.encryptedValue ? Buffer.from(el.encryptedValue, "base64") : null;
const encryptedComment = el.encryptedComment ? Buffer.from(el.encryptedComment, "base64") : null;
const createdAt = new Date(el.createdAt);
const updatedAt = new Date(el.updatedAt);
return { ...el, encryptedComment, encryptedValue, createdAt, updatedAt };
});
return sanitizedSecrets;
}
}
const query = (tx || db.replicaNode())(TableName.SecretV2)
.whereIn(`${TableName.SecretV2}.folderId`, folderIds)
.where((bd) => {
@@ -700,22 +577,6 @@ export const secretV2BridgeDALFactory = ({ db, keyStore }: TSecretV2DalArg) => {
}
]
});
if (useCache) {
const cachedSecrets = data.map((el) => {
const encryptedValue = el.encryptedValue ? el.encryptedValue.toString("base64") : null;
const encryptedComment = el.encryptedComment ? el.encryptedComment.toString("base64") : null;
return { ...el, encryptedValue, encryptedComment };
});
const cache = JSON.stringify(cachedSecrets);
if (Buffer.byteLength(cache, "utf8") < MAX_SECRET_CACHE_BYTES) {
await keyStore.setItemWithExpiry(
SecretDalCacheKeys.findByFolderIds(projectId, secretDalVersion, dto),
SECRET_DAL_TTL,
cache
);
}
}
return data;
} catch (error) {

View File

@@ -509,7 +509,7 @@ export const expandSecretReferencesFactory = ({
const folder = await folderDAL.findBySecretPath(projectId, environment, secretPath);
if (!folder) return { value: "", tags: [] };
const secrets = await secretDAL.findByFolderId({ folderId: folder.id, projectId, useCache: true });
const secrets = await secretDAL.findByFolderId({ folderId: folder.id });
const decryptedSecret = secrets.reduce<Record<string, { value: string; tags: string[] }>>((prev, secret) => {
// eslint-disable-next-line no-param-reassign

View File

@@ -43,7 +43,12 @@ import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal";
import { TSecretImportDALFactory } from "../secret-import/secret-import-dal";
import { fnSecretsV2FromImports } from "../secret-import/secret-import-fns";
import { TSecretTagDALFactory } from "../secret-tag/secret-tag-dal";
import { TSecretV2BridgeDALFactory } from "./secret-v2-bridge-dal";
import {
MAX_SECRET_CACHE_BYTES,
SECRET_DAL_TTL,
SecretDalCacheKeys,
TSecretV2BridgeDALFactory
} from "./secret-v2-bridge-dal";
import {
buildHierarchy,
expandSecretReferencesFactory,
@@ -76,6 +81,7 @@ import {
} from "./secret-v2-bridge-types";
import { TSecretVersionV2DALFactory } from "./secret-version-dal";
import { TSecretVersionV2TagDALFactory } from "./secret-version-tag-dal";
import { TKeyStoreFactory } from "@app/keystore/keystore";
type TSecretV2BridgeServiceFactoryDep = {
secretDAL: TSecretV2BridgeDALFactory;
@@ -105,6 +111,7 @@ type TSecretV2BridgeServiceFactoryDep = {
>;
snapshotService: Pick<TSecretSnapshotServiceFactory, "performSnapshot">;
resourceMetadataDAL: Pick<TResourceMetadataDALFactory, "insertMany" | "delete">;
keyStore: Pick<TKeyStoreFactory, "getItem" | "setExpiry" | "setItemWithExpiry">;
};
export type TSecretV2BridgeServiceFactory = ReturnType<typeof secretV2BridgeServiceFactory>;
@@ -127,7 +134,8 @@ export const secretV2BridgeServiceFactory = ({
secretApprovalRequestDAL,
secretApprovalRequestSecretDAL,
kmsService,
resourceMetadataDAL
resourceMetadataDAL,
keyStore
}: TSecretV2BridgeServiceFactoryDep) => {
const $validateSecretReferences = async (
projectId: string,
@@ -800,12 +808,10 @@ export const secretV2BridgeServiceFactory = ({
const groupedFolderMappings = groupBy(folderMappings, (folderMapping) => folderMapping.folderId);
const secrets = await secretDAL.findByFolderIds({
projectId,
folderIds: folderMappings.map((folderMapping) => folderMapping.folderId),
userId,
tx: undefined,
filters,
useCache: true
filters
});
const { decryptor: secretManagerDecryptor } = await kmsService.createCipherPairWithDataKey({
@@ -909,21 +915,22 @@ export const secretV2BridgeServiceFactory = ({
return decryptedSecrets;
};
const getSecrets = async ({
actorId,
path,
environment,
projectId,
actor,
actorOrgId,
viewSecretValue,
actorAuthMethod,
includeImports,
recursive,
expandSecretReferences: shouldExpandSecretReferences,
throwOnMissingReadValuePermission = true,
...params
}: TGetSecretsDTO) => {
const getSecrets = async (dto: TGetSecretsDTO) => {
const {
actorId,
path,
environment,
projectId,
actor,
actorOrgId,
viewSecretValue,
actorAuthMethod,
includeImports,
recursive,
expandSecretReferences: shouldExpandSecretReferences,
throwOnMissingReadValuePermission = true,
...params
} = dto;
const { permission } = await permissionService.getProjectPermission({
actor,
actorId,
@@ -934,6 +941,32 @@ export const secretV2BridgeServiceFactory = ({
});
throwIfMissingSecretReadValueOrDescribePermission(permission, ProjectPermissionSecretActions.DescribeSecret);
const cachedSecretDalVersion = await keyStore.getItem(SecretDalCacheKeys.getSecretDalVersion(projectId));
const secretDalVersion = Number(cachedSecretDalVersion || 0);
const cacheKey = SecretDalCacheKeys.getSecretsOfServiceLayer(projectId, secretDalVersion, {
...dto,
permissionRules: permission.rules
});
const { decryptor: secretManagerDecryptor, encryptor: secretManagerEncryptor } =
await kmsService.createCipherPairWithDataKey({
type: KmsDataKey.SecretManager,
projectId
});
const encryptedCachedSecrets = await keyStore.getItem(cacheKey);
if (encryptedCachedSecrets) {
await keyStore.setExpiry(cacheKey, SECRET_DAL_TTL);
const cachedSecrets = secretManagerDecryptor({ cipherTextBlob: Buffer.from(encryptedCachedSecrets, "base64") });
const { secrets, imports = [] } = JSON.parse(cachedSecrets.toString("utf8")) as {
secrets: typeof decryptedSecrets;
imports: typeof importedSecrets;
};
return {
secrets: secrets.map((el) => ({ ...el, createdAt: new Date(el.createdAt), updatedAt: new Date(el.updatedAt) })),
imports
};
}
let paths: { folderId: string; path: string }[] = [];
if (recursive) {
@@ -958,17 +991,10 @@ export const secretV2BridgeServiceFactory = ({
const groupedPaths = groupBy(paths, (p) => p.folderId);
const secrets = await secretDAL.findByFolderIds({
projectId,
folderIds: paths.map((p) => p.folderId),
userId: actorId,
tx: undefined,
filters: params,
useCache: true
});
const { decryptor: secretManagerDecryptor } = await kmsService.createCipherPairWithDataKey({
type: KmsDataKey.SecretManager,
projectId
filters: params
});
// scott: if any of this changes it also needs to be mirrored in secret rotation for getting dashboard secrets
@@ -1086,15 +1112,19 @@ export const secretV2BridgeServiceFactory = ({
}
if (!includeImports) {
return {
secrets: decryptedSecrets
};
const payload = { secrets: decryptedSecrets, imports: [] };
const encryptedUpdatedCachedSecrets = secretManagerEncryptor({
plainText: Buffer.from(JSON.stringify(payload))
}).cipherTextBlob;
if (encryptedUpdatedCachedSecrets.byteLength < MAX_SECRET_CACHE_BYTES) {
await keyStore.setItemWithExpiry(cacheKey, SECRET_DAL_TTL, encryptedUpdatedCachedSecrets.toString("base64"));
}
return payload;
}
const secretImports = await secretImportDAL.findByFolderIds(paths.map((p) => p.folderId));
const allowedImports = secretImports.filter(({ isReplication }) => !isReplication);
const importedSecrets = await fnSecretsV2FromImports({
projectId,
viewSecretValue,
secretImports: allowedImports,
secretDAL,
@@ -1129,10 +1159,14 @@ export const secretV2BridgeServiceFactory = ({
}
});
return {
secrets: decryptedSecrets,
imports: importedSecrets
};
const payload = { secrets: decryptedSecrets, imports: importedSecrets };
const encryptedUpdatedCachedSecrets = secretManagerEncryptor({
plainText: Buffer.from(JSON.stringify(payload))
}).cipherTextBlob;
if (encryptedUpdatedCachedSecrets.byteLength < MAX_SECRET_CACHE_BYTES) {
await keyStore.setItemWithExpiry(cacheKey, SECRET_DAL_TTL, encryptedUpdatedCachedSecrets.toString("base64"));
}
return payload;
};
const getSecretById = async ({ actorId, actor, actorOrgId, actorAuthMethod, secretId }: TGetASecretByIdDTO) => {
@@ -1312,7 +1346,6 @@ export const secretV2BridgeServiceFactory = ({
if (!secret && includeImports) {
const secretImports = await secretImportDAL.find({ folderId, isReplication: false });
const importedSecrets = await fnSecretsV2FromImports({
projectId,
secretImports,
viewSecretValue,
secretDAL,
@@ -2729,7 +2762,7 @@ export const secretV2BridgeServiceFactory = ({
generatePaths(folderMap).map(({ folderId, path }) => [folderId, path === "/" ? path : path.substring(1)])
);
const secrets = await secretDAL.findByFolderIds({ folderIds: folders.map((f) => f.id), projectId, useCache: true });
const secrets = await secretDAL.findByFolderIds({ folderIds: folders.map((f) => f.id) });
const { decryptor: secretManagerDecryptor } = await kmsService.createCipherPairWithDataKey({
type: KmsDataKey.SecretManager,

View File

@@ -367,7 +367,7 @@ export const secretQueueFactory = ({
canExpandValue: () => true
});
// process secrets in current folder
const secrets = await secretV2BridgeDAL.findByFolderId({ folderId: dto.folderId, projectId: dto.projectId });
const secrets = await secretV2BridgeDAL.findByFolderId({ folderId: dto.folderId });
await Promise.allSettled(
secrets.map(async (secret) => {
@@ -397,7 +397,6 @@ export const secretQueueFactory = ({
// if no imports then return secrets in the current folder
if (!secretImports.length) return content;
const importedSecrets = await fnSecretsV2FromImports({
projectId: dto.projectId,
decryptor: dto.decryptor,
folderDAL,
secretDAL: secretV2BridgeDAL,