diff --git a/backend/e2e-test/mocks/keystore.ts b/backend/e2e-test/mocks/keystore.ts index 05753995c..48f52f9e7 100644 --- a/backend/e2e-test/mocks/keystore.ts +++ b/backend/e2e-test/mocks/keystore.ts @@ -9,6 +9,7 @@ export const mockKeyStore = (): TKeyStoreFactory => { store[key] = value; return "OK"; }, + setExpiry: async () => 0, setItemWithExpiry: async (key, value) => { store[key] = value; return "OK"; diff --git a/backend/src/ee/services/secret-approval-request/secret-approval-request-service.ts b/backend/src/ee/services/secret-approval-request/secret-approval-request-service.ts index 4a8e74532..29bf64972 100644 --- a/backend/src/ee/services/secret-approval-request/secret-approval-request-service.ts +++ b/backend/src/ee/services/secret-approval-request/secret-approval-request-service.ts @@ -113,7 +113,13 @@ type TSecretApprovalRequestServiceFactoryDep = { kmsService: Pick; secretV2BridgeDAL: Pick< TSecretV2BridgeDALFactory, - "insertMany" | "upsertSecretReferences" | "findBySecretKeys" | "bulkUpdate" | "deleteMany" | "find" + | "insertMany" + | "upsertSecretReferences" + | "findBySecretKeys" + | "bulkUpdate" + | "deleteMany" + | "find" + | "cacheInvalidateSecretByProjectId" >; secretVersionV2BridgeDAL: Pick; secretVersionTagV2BridgeDAL: Pick; @@ -864,6 +870,7 @@ export const secretApprovalRequestServiceFactory = ({ }); } + await secretV2BridgeDAL.cacheInvalidateSecretByProjectId(projectId); await snapshotService.performSnapshot(folderId); const [folder] = await folderDAL.findSecretPathByFolderIds(projectId, [folderId]); if (!folder) { diff --git a/backend/src/ee/services/secret-replication/secret-replication-service.ts b/backend/src/ee/services/secret-replication/secret-replication-service.ts index 5fae2675d..e4e79027b 100644 --- a/backend/src/ee/services/secret-replication/secret-replication-service.ts +++ b/backend/src/ee/services/secret-replication/secret-replication-service.ts @@ -45,7 +45,14 @@ type TSecretReplicationServiceFactoryDep = { secretVersionDAL: Pick; secretV2BridgeDAL: Pick< TSecretV2BridgeDALFactory, - "find" | "findBySecretKeys" | "insertMany" | "bulkUpdate" | "delete" | "upsertSecretReferences" | "transaction" + | "find" + | "findBySecretKeys" + | "insertMany" + | "bulkUpdate" + | "delete" + | "upsertSecretReferences" + | "transaction" + | "cacheInvalidateSecretByProjectId" >; secretVersionV2BridgeDAL: Pick< TSecretVersionV2DALFactory, @@ -260,6 +267,7 @@ export const secretReplicationServiceFactory = ({ const sourceLocalSecrets = await secretV2BridgeDAL.find({ folderId: folder.id, type: SecretType.Shared }); const sourceSecretImports = await secretImportDAL.find({ folderId: folder.id }); const sourceImportedSecrets = await fnSecretsV2FromImports({ + projectId, secretImports: sourceSecretImports, secretDAL: secretV2BridgeDAL, folderDAL, @@ -783,6 +791,7 @@ export const secretReplicationServiceFactory = ({ } }); + await secretV2BridgeDAL.cacheInvalidateSecretByProjectId(projectId); await secretQueueService.syncSecrets({ projectId, orgId, diff --git a/backend/src/ee/services/secret-rotation-v2/secret-rotation-v2-service.ts b/backend/src/ee/services/secret-rotation-v2/secret-rotation-v2-service.ts index d0b38808e..6476349c4 100644 --- a/backend/src/ee/services/secret-rotation-v2/secret-rotation-v2-service.ts +++ b/backend/src/ee/services/secret-rotation-v2/secret-rotation-v2-service.ts @@ -88,7 +88,7 @@ export type TSecretRotationV2ServiceFactoryDep = { folderDAL: Pick; secretV2BridgeDAL: Pick< TSecretV2BridgeDALFactory, - "bulkUpdate" | "insertMany" | "deleteMany" | "upsertSecretReferences" | "find" + "bulkUpdate" | "insertMany" | "deleteMany" | "upsertSecretReferences" | "find" | "cacheInvalidateSecretByProjectId" >; secretVersionV2BridgeDAL: Pick; secretVersionTagV2BridgeDAL: Pick; @@ -515,6 +515,7 @@ export const secretRotationV2ServiceFactory = ({ }); }); + await secretV2BridgeDAL.cacheInvalidateSecretByProjectId(projectId); await snapshotService.performSnapshot(folder.id); await secretQueueService.syncSecrets({ orgId: connection.orgId, @@ -651,6 +652,7 @@ export const secretRotationV2ServiceFactory = ({ }); if (secretsMappingUpdated) { + await secretV2BridgeDAL.cacheInvalidateSecretByProjectId(projectId); await snapshotService.performSnapshot(folder.id); await secretQueueService.syncSecrets({ orgId: connection.orgId, @@ -777,6 +779,7 @@ export const secretRotationV2ServiceFactory = ({ } if (deleteSecrets) { + await secretV2BridgeDAL.cacheInvalidateSecretByProjectId(projectId); await snapshotService.performSnapshot(folder.id); await secretQueueService.syncSecrets({ orgId: connection.orgId, @@ -935,6 +938,7 @@ export const secretRotationV2ServiceFactory = ({ } }); + await secretV2BridgeDAL.cacheInvalidateSecretByProjectId(projectId); await snapshotService.performSnapshot(folder.id); await secretQueueService.syncSecrets({ orgId: connection.orgId, diff --git a/backend/src/keystore/keystore.ts b/backend/src/keystore/keystore.ts index 8fef532f5..ac28e9ade 100644 --- a/backend/src/keystore/keystore.ts +++ b/backend/src/keystore/keystore.ts @@ -77,6 +77,8 @@ export const keyStoreFactory = (redisUrl: string) => { const incrementBy = async (key: string, value: number) => redis.incrby(key, value); + const setExpiry = async (key: string, expiryInSeconds: number) => redis.expire(key, expiryInSeconds); + const waitTillReady = async ({ key, waitingCb, @@ -103,6 +105,7 @@ export const keyStoreFactory = (redisUrl: string) => { return { setItem, getItem, + setExpiry, setItemWithExpiry, deleteItem, incrementBy, diff --git a/backend/src/keystore/memory.ts b/backend/src/keystore/memory.ts index 1fe78cf7e..10b28ffec 100644 --- a/backend/src/keystore/memory.ts +++ b/backend/src/keystore/memory.ts @@ -10,6 +10,7 @@ export const inMemoryKeyStore = (): TKeyStoreFactory => { store[key] = value; return "OK"; }, + setExpiry: async () => 0, setItemWithExpiry: async (key, value) => { store[key] = value; return "OK"; diff --git a/backend/src/lib/crypto/cache.ts b/backend/src/lib/crypto/cache.ts new file mode 100644 index 000000000..eafac8712 --- /dev/null +++ b/backend/src/lib/crypto/cache.ts @@ -0,0 +1,10 @@ +import crypto from "node:crypto"; + +export const getCacheKey = (data: unknown) => + crypto + .createHash("md5") + .update(JSON.stringify(data)) + .digest("base64") + .replace(/\+/g, "-") + .replace(/\//g, "_") + .replace(/=/g, ""); diff --git a/backend/src/lib/types/index.ts b/backend/src/lib/types/index.ts index b8b272017..9f063172f 100644 --- a/backend/src/lib/types/index.ts +++ b/backend/src/lib/types/index.ts @@ -41,6 +41,18 @@ export type RequiredKeys = { [K in keyof T]-?: undefined extends T[K] ? never : K; }[keyof T]; +export type BufferKeysToString = { + [K in keyof T]: T[K] extends Buffer + ? string + : T[K] extends Buffer | null + ? string | null + : T[K] extends Buffer | undefined + ? string | undefined + : T[K] extends Buffer | null | undefined + ? string | null | undefined + : T[K]; +}; + export type PickRequired = Pick>; export type DiscriminativePick = T extends unknown ? Pick : never; diff --git a/backend/src/server/routes/index.ts b/backend/src/server/routes/index.ts index 595a9626c..743577d25 100644 --- a/backend/src/server/routes/index.ts +++ b/backend/src/server/routes/index.ts @@ -315,7 +315,7 @@ export const registerRoutes = async ( const secretVersionTagDAL = secretVersionTagDALFactory(db); const secretBlindIndexDAL = secretBlindIndexDALFactory(db); - const secretV2BridgeDAL = secretV2BridgeDALFactory(db); + const secretV2BridgeDAL = secretV2BridgeDALFactory({ db, keyStore }); const secretVersionV2BridgeDAL = secretVersionV2BridgeDALFactory(db); const secretVersionTagV2BridgeDAL = secretVersionV2TagBridgeDALFactory(db); diff --git a/backend/src/services/integration-auth/integration-delete-secret.ts b/backend/src/services/integration-auth/integration-delete-secret.ts index 4b07245ac..18406f29a 100644 --- a/backend/src/services/integration-auth/integration-delete-secret.ts +++ b/backend/src/services/integration-auth/integration-delete-secret.ts @@ -50,7 +50,7 @@ const getIntegrationSecretsV2 = async ( } // process secrets in current folder - const secrets = await secretV2BridgeDAL.findByFolderId(dto.folderId); + const secrets = await secretV2BridgeDAL.findByFolderId({ folderId: dto.folderId, projectId: dto.projectId }); secrets.forEach((secret) => { const secretKey = secret.key; @@ -63,6 +63,7 @@ const getIntegrationSecretsV2 = async ( // if no imports then return secrets in the current folder if (!secretImports.length) return content; const importedSecrets = await fnSecretsV2FromImports({ + projectId: dto.projectId, decryptor: dto.decryptor, folderDAL, secretDAL: secretV2BridgeDAL, diff --git a/backend/src/services/secret-import/secret-import-fns.ts b/backend/src/services/secret-import/secret-import-fns.ts index e5a450441..2056d5a2c 100644 --- a/backend/src/services/secret-import/secret-import-fns.ts +++ b/backend/src/services/secret-import/secret-import-fns.ts @@ -159,7 +159,8 @@ export const fnSecretsV2FromImports = async ({ decryptor, expandSecretReferences, hasSecretAccess, - viewSecretValue + viewSecretValue, + projectId }: { secretImports: (Omit & { importEnv: { id: string; slug: string; name: string }; @@ -176,6 +177,7 @@ export const fnSecretsV2FromImports = async ({ environment: string; }) => Promise; hasSecretAccess: (environment: string, secretPath: string, secretName: string, secretTagSlugs: string[]) => boolean; + projectId: string; }) => { const cyclicDetector = new Set(); const stack: { @@ -216,7 +218,8 @@ export const fnSecretsV2FromImports = async ({ type: SecretType.Shared }, { - sort: [["id", "asc"]] + sort: [["id", "asc"]], + useCache: { projectId } } ); const importedSecretsGroupByFolderId = groupBy(importedSecrets, (i) => i.folderId); diff --git a/backend/src/services/secret-import/secret-import-service.ts b/backend/src/services/secret-import/secret-import-service.ts index fdd326b5f..393036eab 100644 --- a/backend/src/services/secret-import/secret-import-service.ts +++ b/backend/src/services/secret-import/secret-import-service.ts @@ -43,7 +43,7 @@ type TSecretImportServiceFactoryDep = { secretImportDAL: TSecretImportDALFactory; folderDAL: TSecretFolderDALFactory; secretDAL: Pick; - secretV2BridgeDAL: Pick; + secretV2BridgeDAL: Pick; projectBotService: Pick; projectDAL: Pick; projectEnvDAL: TProjectEnvDALFactory; @@ -184,6 +184,7 @@ export const secretImportServiceFactory = ({ }); } + await secretV2BridgeDAL.cacheInvalidateSecretByProjectId(projectId); return { ...secImport, importEnv }; }; @@ -281,6 +282,8 @@ export const secretImportServiceFactory = ({ ); return doc; }); + + await secretV2BridgeDAL.cacheInvalidateSecretByProjectId(projectId); return { ...updatedSecImport, importEnv: importedEnv }; }; @@ -355,6 +358,7 @@ export const secretImportServiceFactory = ({ actorId }); + await secretV2BridgeDAL.cacheInvalidateSecretByProjectId(projectId); return secImport; }; @@ -693,6 +697,7 @@ export const secretImportServiceFactory = ({ projectId }); const importedSecrets = await fnSecretsV2FromImports({ + projectId, secretImports, folderDAL, viewSecretValue: true, diff --git a/backend/src/services/secret-sync/secret-sync-queue.ts b/backend/src/services/secret-sync/secret-sync-queue.ts index 8afcf6416..17257b8e2 100644 --- a/backend/src/services/secret-sync/secret-sync-queue.ts +++ b/backend/src/services/secret-sync/secret-sync-queue.ts @@ -213,7 +213,7 @@ export const secretSyncQueueFactory = ({ canExpandValue: () => true }); - const secrets = await secretV2BridgeDAL.findByFolderId(folderId); + const secrets = await secretV2BridgeDAL.findByFolderId({ folderId, projectId }); await Promise.allSettled( secrets.map(async (secret) => { @@ -243,6 +243,7 @@ export const secretSyncQueueFactory = ({ if (secretImports.length) { const importedSecrets = await fnSecretsV2FromImports({ + projectId, decryptor: decryptSecretValue, folderDAL, secretDAL: secretV2BridgeDAL, diff --git a/backend/src/services/secret-v2-bridge/secret-v2-bridge-dal.ts b/backend/src/services/secret-v2-bridge/secret-v2-bridge-dal.ts index 9bed01637..148e1e82b 100644 --- a/backend/src/services/secret-v2-bridge/secret-v2-bridge-dal.ts +++ b/backend/src/services/secret-v2-bridge/secret-v2-bridge-dal.ts @@ -2,7 +2,10 @@ import { Knex } from "knex"; import { validate as uuidValidate } from "uuid"; import { TDbClient } from "@app/db"; -import { SecretsV2Schema, SecretType, TableName, TSecretsV2, TSecretsV2Update } from "@app/db/schemas"; +import { ProjectType, SecretsV2Schema, SecretType, TableName, TSecretsV2, TSecretsV2Update } from "@app/db/schemas"; +import { TKeyStoreFactory } from "@app/keystore/keystore"; +import { getConfig } from "@app/lib/config/env"; +import { getCacheKey } from "@app/lib/crypto/cache"; import { BadRequestError, DatabaseError, NotFoundError } from "@app/lib/errors"; import { buildFindFilter, @@ -12,15 +15,64 @@ import { TFindFilter, TFindOpt } from "@app/lib/knex"; -import { OrderByDirection } from "@app/lib/types"; +import { BufferKeysToString, OrderByDirection } from "@app/lib/types"; import { SecretsOrderBy } from "@app/services/secret/secret-types"; -import { TFindSecretsByFolderIdsFilter } from "@app/services/secret-v2-bridge/secret-v2-bridge-types"; +import type { TFindSecretsByFolderIdsFilter } from "@app/services/secret-v2-bridge/secret-v2-bridge-types"; + +export const SecretDalCacheKeys = { + get productKey() { + const { INFISICAL_PLATFORM_VERSION } = getConfig(); + return `${ProjectType.SecretManager}:${INFISICAL_PLATFORM_VERSION || 0}`; + }, + getSecretDalVersion: (projectId: string) => { + return `${SecretDalCacheKeys.productKey}:${projectId}:${TableName.SecretV2}-dal-version`; + }, + findByFolderIds: ( + projectId: string, + version: number, + { useCache, tx, ...cacheKey }: Parameters[0] + ) => { + return `${SecretDalCacheKeys.productKey}:${projectId}:${ + TableName.SecretV2 + }-dal:v${version}:find-by-folder-ids:${getCacheKey(cacheKey)}`; + }, + findByFolderId: ( + projectId: string, + version: number, + { useCache, tx, ...cacheKey }: Parameters[0] + ) => { + return `${SecretDalCacheKeys.productKey}:${projectId}:${ + TableName.SecretV2 + }-dal:v${version}:find-by-folder-id:${getCacheKey(cacheKey)}`; + }, + find: (projectId: string, version: number, ...args: Parameters) => { + const [filter, opts] = args; + delete opts?.tx; + delete opts?.useCache; + return `${SecretDalCacheKeys.productKey}:${projectId}:${TableName.SecretV2}-dal:v${version}:find:${getCacheKey({ + filter, + opts + })}`; + } +}; export type TSecretV2BridgeDALFactory = ReturnType; +interface TSecretV2DalArg { + db: TDbClient; + keyStore: TKeyStoreFactory; +} -export const secretV2BridgeDALFactory = (db: TDbClient) => { +const SECRET_DAL_TTL = 5 * 60; +const SECRET_DAL_VERSION_TTL = 15 * 60; +export const secretV2BridgeDALFactory = ({ db, keyStore }: TSecretV2DalArg) => { const secretOrm = ormify(db, TableName.SecretV2); + const cacheInvalidateSecretByProjectId = async (projectId: string) => { + const secretDalVersionKey = SecretDalCacheKeys.getSecretDalVersion(projectId); + await keyStore.incrementBy(secretDalVersionKey, 1); + await keyStore.setExpiry(secretDalVersionKey, SECRET_DAL_VERSION_TTL); + }; + const findOne = async (filter: Partial, tx?: Knex) => { try { const docs = await (tx || db)(TableName.SecretV2) @@ -73,8 +125,35 @@ export const secretV2BridgeDALFactory = (db: TDbClient) => { } }; - const find = async (filter: TFindFilter, { offset, limit, sort, tx }: TFindOpt = {}) => { + const find = async ( + filter: TFindFilter, + opts: TFindOpt & { useCache?: { projectId: string } } = {} + ) => { + const { offset, limit, sort, tx, useCache } = opts; try { + let secretDalVersion = 0; + if (useCache) { + const cachedSecretDalVersion = await keyStore.getItem( + SecretDalCacheKeys.getSecretDalVersion(useCache.projectId) + ); + secretDalVersion = Number(cachedSecretDalVersion || 0); + const cacheKey = SecretDalCacheKeys.find(useCache.projectId, secretDalVersion, filter, opts); + const cachedSecrets = await keyStore.getItem(cacheKey); + if (cachedSecrets) { + await keyStore.setExpiry(cacheKey, SECRET_DAL_TTL); + + const unsanitizedSecrets = JSON.parse(cachedSecrets) as BufferKeysToString<(typeof data)[number]>[]; + const sanitizedSecrets = unsanitizedSecrets.map((el) => { + const encryptedValue = el.encryptedValue ? Buffer.from(el.encryptedValue, "base64") : null; + const encryptedComment = el.encryptedComment ? Buffer.from(el.encryptedComment, "base64") : null; + const createdAt = new Date(el.createdAt); + const updatedAt = new Date(el.updatedAt); + return { ...el, encryptedComment, encryptedValue, createdAt, updatedAt }; + }); + return sanitizedSecrets; + } + } + const query = (tx || db)(TableName.SecretV2) // eslint-disable-next-line @typescript-eslint/no-misused-promises .where(buildFindFilter(filter)) @@ -142,6 +221,20 @@ export const secretV2BridgeDALFactory = (db: TDbClient) => { } ] }); + + if (useCache) { + const cachedSecrets = data.map((el) => { + const encryptedValue = el.encryptedValue ? el.encryptedValue.toString("base64") : null; + const encryptedComment = el.encryptedComment ? el.encryptedComment.toString("base64") : null; + return { ...el, encryptedValue, encryptedComment }; + }); + await keyStore.setItemWithExpiry( + SecretDalCacheKeys.find(useCache.projectId, secretDalVersion, filter, opts), + SECRET_DAL_TTL, + JSON.stringify(cachedSecrets) + ); + } + return data; } catch (error) { throw new DatabaseError({ error, name: `${TableName.SecretV2}: Find` }); @@ -246,14 +339,43 @@ export const secretV2BridgeDALFactory = (db: TDbClient) => { } }; - const findByFolderId = async (folderId: string, userId?: string, tx?: Knex) => { + const findByFolderId = async (dto: { + folderId: string; + userId?: string; + tx?: Knex; + projectId: string; + useCache?: boolean; + }) => { try { - // check if not uui then userId id is null (corner case because service token's ID is not UUI in effort to keep backwards compatibility from mongo) + const { folderId, tx, projectId } = dto; + let { userId } = dto; + // check if not uui then userId id is null (corner case because service token's ID is not UUI in effort to keep backwards compatibility from mongo if (userId && !uuidValidate(userId)) { // eslint-disable-next-line userId = undefined; } + const cachedSecretDalVersion = await keyStore.getItem(SecretDalCacheKeys.getSecretDalVersion(projectId)); + const secretDalVersion = Number(cachedSecretDalVersion || 0); + + if (dto.useCache) { + const cacheKey = SecretDalCacheKeys.findByFolderId(projectId, secretDalVersion, dto); + const cachedSecrets = await keyStore.getItem(cacheKey); + if (cachedSecrets) { + await keyStore.setExpiry(cacheKey, SECRET_DAL_TTL); + + const unsanitizedSecrets = JSON.parse(cachedSecrets) as BufferKeysToString<(typeof data)[number]>[]; + const sanitizedSecrets = unsanitizedSecrets.map((el) => { + const encryptedValue = el.encryptedValue ? Buffer.from(el.encryptedValue, "base64") : null; + const encryptedComment = el.encryptedComment ? Buffer.from(el.encryptedComment, "base64") : null; + const createdAt = new Date(el.createdAt); + const updatedAt = new Date(el.updatedAt); + return { ...el, encryptedComment, encryptedValue, createdAt, updatedAt }; + }); + return sanitizedSecrets; + } + } + const secs = await (tx || db.replicaNode())(TableName.SecretV2) .where({ folderId }) .where((bd) => { @@ -309,6 +431,18 @@ export const secretV2BridgeDALFactory = (db: TDbClient) => { } ] }); + if (dto.useCache) { + const newCachedSecrets = data.map((el) => { + const encryptedValue = el.encryptedValue ? el.encryptedValue.toString("base64") : null; + const encryptedComment = el.encryptedComment ? el.encryptedComment.toString("base64") : null; + return { ...el, encryptedValue, encryptedComment }; + }); + await keyStore.setItemWithExpiry( + SecretDalCacheKeys.findByFolderId(projectId, secretDalVersion, dto), + SECRET_DAL_TTL, + JSON.stringify(newCachedSecrets) + ); + } return data; } catch (error) { throw new DatabaseError({ error, name: "get all secret" }); @@ -394,12 +528,16 @@ export const secretV2BridgeDALFactory = (db: TDbClient) => { } }; - const findByFolderIds = async ( - folderIds: string[], - userId?: string, - tx?: Knex, - filters?: TFindSecretsByFolderIdsFilter - ) => { + const findByFolderIds = async (dto: { + folderIds: string[]; + userId?: string; + tx?: Knex; + projectId: string; + filters?: TFindSecretsByFolderIdsFilter; + useCache?: boolean; + }) => { + const { folderIds, tx, filters, useCache, projectId } = dto; + let { userId } = dto; try { // check if not uui then userId id is null (corner case because service token's ID is not UUI in effort to keep backwards compatibility from mongo) if (userId && !uuidValidate(userId)) { @@ -407,6 +545,26 @@ export const secretV2BridgeDALFactory = (db: TDbClient) => { userId = undefined; } + const cachedSecretDalVersion = await keyStore.getItem(SecretDalCacheKeys.getSecretDalVersion(projectId)); + const secretDalVersion = Number(cachedSecretDalVersion || 0); + if (useCache) { + const cacheKey = SecretDalCacheKeys.findByFolderIds(projectId, secretDalVersion, dto); + const cachedSecrets = await keyStore.getItem(cacheKey); + if (cachedSecrets) { + await keyStore.setExpiry(cacheKey, SECRET_DAL_TTL); + + const unsanitizedSecrets = JSON.parse(cachedSecrets) as BufferKeysToString<(typeof data)[number]>[]; + const sanitizedSecrets = unsanitizedSecrets.map((el) => { + const encryptedValue = el.encryptedValue ? Buffer.from(el.encryptedValue, "base64") : null; + const encryptedComment = el.encryptedComment ? Buffer.from(el.encryptedComment, "base64") : null; + const createdAt = new Date(el.createdAt); + const updatedAt = new Date(el.updatedAt); + return { ...el, encryptedComment, encryptedValue, createdAt, updatedAt }; + }); + return sanitizedSecrets; + } + } + const query = (tx || db.replicaNode())(TableName.SecretV2) .whereIn(`${TableName.SecretV2}.folderId`, folderIds) .where((bd) => { @@ -532,6 +690,18 @@ export const secretV2BridgeDALFactory = (db: TDbClient) => { } ] }); + if (useCache) { + const cachedSecrets = data.map((el) => { + const encryptedValue = el.encryptedValue ? el.encryptedValue.toString("base64") : null; + const encryptedComment = el.encryptedComment ? el.encryptedComment.toString("base64") : null; + return { ...el, encryptedValue, encryptedComment }; + }); + await keyStore.setItemWithExpiry( + SecretDalCacheKeys.findByFolderIds(projectId, secretDalVersion, dto), + SECRET_DAL_TTL, + JSON.stringify(cachedSecrets) + ); + } return data; } catch (error) { @@ -724,6 +894,7 @@ export const secretV2BridgeDALFactory = (db: TDbClient) => { findAllProjectSecretValues, countByFolderIds, findOne, - find + find, + cacheInvalidateSecretByProjectId }; }; diff --git a/backend/src/services/secret-v2-bridge/secret-v2-bridge-fns.ts b/backend/src/services/secret-v2-bridge/secret-v2-bridge-fns.ts index 4ab021510..3b35ab41a 100644 --- a/backend/src/services/secret-v2-bridge/secret-v2-bridge-fns.ts +++ b/backend/src/services/secret-v2-bridge/secret-v2-bridge-fns.ts @@ -501,7 +501,7 @@ export const expandSecretReferencesFactory = ({ const folder = await folderDAL.findBySecretPath(projectId, environment, secretPath); if (!folder) return { value: "", tags: [] }; - const secrets = await secretDAL.findByFolderId(folder.id); + const secrets = await secretDAL.findByFolderId({ folderId: folder.id, projectId, useCache: true }); const decryptedSecret = secrets.reduce>((prev, secret) => { // eslint-disable-next-line no-param-reassign diff --git a/backend/src/services/secret-v2-bridge/secret-v2-bridge-service.ts b/backend/src/services/secret-v2-bridge/secret-v2-bridge-service.ts index 509144e21..fd9bcc4a5 100644 --- a/backend/src/services/secret-v2-bridge/secret-v2-bridge-service.ts +++ b/backend/src/services/secret-v2-bridge/secret-v2-bridge-service.ts @@ -331,6 +331,7 @@ export const secretV2BridgeServiceFactory = ({ return createdSecret; }); + await secretDAL.cacheInvalidateSecretByProjectId(projectId); if (inputSecret.type === SecretType.Shared) { await snapshotService.performSnapshot(folderId); await secretQueueService.syncSecrets({ @@ -539,6 +540,7 @@ export const secretV2BridgeServiceFactory = ({ projectId }); + await secretDAL.cacheInvalidateSecretByProjectId(projectId); if (inputSecret.type === SecretType.Shared) { await snapshotService.performSnapshot(folderId); await secretQueueService.syncSecrets({ @@ -647,6 +649,7 @@ export const secretV2BridgeServiceFactory = ({ }) ); + await secretDAL.cacheInvalidateSecretByProjectId(projectId); if (inputSecret.type === SecretType.Shared) { await snapshotService.performSnapshot(folderId); await secretQueueService.syncSecrets({ @@ -796,12 +799,14 @@ export const secretV2BridgeServiceFactory = ({ ) => { const groupedFolderMappings = groupBy(folderMappings, (folderMapping) => folderMapping.folderId); - const secrets = await secretDAL.findByFolderIds( - folderMappings.map((folderMapping) => folderMapping.folderId), + const secrets = await secretDAL.findByFolderIds({ + projectId, + folderIds: folderMappings.map((folderMapping) => folderMapping.folderId), userId, - undefined, - filters - ); + tx: undefined, + filters, + useCache: true + }); const { decryptor: secretManagerDecryptor } = await kmsService.createCipherPairWithDataKey({ type: KmsDataKey.SecretManager, @@ -952,12 +957,14 @@ export const secretV2BridgeServiceFactory = ({ const groupedPaths = groupBy(paths, (p) => p.folderId); - const secrets = await secretDAL.findByFolderIds( - paths.map((p) => p.folderId), - actorId, - undefined, - params - ); + const secrets = await secretDAL.findByFolderIds({ + projectId, + folderIds: paths.map((p) => p.folderId), + userId: actorId, + tx: undefined, + filters: params, + useCache: true + }); const { decryptor: secretManagerDecryptor } = await kmsService.createCipherPairWithDataKey({ type: KmsDataKey.SecretManager, @@ -1087,6 +1094,7 @@ export const secretV2BridgeServiceFactory = ({ const secretImports = await secretImportDAL.findByFolderIds(paths.map((p) => p.folderId)); const allowedImports = secretImports.filter(({ isReplication }) => !isReplication); const importedSecrets = await fnSecretsV2FromImports({ + projectId, viewSecretValue, secretImports: allowedImports, secretDAL, @@ -1304,6 +1312,7 @@ export const secretV2BridgeServiceFactory = ({ if (!secret && includeImports) { const secretImports = await secretImportDAL.find({ folderId, isReplication: false }); const importedSecrets = await fnSecretsV2FromImports({ + projectId, secretImports, viewSecretValue, secretDAL, @@ -1543,7 +1552,7 @@ export const secretV2BridgeServiceFactory = ({ tx }) ); - + await secretDAL.cacheInvalidateSecretByProjectId(projectId); await snapshotService.performSnapshot(folderId); await secretQueueService.syncSecrets({ actor, @@ -1883,6 +1892,7 @@ export const secretV2BridgeServiceFactory = ({ } }); + await secretDAL.cacheInvalidateSecretByProjectId(projectId); await Promise.allSettled(folders.map((el) => (el?.id ? snapshotService.performSnapshot(el.id) : undefined))); await Promise.allSettled( folders.map((el) => @@ -2014,6 +2024,7 @@ export const secretV2BridgeServiceFactory = ({ }) ); + await secretDAL.cacheInvalidateSecretByProjectId(projectId); await snapshotService.performSnapshot(folderId); await secretQueueService.syncSecrets({ actor, @@ -2537,6 +2548,9 @@ export const secretV2BridgeServiceFactory = ({ } }); + if (isDestinationUpdated || isSourceUpdated) { + await secretDAL.cacheInvalidateSecretByProjectId(projectId); + } if (isDestinationUpdated) { await snapshotService.performSnapshot(destinationFolder.id); await secretQueueService.syncSecrets({ @@ -2715,7 +2729,7 @@ export const secretV2BridgeServiceFactory = ({ generatePaths(folderMap).map(({ folderId, path }) => [folderId, path === "/" ? path : path.substring(1)]) ); - const secrets = await secretDAL.findByFolderIds(folders.map((f) => f.id)); + const secrets = await secretDAL.findByFolderIds({ folderIds: folders.map((f) => f.id), projectId }); const { decryptor: secretManagerDecryptor } = await kmsService.createCipherPairWithDataKey({ type: KmsDataKey.SecretManager, diff --git a/backend/src/services/secret-v2-bridge/secret-v2-bridge-types.ts b/backend/src/services/secret-v2-bridge/secret-v2-bridge-types.ts index 7b4ea1ee1..11149c605 100644 --- a/backend/src/services/secret-v2-bridge/secret-v2-bridge-types.ts +++ b/backend/src/services/secret-v2-bridge/secret-v2-bridge-types.ts @@ -279,6 +279,13 @@ export type TUpdateManySecretsFnFactory = { folderDAL: TSecretFolderDALFactory; }; +export type TFindByFolderIdDALDTO = { + folderId: string; + userId?: string; + tx?: Knex; + projectId: string; +}; + export type TUpdateManySecretsFn = { projectId: string; environment: string; diff --git a/backend/src/services/secret/secret-queue.ts b/backend/src/services/secret/secret-queue.ts index 60eec9cb1..0d36250fb 100644 --- a/backend/src/services/secret/secret-queue.ts +++ b/backend/src/services/secret/secret-queue.ts @@ -367,7 +367,7 @@ export const secretQueueFactory = ({ canExpandValue: () => true }); // process secrets in current folder - const secrets = await secretV2BridgeDAL.findByFolderId(dto.folderId); + const secrets = await secretV2BridgeDAL.findByFolderId({ folderId: dto.folderId, projectId: dto.projectId }); await Promise.allSettled( secrets.map(async (secret) => { @@ -397,6 +397,7 @@ export const secretQueueFactory = ({ // if no imports then return secrets in the current folder if (!secretImports.length) return content; const importedSecrets = await fnSecretsV2FromImports({ + projectId: dto.projectId, decryptor: dto.decryptor, folderDAL, secretDAL: secretV2BridgeDAL,