awatertrevi/infisical
1
0
Fork 0
mirror of https://github.com/awatertrevi/infisical.git synced 2026-09-22 13:39:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #3394 from akhilmhdh/feat/secret-cache

Browse Source 
Added caching for secret dal
This commit is contained in:
Maidul Islam
2025-04-11 16:11:30 -04:00
committed by GitHub
parent fe58508136 9565ef29d0
commit 32641cfc3a
19 changed files with 305 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
backend/e2e-test/mocks/keystore.ts
View File

@@ -9,6 +9,7 @@ export const mockKeyStore = (): TKeyStoreFactory => {
store[key] = value;
return "OK";
},
setExpiry: async () => 0,
setItemWithExpiry: async (key, value) => {
store[key] = value;
return "OK";

9
backend/src/ee/services/secret-approval-request/secret-approval-request-service.ts
View File

@@ -113,7 +113,13 @@ type TSecretApprovalRequestServiceFactoryDep = {
kmsService: Pick<TKmsServiceFactory, "createCipherPairWithDataKey" | "encryptWithInputKey" | "decryptWithInputKey">;
secretV2BridgeDAL: Pick<
TSecretV2BridgeDALFactory,
"insertMany" | "upsertSecretReferences" | "findBySecretKeys" | "bulkUpdate" | "deleteMany" | "find"
| "insertMany"
| "upsertSecretReferences"
| "findBySecretKeys"
| "bulkUpdate"
| "deleteMany"
| "find"
| "invalidateSecretCacheByProjectId"
>;
secretVersionV2BridgeDAL: Pick<TSecretVersionV2DALFactory, "insertMany" | "findLatestVersionMany">;
secretVersionTagV2BridgeDAL: Pick<TSecretVersionV2TagDALFactory, "insertMany">;
@@ -864,6 +870,7 @@ export const secretApprovalRequestServiceFactory = ({
});
}
await secretV2BridgeDAL.invalidateSecretCacheByProjectId(projectId);
await snapshotService.performSnapshot(folderId);
const [folder] = await folderDAL.findSecretPathByFolderIds(projectId, [folderId]);
if (!folder) {

11
backend/src/ee/services/secret-replication/secret-replication-service.ts
View File

@@ -45,7 +45,14 @@ type TSecretReplicationServiceFactoryDep = {
secretVersionDAL: Pick<TSecretVersionDALFactory, "find" | "insertMany" | "update" | "findLatestVersionMany">;
secretV2BridgeDAL: Pick<
TSecretV2BridgeDALFactory,
"find" | "findBySecretKeys" | "insertMany" | "bulkUpdate" | "delete" | "upsertSecretReferences" | "transaction"
| "find"
| "findBySecretKeys"
| "insertMany"
| "bulkUpdate"
| "delete"
| "upsertSecretReferences"
| "transaction"
| "invalidateSecretCacheByProjectId"
>;
secretVersionV2BridgeDAL: Pick<
TSecretVersionV2DALFactory,
@@ -260,6 +267,7 @@ export const secretReplicationServiceFactory = ({
const sourceLocalSecrets = await secretV2BridgeDAL.find({ folderId: folder.id, type: SecretType.Shared });
const sourceSecretImports = await secretImportDAL.find({ folderId: folder.id });
const sourceImportedSecrets = await fnSecretsV2FromImports({
projectId,
secretImports: sourceSecretImports,
secretDAL: secretV2BridgeDAL,
folderDAL,
@@ -497,6 +505,7 @@ export const secretReplicationServiceFactory = ({
}
});
await secretV2BridgeDAL.invalidateSecretCacheByProjectId(projectId);
await secretQueueService.syncSecrets({
projectId,
orgId,

6
backend/src/ee/services/secret-rotation-v2/secret-rotation-v2-service.ts
View File

@@ -88,7 +88,7 @@ export type TSecretRotationV2ServiceFactoryDep = {
folderDAL: Pick<TSecretFolderDALFactory, "findBySecretPath" | "findBySecretPathMultiEnv">;
secretV2BridgeDAL: Pick<
TSecretV2BridgeDALFactory,
"bulkUpdate" | "insertMany" | "deleteMany" | "upsertSecretReferences" | "find"
"bulkUpdate" | "insertMany" | "deleteMany" | "upsertSecretReferences" | "find" | "invalidateSecretCacheByProjectId"
>;
secretVersionV2BridgeDAL: Pick<TSecretVersionV2DALFactory, "insertMany">;
secretVersionTagV2BridgeDAL: Pick<TSecretVersionV2TagDALFactory, "insertMany">;
@@ -515,6 +515,7 @@ export const secretRotationV2ServiceFactory = ({
});
});
await secretV2BridgeDAL.invalidateSecretCacheByProjectId(projectId);
await snapshotService.performSnapshot(folder.id);
await secretQueueService.syncSecrets({
orgId: connection.orgId,
@@ -651,6 +652,7 @@ export const secretRotationV2ServiceFactory = ({
});
if (secretsMappingUpdated) {
await secretV2BridgeDAL.invalidateSecretCacheByProjectId(projectId);
await snapshotService.performSnapshot(folder.id);
await secretQueueService.syncSecrets({
orgId: connection.orgId,
@@ -777,6 +779,7 @@ export const secretRotationV2ServiceFactory = ({
}
if (deleteSecrets) {
await secretV2BridgeDAL.invalidateSecretCacheByProjectId(projectId);
await snapshotService.performSnapshot(folder.id);
await secretQueueService.syncSecrets({
orgId: connection.orgId,
@@ -935,6 +938,7 @@ export const secretRotationV2ServiceFactory = ({
}
});
await secretV2BridgeDAL.invalidateSecretCacheByProjectId(projectId);
await snapshotService.performSnapshot(folder.id);
await secretQueueService.syncSecrets({
orgId: connection.orgId,

4
backend/src/ee/services/secret-rotation/secret-rotation-queue/secret-rotation-queue.ts
View File

@@ -48,7 +48,7 @@ type TSecretRotationQueueFactoryDep = {
secretRotationDAL: TSecretRotationDALFactory;
projectBotService: Pick<TProjectBotServiceFactory, "getBotKey">;
secretDAL: Pick<TSecretDALFactory, "bulkUpdate" | "find">;
secretV2BridgeDAL: Pick<TSecretV2BridgeDALFactory, "bulkUpdate" | "find">;
secretV2BridgeDAL: Pick<TSecretV2BridgeDALFactory, "bulkUpdate" | "find" | "invalidateSecretCacheByProjectId">;
secretVersionDAL: Pick<TSecretVersionDALFactory, "insertMany" | "findLatestVersionMany">;
secretVersionV2BridgeDAL: Pick<TSecretVersionV2DALFactory, "insertMany" | "findLatestVersionMany">;
telemetryService: Pick<TTelemetryServiceFactory, "sendPostHogEvents">;
@@ -339,6 +339,8 @@ export const secretRotationQueueFactory = ({
tx
);
});
await secretV2BridgeDAL.invalidateSecretCacheByProjectId(secretRotation.projectId);
} else {
if (!botKey)
throw new NotFoundError({

3
backend/src/keystore/keystore.ts
View File

@@ -77,6 +77,8 @@ export const keyStoreFactory = (redisUrl: string) => {
const incrementBy = async (key: string, value: number) => redis.incrby(key, value);
const setExpiry = async (key: string, expiryInSeconds: number) => redis.expire(key, expiryInSeconds);
const waitTillReady = async ({
key,
waitingCb,
@@ -103,6 +105,7 @@ export const keyStoreFactory = (redisUrl: string) => {
return {
setItem,
getItem,
setExpiry,
setItemWithExpiry,
deleteItem,
incrementBy,

1
backend/src/keystore/memory.ts
View File

@@ -10,6 +10,7 @@ export const inMemoryKeyStore = (): TKeyStoreFactory => {
store[key] = value;
return "OK";
},
setExpiry: async () => 0,
setItemWithExpiry: async (key, value) => {
store[key] = value;
return "OK";

10
backend/src/lib/crypto/cache.ts Normal file
View File

@@ -0,0 +1,10 @@
import crypto from "node:crypto";
export const generateCacheKeyFromData = (data: unknown) =>
crypto
.createHash("md5")
.update(JSON.stringify(data))
.digest("base64")
.replace(/\+/g, "-")
.replace(/\//g, "_")
.replace(/=/g, "");

12
backend/src/lib/types/index.ts
View File

@@ -41,6 +41,18 @@ export type RequiredKeys<T> = {
[K in keyof T]-?: undefined extends T[K] ? never : K;
}[keyof T];
export type BufferKeysToString<T> = {
[K in keyof T]: T[K] extends Buffer
? string
: T[K] extends Buffer | null
? string | null
: T[K] extends Buffer | undefined
? string | undefined
: T[K] extends Buffer | null | undefined
? string | null | undefined
: T[K];
};
export type PickRequired<T> = Pick<T, RequiredKeys<T>>;
export type DiscriminativePick<T, K extends keyof T> = T extends unknown ? Pick<T, K> : never;

2
backend/src/server/routes/index.ts
View File

@@ -315,7 +315,7 @@ export const registerRoutes = async (
const secretVersionTagDAL = secretVersionTagDALFactory(db);
const secretBlindIndexDAL = secretBlindIndexDALFactory(db);
const secretV2BridgeDAL = secretV2BridgeDALFactory(db);
const secretV2BridgeDAL = secretV2BridgeDALFactory({ db, keyStore });
const secretVersionV2BridgeDAL = secretVersionV2BridgeDALFactory(db);
const secretVersionTagV2BridgeDAL = secretVersionV2TagBridgeDALFactory(db);

3
backend/src/services/integration-auth/integration-delete-secret.ts
View File

@@ -50,7 +50,7 @@ const getIntegrationSecretsV2 = async (
}
// process secrets in current folder
const secrets = await secretV2BridgeDAL.findByFolderId(dto.folderId);
const secrets = await secretV2BridgeDAL.findByFolderId({ folderId: dto.folderId, projectId: dto.projectId });
secrets.forEach((secret) => {
const secretKey = secret.key;
@@ -63,6 +63,7 @@ const getIntegrationSecretsV2 = async (
// if no imports then return secrets in the current folder
if (!secretImports.length) return content;
const importedSecrets = await fnSecretsV2FromImports({
projectId: dto.projectId,
decryptor: dto.decryptor,
folderDAL,
secretDAL: secretV2BridgeDAL,

7
backend/src/services/secret-import/secret-import-fns.ts
View File

@@ -159,7 +159,8 @@ export const fnSecretsV2FromImports = async ({
decryptor,
expandSecretReferences,
hasSecretAccess,
viewSecretValue
viewSecretValue,
projectId
}: {
secretImports: (Omit<TSecretImports, "importEnv"> & {
importEnv: { id: string; slug: string; name: string };
@@ -176,6 +177,7 @@ export const fnSecretsV2FromImports = async ({
environment: string;
}) => Promise<string | undefined>;
hasSecretAccess: (environment: string, secretPath: string, secretName: string, secretTagSlugs: string[]) => boolean;
projectId: string;
}) => {
const cyclicDetector = new Set();
const stack: {
@@ -216,7 +218,8 @@ export const fnSecretsV2FromImports = async ({
type: SecretType.Shared
},
{
sort: [["id", "asc"]]
sort: [["id", "asc"]],
useCache: { projectId }
}
);
const importedSecretsGroupByFolderId = groupBy(importedSecrets, (i) => i.folderId);

7
backend/src/services/secret-import/secret-import-service.ts
View File

@@ -43,7 +43,7 @@ type TSecretImportServiceFactoryDep = {
secretImportDAL: TSecretImportDALFactory;
folderDAL: TSecretFolderDALFactory;
secretDAL: Pick<TSecretDALFactory, "find">;
secretV2BridgeDAL: Pick<TSecretV2BridgeDALFactory, "find">;
secretV2BridgeDAL: Pick<TSecretV2BridgeDALFactory, "find" | "invalidateSecretCacheByProjectId">;
projectBotService: Pick<TProjectBotServiceFactory, "getBotKey">;
projectDAL: Pick<TProjectDALFactory, "checkProjectUpgradeStatus">;
projectEnvDAL: TProjectEnvDALFactory;
@@ -184,6 +184,7 @@ export const secretImportServiceFactory = ({
});
}
await secretV2BridgeDAL.invalidateSecretCacheByProjectId(projectId);
return { ...secImport, importEnv };
};
@@ -281,6 +282,8 @@ export const secretImportServiceFactory = ({
);
return doc;
});
await secretV2BridgeDAL.invalidateSecretCacheByProjectId(projectId);
return { ...updatedSecImport, importEnv: importedEnv };
};
@@ -355,6 +358,7 @@ export const secretImportServiceFactory = ({
actorId
});
await secretV2BridgeDAL.invalidateSecretCacheByProjectId(projectId);
return secImport;
};
@@ -693,6 +697,7 @@ export const secretImportServiceFactory = ({
projectId
});
const importedSecrets = await fnSecretsV2FromImports({
projectId,
secretImports,
folderDAL,
viewSecretValue: true,

3
backend/src/services/secret-sync/secret-sync-queue.ts
View File

@@ -213,7 +213,7 @@ export const secretSyncQueueFactory = ({
canExpandValue: () => true
});
const secrets = await secretV2BridgeDAL.findByFolderId(folderId);
const secrets = await secretV2BridgeDAL.findByFolderId({ folderId, projectId });
await Promise.allSettled(
secrets.map(async (secret) => {
@@ -243,6 +243,7 @@ export const secretSyncQueueFactory = ({
if (secretImports.length) {
const importedSecrets = await fnSecretsV2FromImports({
projectId,
decryptor: decryptSecretValue,
folderDAL,
secretDAL: secretV2BridgeDAL,

213
backend/src/services/secret-v2-bridge/secret-v2-bridge-dal.ts
View File

@@ -2,7 +2,10 @@ import { Knex } from "knex";
import { validate as uuidValidate } from "uuid";
import { TDbClient } from "@app/db";
import { SecretsV2Schema, SecretType, TableName, TSecretsV2, TSecretsV2Update } from "@app/db/schemas";
import { ProjectType, SecretsV2Schema, SecretType, TableName, TSecretsV2, TSecretsV2Update } from "@app/db/schemas";
import { TKeyStoreFactory } from "@app/keystore/keystore";
import { getConfig } from "@app/lib/config/env";
import { generateCacheKeyFromData } from "@app/lib/crypto/cache";
import { BadRequestError, DatabaseError, NotFoundError } from "@app/lib/errors";
import {
buildFindFilter,
@@ -12,15 +15,67 @@ import {
TFindFilter,
TFindOpt
} from "@app/lib/knex";
import { OrderByDirection } from "@app/lib/types";
import { BufferKeysToString, OrderByDirection } from "@app/lib/types";
import { SecretsOrderBy } from "@app/services/secret/secret-types";
import { TFindSecretsByFolderIdsFilter } from "@app/services/secret-v2-bridge/secret-v2-bridge-types";
import type { TFindSecretsByFolderIdsFilter } from "@app/services/secret-v2-bridge/secret-v2-bridge-types";
export const SecretDalCacheKeys = {
get productKey() {
const { INFISICAL_PLATFORM_VERSION } = getConfig();
return `${ProjectType.SecretManager}:${INFISICAL_PLATFORM_VERSION || 0}`;
},
getSecretDalVersion: (projectId: string) => {
return `${SecretDalCacheKeys.productKey}:${projectId}:${TableName.SecretV2}-dal-version`;
},
findByFolderIds: (
projectId: string,
version: number,
{ useCache, tx, ...cacheKey }: Parameters<TSecretV2BridgeDALFactory["findByFolderIds"]>[0]
) => {
return `${SecretDalCacheKeys.productKey}:${projectId}:${
TableName.SecretV2
}-dal:v${version}:find-by-folder-ids:${generateCacheKeyFromData(cacheKey)}`;
},
findByFolderId: (
projectId: string,
version: number,
{ useCache, tx, ...cacheKey }: Parameters<TSecretV2BridgeDALFactory["findByFolderId"]>[0]
) => {
return `${SecretDalCacheKeys.productKey}:${projectId}:${
TableName.SecretV2
}-dal:v${version}:find-by-folder-id:${generateCacheKeyFromData(cacheKey)}`;
},
find: (projectId: string, version: number, ...args: Parameters<TSecretV2BridgeDALFactory["find"]>) => {
const [filter, opts] = args;
delete opts?.tx;
delete opts?.useCache;
return `${SecretDalCacheKeys.productKey}:${projectId}:${
TableName.SecretV2
}-dal:v${version}:find:${generateCacheKeyFromData({
filter,
opts
})}`;
}
};
export type TSecretV2BridgeDALFactory = ReturnType<typeof secretV2BridgeDALFactory>;
interface TSecretV2DalArg {
db: TDbClient;
keyStore: TKeyStoreFactory;
}
export const secretV2BridgeDALFactory = (db: TDbClient) => {
const SECRET_DAL_TTL = 5 * 60;
const SECRET_DAL_VERSION_TTL = 15 * 60;
const MAX_SECRET_CACHE_BYTES = 25 * 1024 * 1024;
export const secretV2BridgeDALFactory = ({ db, keyStore }: TSecretV2DalArg) => {
const secretOrm = ormify(db, TableName.SecretV2);
const invalidateSecretCacheByProjectId = async (projectId: string) => {
const secretDalVersionKey = SecretDalCacheKeys.getSecretDalVersion(projectId);
await keyStore.incrementBy(secretDalVersionKey, 1);
await keyStore.setExpiry(secretDalVersionKey, SECRET_DAL_VERSION_TTL);
};
const findOne = async (filter: Partial<TSecretsV2>, tx?: Knex) => {
try {
const docs = await (tx || db)(TableName.SecretV2)
@@ -73,8 +128,35 @@ export const secretV2BridgeDALFactory = (db: TDbClient) => {
}
};
const find = async (filter: TFindFilter<TSecretsV2>, { offset, limit, sort, tx }: TFindOpt<TSecretsV2> = {}) => {
const find = async (
filter: TFindFilter<TSecretsV2>,
opts: TFindOpt<TSecretsV2> & { useCache?: { projectId: string } } = {}
) => {
const { offset, limit, sort, tx, useCache } = opts;
try {
let secretDalVersion = 0;
if (useCache) {
const cachedSecretDalVersion = await keyStore.getItem(
SecretDalCacheKeys.getSecretDalVersion(useCache.projectId)
);
secretDalVersion = Number(cachedSecretDalVersion || 0);
const cacheKey = SecretDalCacheKeys.find(useCache.projectId, secretDalVersion, filter, opts);
const cachedSecrets = await keyStore.getItem(cacheKey);
if (cachedSecrets) {
await keyStore.setExpiry(cacheKey, SECRET_DAL_TTL);
const unsanitizedSecrets = JSON.parse(cachedSecrets) as BufferKeysToString<(typeof data)[number]>[];
const sanitizedSecrets = unsanitizedSecrets.map((el) => {
const encryptedValue = el.encryptedValue ? Buffer.from(el.encryptedValue, "base64") : null;
const encryptedComment = el.encryptedComment ? Buffer.from(el.encryptedComment, "base64") : null;
const createdAt = new Date(el.createdAt);
const updatedAt = new Date(el.updatedAt);
return { ...el, encryptedComment, encryptedValue, createdAt, updatedAt };
});
return sanitizedSecrets;
}
}
const query = (tx || db)(TableName.SecretV2)
// eslint-disable-next-line @typescript-eslint/no-misused-promises
.where(buildFindFilter(filter))
@@ -142,6 +224,23 @@ export const secretV2BridgeDALFactory = (db: TDbClient) => {
}
]
});
if (useCache) {
const cachedSecrets = data.map((el) => {
const encryptedValue = el.encryptedValue ? el.encryptedValue.toString("base64") : null;
const encryptedComment = el.encryptedComment ? el.encryptedComment.toString("base64") : null;
return { ...el, encryptedValue, encryptedComment };
});
const cache = JSON.stringify(cachedSecrets);
if (Buffer.byteLength(cache, "utf8") < MAX_SECRET_CACHE_BYTES) {
await keyStore.setItemWithExpiry(
SecretDalCacheKeys.find(useCache.projectId, secretDalVersion, filter, opts),
SECRET_DAL_TTL,
cache
);
}
}
return data;
} catch (error) {
throw new DatabaseError({ error, name: `${TableName.SecretV2}: Find` });
@@ -246,14 +345,43 @@ export const secretV2BridgeDALFactory = (db: TDbClient) => {
}
};
const findByFolderId = async (folderId: string, userId?: string, tx?: Knex) => {
const findByFolderId = async (dto: {
folderId: string;
userId?: string;
tx?: Knex;
projectId: string;
useCache?: boolean;
}) => {
try {
// check if not uui then userId id is null (corner case because service token's ID is not UUI in effort to keep backwards compatibility from mongo)
const { folderId, tx, projectId } = dto;
let { userId } = dto;
// check if not uui then userId id is null (corner case because service token's ID is not UUI in effort to keep backwards compatibility from mongo
if (userId && !uuidValidate(userId)) {
// eslint-disable-next-line
userId = undefined;
}
const cachedSecretDalVersion = await keyStore.getItem(SecretDalCacheKeys.getSecretDalVersion(projectId));
const secretDalVersion = Number(cachedSecretDalVersion || 0);
if (dto.useCache) {
const cacheKey = SecretDalCacheKeys.findByFolderId(projectId, secretDalVersion, dto);
const cachedSecrets = await keyStore.getItem(cacheKey);
if (cachedSecrets) {
await keyStore.setExpiry(cacheKey, SECRET_DAL_TTL);
const unsanitizedSecrets = JSON.parse(cachedSecrets) as BufferKeysToString<(typeof data)[number]>[];
const sanitizedSecrets = unsanitizedSecrets.map((el) => {
const encryptedValue = el.encryptedValue ? Buffer.from(el.encryptedValue, "base64") : null;
const encryptedComment = el.encryptedComment ? Buffer.from(el.encryptedComment, "base64") : null;
const createdAt = new Date(el.createdAt);
const updatedAt = new Date(el.updatedAt);
return { ...el, encryptedComment, encryptedValue, createdAt, updatedAt };
});
return sanitizedSecrets;
}
}
const secs = await (tx || db.replicaNode())(TableName.SecretV2)
.where({ folderId })
.where((bd) => {
@@ -309,6 +437,22 @@ export const secretV2BridgeDALFactory = (db: TDbClient) => {
}
]
});
if (dto.useCache) {
const newCachedSecrets = data.map((el) => {
const encryptedValue = el.encryptedValue ? el.encryptedValue.toString("base64") : null;
const encryptedComment = el.encryptedComment ? el.encryptedComment.toString("base64") : null;
return { ...el, encryptedValue, encryptedComment };
});
const cache = JSON.stringify(newCachedSecrets);
if (Buffer.byteLength(cache, "utf8") < MAX_SECRET_CACHE_BYTES) {
await keyStore.setItemWithExpiry(
SecretDalCacheKeys.findByFolderId(projectId, secretDalVersion, dto),
SECRET_DAL_TTL,
cache
);
}
}
return data;
} catch (error) {
throw new DatabaseError({ error, name: "get all secret" });
@@ -394,12 +538,16 @@ export const secretV2BridgeDALFactory = (db: TDbClient) => {
}
};
const findByFolderIds = async (
folderIds: string[],
userId?: string,
tx?: Knex,
filters?: TFindSecretsByFolderIdsFilter
) => {
const findByFolderIds = async (dto: {
folderIds: string[];
userId?: string;
tx?: Knex;
projectId: string;
filters?: TFindSecretsByFolderIdsFilter;
useCache?: boolean;
}) => {
const { folderIds, tx, filters, useCache, projectId } = dto;
let { userId } = dto;
try {
// check if not uui then userId id is null (corner case because service token's ID is not UUI in effort to keep backwards compatibility from mongo)
if (userId && !uuidValidate(userId)) {
@@ -407,6 +555,26 @@ export const secretV2BridgeDALFactory = (db: TDbClient) => {
userId = undefined;
}
const cachedSecretDalVersion = await keyStore.getItem(SecretDalCacheKeys.getSecretDalVersion(projectId));
const secretDalVersion = Number(cachedSecretDalVersion || 0);
if (useCache) {
const cacheKey = SecretDalCacheKeys.findByFolderIds(projectId, secretDalVersion, dto);
const cachedSecrets = await keyStore.getItem(cacheKey);
if (cachedSecrets) {
await keyStore.setExpiry(cacheKey, SECRET_DAL_TTL);
const unsanitizedSecrets = JSON.parse(cachedSecrets) as BufferKeysToString<(typeof data)[number]>[];
const sanitizedSecrets = unsanitizedSecrets.map((el) => {
const encryptedValue = el.encryptedValue ? Buffer.from(el.encryptedValue, "base64") : null;
const encryptedComment = el.encryptedComment ? Buffer.from(el.encryptedComment, "base64") : null;
const createdAt = new Date(el.createdAt);
const updatedAt = new Date(el.updatedAt);
return { ...el, encryptedComment, encryptedValue, createdAt, updatedAt };
});
return sanitizedSecrets;
}
}
const query = (tx || db.replicaNode())(TableName.SecretV2)
.whereIn(`${TableName.SecretV2}.folderId`, folderIds)
.where((bd) => {
@@ -532,6 +700,22 @@ export const secretV2BridgeDALFactory = (db: TDbClient) => {
}
]
});
if (useCache) {
const cachedSecrets = data.map((el) => {
const encryptedValue = el.encryptedValue ? el.encryptedValue.toString("base64") : null;
const encryptedComment = el.encryptedComment ? el.encryptedComment.toString("base64") : null;
return { ...el, encryptedValue, encryptedComment };
});
const cache = JSON.stringify(cachedSecrets);
if (Buffer.byteLength(cache, "utf8") < MAX_SECRET_CACHE_BYTES) {
await keyStore.setItemWithExpiry(
SecretDalCacheKeys.findByFolderIds(projectId, secretDalVersion, dto),
SECRET_DAL_TTL,
cache
);
}
}
return data;
} catch (error) {
@@ -724,6 +908,7 @@ export const secretV2BridgeDALFactory = (db: TDbClient) => {
findAllProjectSecretValues,
countByFolderIds,
findOne,
find
find,
invalidateSecretCacheByProjectId
};
};

2
backend/src/services/secret-v2-bridge/secret-v2-bridge-fns.ts
View File

@@ -501,7 +501,7 @@ export const expandSecretReferencesFactory = ({
const folder = await folderDAL.findBySecretPath(projectId, environment, secretPath);
if (!folder) return { value: "", tags: [] };
const secrets = await secretDAL.findByFolderId(folder.id);
const secrets = await secretDAL.findByFolderId({ folderId: folder.id, projectId, useCache: true });
const decryptedSecret = secrets.reduce<Record<string, { value: string; tags: string[] }>>((prev, secret) => {
// eslint-disable-next-line no-param-reassign

40
backend/src/services/secret-v2-bridge/secret-v2-bridge-service.ts
View File

@@ -331,6 +331,7 @@ export const secretV2BridgeServiceFactory = ({
return createdSecret;
});
await secretDAL.invalidateSecretCacheByProjectId(projectId);
if (inputSecret.type === SecretType.Shared) {
await snapshotService.performSnapshot(folderId);
await secretQueueService.syncSecrets({
@@ -539,6 +540,7 @@ export const secretV2BridgeServiceFactory = ({
projectId
});
await secretDAL.invalidateSecretCacheByProjectId(projectId);
if (inputSecret.type === SecretType.Shared) {
await snapshotService.performSnapshot(folderId);
await secretQueueService.syncSecrets({
@@ -647,6 +649,7 @@ export const secretV2BridgeServiceFactory = ({
})
);
await secretDAL.invalidateSecretCacheByProjectId(projectId);
if (inputSecret.type === SecretType.Shared) {
await snapshotService.performSnapshot(folderId);
await secretQueueService.syncSecrets({
@@ -796,12 +799,14 @@ export const secretV2BridgeServiceFactory = ({
) => {
const groupedFolderMappings = groupBy(folderMappings, (folderMapping) => folderMapping.folderId);
const secrets = await secretDAL.findByFolderIds(
folderMappings.map((folderMapping) => folderMapping.folderId),
const secrets = await secretDAL.findByFolderIds({
projectId,
folderIds: folderMappings.map((folderMapping) => folderMapping.folderId),
userId,
undefined,
filters
);
tx: undefined,
filters,
useCache: true
});
const { decryptor: secretManagerDecryptor } = await kmsService.createCipherPairWithDataKey({
type: KmsDataKey.SecretManager,
@@ -952,12 +957,14 @@ export const secretV2BridgeServiceFactory = ({
const groupedPaths = groupBy(paths, (p) => p.folderId);
const secrets = await secretDAL.findByFolderIds(
paths.map((p) => p.folderId),
actorId,
undefined,
params
);
const secrets = await secretDAL.findByFolderIds({
projectId,
folderIds: paths.map((p) => p.folderId),
userId: actorId,
tx: undefined,
filters: params,
useCache: true
});
const { decryptor: secretManagerDecryptor } = await kmsService.createCipherPairWithDataKey({
type: KmsDataKey.SecretManager,
@@ -1087,6 +1094,7 @@ export const secretV2BridgeServiceFactory = ({
const secretImports = await secretImportDAL.findByFolderIds(paths.map((p) => p.folderId));
const allowedImports = secretImports.filter(({ isReplication }) => !isReplication);
const importedSecrets = await fnSecretsV2FromImports({
projectId,
viewSecretValue,
secretImports: allowedImports,
secretDAL,
@@ -1304,6 +1312,7 @@ export const secretV2BridgeServiceFactory = ({
if (!secret && includeImports) {
const secretImports = await secretImportDAL.find({ folderId, isReplication: false });
const importedSecrets = await fnSecretsV2FromImports({
projectId,
secretImports,
viewSecretValue,
secretDAL,
@@ -1543,7 +1552,7 @@ export const secretV2BridgeServiceFactory = ({
tx
})
);
await secretDAL.invalidateSecretCacheByProjectId(projectId);
await snapshotService.performSnapshot(folderId);
await secretQueueService.syncSecrets({
actor,
@@ -1883,6 +1892,7 @@ export const secretV2BridgeServiceFactory = ({
}
});
await secretDAL.invalidateSecretCacheByProjectId(projectId);
await Promise.allSettled(folders.map((el) => (el?.id ? snapshotService.performSnapshot(el.id) : undefined)));
await Promise.allSettled(
folders.map((el) =>
@@ -2014,6 +2024,7 @@ export const secretV2BridgeServiceFactory = ({
})
);
await secretDAL.invalidateSecretCacheByProjectId(projectId);
await snapshotService.performSnapshot(folderId);
await secretQueueService.syncSecrets({
actor,
@@ -2537,6 +2548,9 @@ export const secretV2BridgeServiceFactory = ({
}
});
if (isDestinationUpdated || isSourceUpdated) {
await secretDAL.invalidateSecretCacheByProjectId(projectId);
}
if (isDestinationUpdated) {
await snapshotService.performSnapshot(destinationFolder.id);
await secretQueueService.syncSecrets({
@@ -2715,7 +2729,7 @@ export const secretV2BridgeServiceFactory = ({
generatePaths(folderMap).map(({ folderId, path }) => [folderId, path === "/" ? path : path.substring(1)])
);
const secrets = await secretDAL.findByFolderIds(folders.map((f) => f.id));
const secrets = await secretDAL.findByFolderIds({ folderIds: folders.map((f) => f.id), projectId, useCache: true });
const { decryptor: secretManagerDecryptor } = await kmsService.createCipherPairWithDataKey({
type: KmsDataKey.SecretManager,

7
backend/src/services/secret-v2-bridge/secret-v2-bridge-types.ts
View File

@@ -279,6 +279,13 @@ export type TUpdateManySecretsFnFactory = {
folderDAL: TSecretFolderDALFactory;
};
export type TFindByFolderIdDALDTO = {
folderId: string;
userId?: string;
tx?: Knex;
projectId: string;
};
export type TUpdateManySecretsFn = {
projectId: string;
environment: string;

3
backend/src/services/secret/secret-queue.ts
View File

@@ -367,7 +367,7 @@ export const secretQueueFactory = ({
canExpandValue: () => true
});
// process secrets in current folder
const secrets = await secretV2BridgeDAL.findByFolderId(dto.folderId);
const secrets = await secretV2BridgeDAL.findByFolderId({ folderId: dto.folderId, projectId: dto.projectId });
await Promise.allSettled(
secrets.map(async (secret) => {
@@ -397,6 +397,7 @@ export const secretQueueFactory = ({
// if no imports then return secrets in the current folder
if (!secretImports.length) return content;
const importedSecrets = await fnSecretsV2FromImports({
projectId: dto.projectId,
decryptor: dto.decryptor,
folderDAL,
secretDAL: secretV2BridgeDAL,