awatertrevi/infisical
1
0
Fork 0
mirror of https://github.com/awatertrevi/infisical.git synced 2026-09-22 13:39:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(api): implemented secret caching version 2

Browse Source
This commit is contained in:
=
2025-04-23 19:15:50 +05:30
parent d7af9e84be
commit b60172f2be
10 changed files with 96 additions and 209 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
backend/src/ee/services/secret-replication/secret-replication-service.ts
View File

@@ -267,7 +267,6 @@ export const secretReplicationServiceFactory = ({
const sourceLocalSecrets = await secretV2BridgeDAL.find({ folderId: folder.id, type: SecretType.Shared }); const sourceLocalSecrets = await secretV2BridgeDAL.find({ folderId: folder.id, type: SecretType.Shared });
const sourceSecretImports = await secretImportDAL.find({ folderId: folder.id }); const sourceSecretImports = await secretImportDAL.find({ folderId: folder.id });
const sourceImportedSecrets = await fnSecretsV2FromImports({ const sourceImportedSecrets = await fnSecretsV2FromImports({
projectId,
secretImports: sourceSecretImports, secretImports: sourceSecretImports,
secretDAL: secretV2BridgeDAL, secretDAL: secretV2BridgeDAL,
folderDAL, folderDAL,

3
backend/src/server/routes/index.ts
View File

@@ -1089,7 +1089,8 @@ export const registerRoutes = async (
secretApprovalRequestSecretDAL, secretApprovalRequestSecretDAL,
kmsService, kmsService,
snapshotService, snapshotService,
resourceMetadataDAL resourceMetadataDAL,
keyStore
}); });
const secretApprovalRequestService = secretApprovalRequestServiceFactory({ const secretApprovalRequestService = secretApprovalRequestServiceFactory({

3
backend/src/services/integration-auth/integration-delete-secret.ts
View File

@@ -50,7 +50,7 @@ const getIntegrationSecretsV2 = async (
} }
// process secrets in current folder // process secrets in current folder
const secrets = await secretV2BridgeDAL.findByFolderId({ folderId: dto.folderId, projectId: dto.projectId }); const secrets = await secretV2BridgeDAL.findByFolderId({ folderId: dto.folderId });
secrets.forEach((secret) => { secrets.forEach((secret) => {
const secretKey = secret.key; const secretKey = secret.key;
@@ -63,7 +63,6 @@ const getIntegrationSecretsV2 = async (
// if no imports then return secrets in the current folder // if no imports then return secrets in the current folder
if (!secretImports.length) return content; if (!secretImports.length) return content;
const importedSecrets = await fnSecretsV2FromImports({ const importedSecrets = await fnSecretsV2FromImports({
projectId: dto.projectId,
decryptor: dto.decryptor, decryptor: dto.decryptor,
folderDAL, folderDAL,
secretDAL: secretV2BridgeDAL, secretDAL: secretV2BridgeDAL,

7
backend/src/services/secret-import/secret-import-fns.ts
View File

@@ -159,8 +159,7 @@ export const fnSecretsV2FromImports = async ({
decryptor, decryptor,
expandSecretReferences, expandSecretReferences,
hasSecretAccess, hasSecretAccess,
viewSecretValue, viewSecretValue
projectId
}: { }: {
secretImports: (Omit<TSecretImports, "importEnv"> & { secretImports: (Omit<TSecretImports, "importEnv"> & {
importEnv: { id: string; slug: string; name: string }; importEnv: { id: string; slug: string; name: string };
@@ -177,7 +176,6 @@ export const fnSecretsV2FromImports = async ({
environment: string; environment: string;
}) => Promise<string | undefined>; }) => Promise<string | undefined>;
hasSecretAccess: (environment: string, secretPath: string, secretName: string, secretTagSlugs: string[]) => boolean; hasSecretAccess: (environment: string, secretPath: string, secretName: string, secretTagSlugs: string[]) => boolean;
projectId: string;
}) => { }) => {
const cyclicDetector = new Set(); const cyclicDetector = new Set();
const stack: { const stack: {
@@ -218,8 +216,7 @@ export const fnSecretsV2FromImports = async ({
type: SecretType.Shared type: SecretType.Shared
}, },
{ {
sort: [["id", "asc"]], sort: [["id", "asc"]]
useCache: { projectId }
} }
); );
const importedSecretsGroupByFolderId = groupBy(importedSecrets, (i) => i.folderId); const importedSecretsGroupByFolderId = groupBy(importedSecrets, (i) => i.folderId);

1
backend/src/services/secret-import/secret-import-service.ts
View File

@@ -698,7 +698,6 @@ export const secretImportServiceFactory = ({
projectId projectId
}); });
const importedSecrets = await fnSecretsV2FromImports({ const importedSecrets = await fnSecretsV2FromImports({
projectId,
secretImports, secretImports,
folderDAL, folderDAL,
viewSecretValue: true, viewSecretValue: true,

3
backend/src/services/secret-sync/secret-sync-queue.ts
View File

@@ -214,7 +214,7 @@ export const secretSyncQueueFactory = ({
canExpandValue: () => true canExpandValue: () => true
}); });
const secrets = await secretV2BridgeDAL.findByFolderId({ folderId, projectId }); const secrets = await secretV2BridgeDAL.findByFolderId({ folderId });
await Promise.allSettled( await Promise.allSettled(
secrets.map(async (secret) => { secrets.map(async (secret) => {
@@ -244,7 +244,6 @@ export const secretSyncQueueFactory = ({
if (secretImports.length) { if (secretImports.length) {
const importedSecrets = await fnSecretsV2FromImports({ const importedSecrets = await fnSecretsV2FromImports({
projectId,
decryptor: decryptSecretValue, decryptor: decryptSecretValue,
folderDAL, folderDAL,
secretDAL: secretV2BridgeDAL, secretDAL: secretV2BridgeDAL,

173
backend/src/services/secret-v2-bridge/secret-v2-bridge-dal.ts
View File

@@ -15,9 +15,13 @@ import {
TFindFilter, TFindFilter,
TFindOpt TFindOpt
} from "@app/lib/knex"; } from "@app/lib/knex";
import { BufferKeysToString, OrderByDirection } from "@app/lib/types"; import { OrderByDirection } from "@app/lib/types";
import { SecretsOrderBy } from "@app/services/secret/secret-types"; import { SecretsOrderBy } from "@app/services/secret/secret-types";
import type { TFindSecretsByFolderIdsFilter } from "@app/services/secret-v2-bridge/secret-v2-bridge-types"; import type {
TFindSecretsByFolderIdsFilter,
TGetSecretsDTO
} from "@app/services/secret-v2-bridge/secret-v2-bridge-types";
import { MongoAbility } from "@casl/ability";
export const SecretDalCacheKeys = { export const SecretDalCacheKeys = {
get productKey() { get productKey() {
@@ -27,34 +31,14 @@ export const SecretDalCacheKeys = {
getSecretDalVersion: (projectId: string) => { getSecretDalVersion: (projectId: string) => {
return `${SecretDalCacheKeys.productKey}:${projectId}:${TableName.SecretV2}-dal-version`; return `${SecretDalCacheKeys.productKey}:${projectId}:${TableName.SecretV2}-dal-version`;
}, },
findByFolderIds: ( getSecretsOfServiceLayer: (
projectId: string, projectId: string,
version: number, version: number,
{ useCache, tx, ...cacheKey }: Parameters<TSecretV2BridgeDALFactory["findByFolderIds"]>[0] dto: TGetSecretsDTO & { permissionRules: MongoAbility["rules"] }
) => { ) => {
return `${SecretDalCacheKeys.productKey}:${projectId}:${ return `${SecretDalCacheKeys.productKey}:${projectId}:${
TableName.SecretV2 TableName.SecretV2
}-dal:v${version}:find-by-folder-ids:${generateCacheKeyFromData(cacheKey)}`; }-dal:v${version}:get-secrets-service-layer:${dto.actorId}-${generateCacheKeyFromData(dto)}`;
},
findByFolderId: (
projectId: string,
version: number,
{ useCache, tx, ...cacheKey }: Parameters<TSecretV2BridgeDALFactory["findByFolderId"]>[0]
) => {
return `${SecretDalCacheKeys.productKey}:${projectId}:${
TableName.SecretV2
}-dal:v${version}:find-by-folder-id:${generateCacheKeyFromData(cacheKey)}`;
},
find: (projectId: string, version: number, ...args: Parameters<TSecretV2BridgeDALFactory["find"]>) => {
const [filter, opts] = args;
delete opts?.tx;
delete opts?.useCache;
return `${SecretDalCacheKeys.productKey}:${projectId}:${
TableName.SecretV2
}-dal:v${version}:find:${generateCacheKeyFromData({
filter,
opts
})}`;
} }
}; };
@@ -64,9 +48,9 @@ interface TSecretV2DalArg {
keyStore: TKeyStoreFactory; keyStore: TKeyStoreFactory;
} }
const SECRET_DAL_TTL = 5 * 60; export const SECRET_DAL_TTL = 5 * 60;
const SECRET_DAL_VERSION_TTL = 15 * 60; export const SECRET_DAL_VERSION_TTL = 15 * 60;
const MAX_SECRET_CACHE_BYTES = 25 * 1024 * 1024; export const MAX_SECRET_CACHE_BYTES = 25 * 1024 * 1024;
export const secretV2BridgeDALFactory = ({ db, keyStore }: TSecretV2DalArg) => { export const secretV2BridgeDALFactory = ({ db, keyStore }: TSecretV2DalArg) => {
const secretOrm = ormify(db, TableName.SecretV2); const secretOrm = ormify(db, TableName.SecretV2);
@@ -128,35 +112,9 @@ export const secretV2BridgeDALFactory = ({ db, keyStore }: TSecretV2DalArg) => {
} }
}; };
const find = async ( const find = async (filter: TFindFilter<TSecretsV2>, opts: TFindOpt<TSecretsV2> = {}) => {
filter: TFindFilter<TSecretsV2>, const { offset, limit, sort, tx } = opts;
opts: TFindOpt<TSecretsV2> & { useCache?: { projectId: string } } = {}
) => {
const { offset, limit, sort, tx, useCache } = opts;
try { try {
let secretDalVersion = 0;
if (useCache) {
const cachedSecretDalVersion = await keyStore.getItem(
SecretDalCacheKeys.getSecretDalVersion(useCache.projectId)
);
secretDalVersion = Number(cachedSecretDalVersion || 0);
const cacheKey = SecretDalCacheKeys.find(useCache.projectId, secretDalVersion, filter, opts);
const cachedSecrets = await keyStore.getItem(cacheKey);
if (cachedSecrets) {
await keyStore.setExpiry(cacheKey, SECRET_DAL_TTL);
const unsanitizedSecrets = JSON.parse(cachedSecrets) as BufferKeysToString<(typeof data)[number]>[];
const sanitizedSecrets = unsanitizedSecrets.map((el) => {
const encryptedValue = el.encryptedValue ? Buffer.from(el.encryptedValue, "base64") : null;
const encryptedComment = el.encryptedComment ? Buffer.from(el.encryptedComment, "base64") : null;
const createdAt = new Date(el.createdAt);
const updatedAt = new Date(el.updatedAt);
return { ...el, encryptedComment, encryptedValue, createdAt, updatedAt };
});
return sanitizedSecrets;
}
}
const query = (tx || db)(TableName.SecretV2) const query = (tx || db)(TableName.SecretV2)
// eslint-disable-next-line @typescript-eslint/no-misused-promises // eslint-disable-next-line @typescript-eslint/no-misused-promises
.where(buildFindFilter(filter)) .where(buildFindFilter(filter))
@@ -225,22 +183,6 @@ export const secretV2BridgeDALFactory = ({ db, keyStore }: TSecretV2DalArg) => {
] ]
}); });
if (useCache) {
const cachedSecrets = data.map((el) => {
const encryptedValue = el.encryptedValue ? el.encryptedValue.toString("base64") : null;
const encryptedComment = el.encryptedComment ? el.encryptedComment.toString("base64") : null;
return { ...el, encryptedValue, encryptedComment };
});
const cache = JSON.stringify(cachedSecrets);
if (Buffer.byteLength(cache, "utf8") < MAX_SECRET_CACHE_BYTES) {
await keyStore.setItemWithExpiry(
SecretDalCacheKeys.find(useCache.projectId, secretDalVersion, filter, opts),
SECRET_DAL_TTL,
cache
);
}
}
return data; return data;
} catch (error) { } catch (error) {
throw new DatabaseError({ error, name: `${TableName.SecretV2}: Find` }); throw new DatabaseError({ error, name: `${TableName.SecretV2}: Find` });
@@ -345,15 +287,9 @@ export const secretV2BridgeDALFactory = ({ db, keyStore }: TSecretV2DalArg) => {
} }
}; };
const findByFolderId = async (dto: { const findByFolderId = async (dto: { folderId: string; userId?: string; tx?: Knex; useCache?: boolean }) => {
folderId: string;
userId?: string;
tx?: Knex;
projectId: string;
useCache?: boolean;
}) => {
try { try {
const { folderId, tx, projectId } = dto; const { folderId, tx } = dto;
let { userId } = dto; let { userId } = dto;
// check if not uui then userId id is null (corner case because service token's ID is not UUI in effort to keep backwards compatibility from mongo // check if not uui then userId id is null (corner case because service token's ID is not UUI in effort to keep backwards compatibility from mongo
if (userId && !uuidValidate(userId)) { if (userId && !uuidValidate(userId)) {
@@ -361,27 +297,6 @@ export const secretV2BridgeDALFactory = ({ db, keyStore }: TSecretV2DalArg) => {
userId = undefined; userId = undefined;
} }
const cachedSecretDalVersion = await keyStore.getItem(SecretDalCacheKeys.getSecretDalVersion(projectId));
const secretDalVersion = Number(cachedSecretDalVersion || 0);
if (dto.useCache) {
const cacheKey = SecretDalCacheKeys.findByFolderId(projectId, secretDalVersion, dto);
const cachedSecrets = await keyStore.getItem(cacheKey);
if (cachedSecrets) {
await keyStore.setExpiry(cacheKey, SECRET_DAL_TTL);
const unsanitizedSecrets = JSON.parse(cachedSecrets) as BufferKeysToString<(typeof data)[number]>[];
const sanitizedSecrets = unsanitizedSecrets.map((el) => {
const encryptedValue = el.encryptedValue ? Buffer.from(el.encryptedValue, "base64") : null;
const encryptedComment = el.encryptedComment ? Buffer.from(el.encryptedComment, "base64") : null;
const createdAt = new Date(el.createdAt);
const updatedAt = new Date(el.updatedAt);
return { ...el, encryptedComment, encryptedValue, createdAt, updatedAt };
});
return sanitizedSecrets;
}
}
const secs = await (tx || db.replicaNode())(TableName.SecretV2) const secs = await (tx || db.replicaNode())(TableName.SecretV2)
.where({ folderId }) .where({ folderId })
.where((bd) => { .where((bd) => {
@@ -437,22 +352,6 @@ export const secretV2BridgeDALFactory = ({ db, keyStore }: TSecretV2DalArg) => {
} }
] ]
}); });
if (dto.useCache) {
const newCachedSecrets = data.map((el) => {
const encryptedValue = el.encryptedValue ? el.encryptedValue.toString("base64") : null;
const encryptedComment = el.encryptedComment ? el.encryptedComment.toString("base64") : null;
return { ...el, encryptedValue, encryptedComment };
});
const cache = JSON.stringify(newCachedSecrets);
if (Buffer.byteLength(cache, "utf8") < MAX_SECRET_CACHE_BYTES) {
await keyStore.setItemWithExpiry(
SecretDalCacheKeys.findByFolderId(projectId, secretDalVersion, dto),
SECRET_DAL_TTL,
cache
);
}
}
return data; return data;
} catch (error) { } catch (error) {
throw new DatabaseError({ error, name: "get all secret" }); throw new DatabaseError({ error, name: "get all secret" });
@@ -542,11 +441,9 @@ export const secretV2BridgeDALFactory = ({ db, keyStore }: TSecretV2DalArg) => {
folderIds: string[]; folderIds: string[];
userId?: string; userId?: string;
tx?: Knex; tx?: Knex;
projectId: string;
filters?: TFindSecretsByFolderIdsFilter; filters?: TFindSecretsByFolderIdsFilter;
useCache?: boolean;
}) => { }) => {
const { folderIds, tx, filters, useCache, projectId } = dto; const { folderIds, tx, filters } = dto;
let { userId } = dto; let { userId } = dto;
try { try {
// check if not uui then userId id is null (corner case because service token's ID is not UUI in effort to keep backwards compatibility from mongo) // check if not uui then userId id is null (corner case because service token's ID is not UUI in effort to keep backwards compatibility from mongo)
@@ -555,26 +452,6 @@ export const secretV2BridgeDALFactory = ({ db, keyStore }: TSecretV2DalArg) => {
userId = undefined; userId = undefined;
} }
const cachedSecretDalVersion = await keyStore.getItem(SecretDalCacheKeys.getSecretDalVersion(projectId));
const secretDalVersion = Number(cachedSecretDalVersion || 0);
if (useCache) {
const cacheKey = SecretDalCacheKeys.findByFolderIds(projectId, secretDalVersion, dto);
const cachedSecrets = await keyStore.getItem(cacheKey);
if (cachedSecrets) {
await keyStore.setExpiry(cacheKey, SECRET_DAL_TTL);
const unsanitizedSecrets = JSON.parse(cachedSecrets) as BufferKeysToString<(typeof data)[number]>[];
const sanitizedSecrets = unsanitizedSecrets.map((el) => {
const encryptedValue = el.encryptedValue ? Buffer.from(el.encryptedValue, "base64") : null;
const encryptedComment = el.encryptedComment ? Buffer.from(el.encryptedComment, "base64") : null;
const createdAt = new Date(el.createdAt);
const updatedAt = new Date(el.updatedAt);
return { ...el, encryptedComment, encryptedValue, createdAt, updatedAt };
});
return sanitizedSecrets;
}
}
const query = (tx || db.replicaNode())(TableName.SecretV2) const query = (tx || db.replicaNode())(TableName.SecretV2)
.whereIn(`${TableName.SecretV2}.folderId`, folderIds) .whereIn(`${TableName.SecretV2}.folderId`, folderIds)
.where((bd) => { .where((bd) => {
@@ -700,22 +577,6 @@ export const secretV2BridgeDALFactory = ({ db, keyStore }: TSecretV2DalArg) => {
} }
] ]
}); });
if (useCache) {
const cachedSecrets = data.map((el) => {
const encryptedValue = el.encryptedValue ? el.encryptedValue.toString("base64") : null;
const encryptedComment = el.encryptedComment ? el.encryptedComment.toString("base64") : null;
return { ...el, encryptedValue, encryptedComment };
});
const cache = JSON.stringify(cachedSecrets);
if (Buffer.byteLength(cache, "utf8") < MAX_SECRET_CACHE_BYTES) {
await keyStore.setItemWithExpiry(
SecretDalCacheKeys.findByFolderIds(projectId, secretDalVersion, dto),
SECRET_DAL_TTL,
cache
);
}
}
return data; return data;
} catch (error) { } catch (error) {

2
backend/src/services/secret-v2-bridge/secret-v2-bridge-fns.ts
View File

@@ -509,7 +509,7 @@ export const expandSecretReferencesFactory = ({
const folder = await folderDAL.findBySecretPath(projectId, environment, secretPath); const folder = await folderDAL.findBySecretPath(projectId, environment, secretPath);
if (!folder) return { value: "", tags: [] }; if (!folder) return { value: "", tags: [] };
const secrets = await secretDAL.findByFolderId({ folderId: folder.id, projectId, useCache: true }); const secrets = await secretDAL.findByFolderId({ folderId: folder.id });
const decryptedSecret = secrets.reduce<Record<string, { value: string; tags: string[] }>>((prev, secret) => { const decryptedSecret = secrets.reduce<Record<string, { value: string; tags: string[] }>>((prev, secret) => {
// eslint-disable-next-line no-param-reassign // eslint-disable-next-line no-param-reassign

109
backend/src/services/secret-v2-bridge/secret-v2-bridge-service.ts
View File

@@ -43,7 +43,12 @@ import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal";
import { TSecretImportDALFactory } from "../secret-import/secret-import-dal"; import { TSecretImportDALFactory } from "../secret-import/secret-import-dal";
import { fnSecretsV2FromImports } from "../secret-import/secret-import-fns"; import { fnSecretsV2FromImports } from "../secret-import/secret-import-fns";
import { TSecretTagDALFactory } from "../secret-tag/secret-tag-dal"; import { TSecretTagDALFactory } from "../secret-tag/secret-tag-dal";
import { TSecretV2BridgeDALFactory } from "./secret-v2-bridge-dal"; import {
MAX_SECRET_CACHE_BYTES,
SECRET_DAL_TTL,
SecretDalCacheKeys,
TSecretV2BridgeDALFactory
} from "./secret-v2-bridge-dal";
import { import {
buildHierarchy, buildHierarchy,
expandSecretReferencesFactory, expandSecretReferencesFactory,
@@ -76,6 +81,7 @@ import {
} from "./secret-v2-bridge-types"; } from "./secret-v2-bridge-types";
import { TSecretVersionV2DALFactory } from "./secret-version-dal"; import { TSecretVersionV2DALFactory } from "./secret-version-dal";
import { TSecretVersionV2TagDALFactory } from "./secret-version-tag-dal"; import { TSecretVersionV2TagDALFactory } from "./secret-version-tag-dal";
import { TKeyStoreFactory } from "@app/keystore/keystore";
type TSecretV2BridgeServiceFactoryDep = { type TSecretV2BridgeServiceFactoryDep = {
secretDAL: TSecretV2BridgeDALFactory; secretDAL: TSecretV2BridgeDALFactory;
@@ -105,6 +111,7 @@ type TSecretV2BridgeServiceFactoryDep = {
>; >;
snapshotService: Pick<TSecretSnapshotServiceFactory, "performSnapshot">; snapshotService: Pick<TSecretSnapshotServiceFactory, "performSnapshot">;
resourceMetadataDAL: Pick<TResourceMetadataDALFactory, "insertMany" | "delete">; resourceMetadataDAL: Pick<TResourceMetadataDALFactory, "insertMany" | "delete">;
keyStore: Pick<TKeyStoreFactory, "getItem" | "setExpiry" | "setItemWithExpiry">;
}; };
export type TSecretV2BridgeServiceFactory = ReturnType<typeof secretV2BridgeServiceFactory>; export type TSecretV2BridgeServiceFactory = ReturnType<typeof secretV2BridgeServiceFactory>;
@@ -127,7 +134,8 @@ export const secretV2BridgeServiceFactory = ({
secretApprovalRequestDAL, secretApprovalRequestDAL,
secretApprovalRequestSecretDAL, secretApprovalRequestSecretDAL,
kmsService, kmsService,
resourceMetadataDAL resourceMetadataDAL,
keyStore
}: TSecretV2BridgeServiceFactoryDep) => { }: TSecretV2BridgeServiceFactoryDep) => {
const $validateSecretReferences = async ( const $validateSecretReferences = async (
projectId: string, projectId: string,
@@ -800,12 +808,10 @@ export const secretV2BridgeServiceFactory = ({
const groupedFolderMappings = groupBy(folderMappings, (folderMapping) => folderMapping.folderId); const groupedFolderMappings = groupBy(folderMappings, (folderMapping) => folderMapping.folderId);
const secrets = await secretDAL.findByFolderIds({ const secrets = await secretDAL.findByFolderIds({
projectId,
folderIds: folderMappings.map((folderMapping) => folderMapping.folderId), folderIds: folderMappings.map((folderMapping) => folderMapping.folderId),
userId, userId,
tx: undefined, tx: undefined,
filters, filters
useCache: true
}); });
const { decryptor: secretManagerDecryptor } = await kmsService.createCipherPairWithDataKey({ const { decryptor: secretManagerDecryptor } = await kmsService.createCipherPairWithDataKey({
@@ -909,21 +915,22 @@ export const secretV2BridgeServiceFactory = ({
return decryptedSecrets; return decryptedSecrets;
}; };
const getSecrets = async ({ const getSecrets = async (dto: TGetSecretsDTO) => {
actorId, const {
path, actorId,
environment, path,
projectId, environment,
actor, projectId,
actorOrgId, actor,
viewSecretValue, actorOrgId,
actorAuthMethod, viewSecretValue,
includeImports, actorAuthMethod,
recursive, includeImports,
expandSecretReferences: shouldExpandSecretReferences, recursive,
throwOnMissingReadValuePermission = true, expandSecretReferences: shouldExpandSecretReferences,
...params throwOnMissingReadValuePermission = true,
}: TGetSecretsDTO) => { ...params
} = dto;
const { permission } = await permissionService.getProjectPermission({ const { permission } = await permissionService.getProjectPermission({
actor, actor,
actorId, actorId,
@@ -934,6 +941,32 @@ export const secretV2BridgeServiceFactory = ({
}); });
throwIfMissingSecretReadValueOrDescribePermission(permission, ProjectPermissionSecretActions.DescribeSecret); throwIfMissingSecretReadValueOrDescribePermission(permission, ProjectPermissionSecretActions.DescribeSecret);
const cachedSecretDalVersion = await keyStore.getItem(SecretDalCacheKeys.getSecretDalVersion(projectId));
const secretDalVersion = Number(cachedSecretDalVersion || 0);
const cacheKey = SecretDalCacheKeys.getSecretsOfServiceLayer(projectId, secretDalVersion, {
...dto,
permissionRules: permission.rules
});
const { decryptor: secretManagerDecryptor, encryptor: secretManagerEncryptor } =
await kmsService.createCipherPairWithDataKey({
type: KmsDataKey.SecretManager,
projectId
});
const encryptedCachedSecrets = await keyStore.getItem(cacheKey);
if (encryptedCachedSecrets) {
await keyStore.setExpiry(cacheKey, SECRET_DAL_TTL);
const cachedSecrets = secretManagerDecryptor({ cipherTextBlob: Buffer.from(encryptedCachedSecrets, "base64") });
const { secrets, imports = [] } = JSON.parse(cachedSecrets.toString("utf8")) as {
secrets: typeof decryptedSecrets;
imports: typeof importedSecrets;
};
return {
secrets: secrets.map((el) => ({ ...el, createdAt: new Date(el.createdAt), updatedAt: new Date(el.updatedAt) })),
imports
};
}
let paths: { folderId: string; path: string }[] = []; let paths: { folderId: string; path: string }[] = [];
if (recursive) { if (recursive) {
@@ -958,17 +991,10 @@ export const secretV2BridgeServiceFactory = ({
const groupedPaths = groupBy(paths, (p) => p.folderId); const groupedPaths = groupBy(paths, (p) => p.folderId);
const secrets = await secretDAL.findByFolderIds({ const secrets = await secretDAL.findByFolderIds({
projectId,
folderIds: paths.map((p) => p.folderId), folderIds: paths.map((p) => p.folderId),
userId: actorId, userId: actorId,
tx: undefined, tx: undefined,
filters: params, filters: params
useCache: true
});
const { decryptor: secretManagerDecryptor } = await kmsService.createCipherPairWithDataKey({
type: KmsDataKey.SecretManager,
projectId
}); });
// scott: if any of this changes it also needs to be mirrored in secret rotation for getting dashboard secrets // scott: if any of this changes it also needs to be mirrored in secret rotation for getting dashboard secrets
@@ -1086,15 +1112,19 @@ export const secretV2BridgeServiceFactory = ({
} }
if (!includeImports) { if (!includeImports) {
return { const payload = { secrets: decryptedSecrets, imports: [] };
secrets: decryptedSecrets const encryptedUpdatedCachedSecrets = secretManagerEncryptor({
}; plainText: Buffer.from(JSON.stringify(payload))
}).cipherTextBlob;
if (encryptedUpdatedCachedSecrets.byteLength < MAX_SECRET_CACHE_BYTES) {
await keyStore.setItemWithExpiry(cacheKey, SECRET_DAL_TTL, encryptedUpdatedCachedSecrets.toString("base64"));
}
return payload;
} }
const secretImports = await secretImportDAL.findByFolderIds(paths.map((p) => p.folderId)); const secretImports = await secretImportDAL.findByFolderIds(paths.map((p) => p.folderId));
const allowedImports = secretImports.filter(({ isReplication }) => !isReplication); const allowedImports = secretImports.filter(({ isReplication }) => !isReplication);
const importedSecrets = await fnSecretsV2FromImports({ const importedSecrets = await fnSecretsV2FromImports({
projectId,
viewSecretValue, viewSecretValue,
secretImports: allowedImports, secretImports: allowedImports,
secretDAL, secretDAL,
@@ -1129,10 +1159,14 @@ export const secretV2BridgeServiceFactory = ({
} }
}); });
return { const payload = { secrets: decryptedSecrets, imports: importedSecrets };
secrets: decryptedSecrets, const encryptedUpdatedCachedSecrets = secretManagerEncryptor({
imports: importedSecrets plainText: Buffer.from(JSON.stringify(payload))
}; }).cipherTextBlob;
if (encryptedUpdatedCachedSecrets.byteLength < MAX_SECRET_CACHE_BYTES) {
await keyStore.setItemWithExpiry(cacheKey, SECRET_DAL_TTL, encryptedUpdatedCachedSecrets.toString("base64"));
}
return payload;
}; };
const getSecretById = async ({ actorId, actor, actorOrgId, actorAuthMethod, secretId }: TGetASecretByIdDTO) => { const getSecretById = async ({ actorId, actor, actorOrgId, actorAuthMethod, secretId }: TGetASecretByIdDTO) => {
@@ -1312,7 +1346,6 @@ export const secretV2BridgeServiceFactory = ({
if (!secret && includeImports) { if (!secret && includeImports) {
const secretImports = await secretImportDAL.find({ folderId, isReplication: false }); const secretImports = await secretImportDAL.find({ folderId, isReplication: false });
const importedSecrets = await fnSecretsV2FromImports({ const importedSecrets = await fnSecretsV2FromImports({
projectId,
secretImports, secretImports,
viewSecretValue, viewSecretValue,
secretDAL, secretDAL,
@@ -2729,7 +2762,7 @@ export const secretV2BridgeServiceFactory = ({
generatePaths(folderMap).map(({ folderId, path }) => [folderId, path === "/" ? path : path.substring(1)]) generatePaths(folderMap).map(({ folderId, path }) => [folderId, path === "/" ? path : path.substring(1)])
); );
const secrets = await secretDAL.findByFolderIds({ folderIds: folders.map((f) => f.id), projectId, useCache: true }); const secrets = await secretDAL.findByFolderIds({ folderIds: folders.map((f) => f.id) });
const { decryptor: secretManagerDecryptor } = await kmsService.createCipherPairWithDataKey({ const { decryptor: secretManagerDecryptor } = await kmsService.createCipherPairWithDataKey({
type: KmsDataKey.SecretManager, type: KmsDataKey.SecretManager,

3
backend/src/services/secret/secret-queue.ts
View File

@@ -367,7 +367,7 @@ export const secretQueueFactory = ({
canExpandValue: () => true canExpandValue: () => true
}); });
// process secrets in current folder // process secrets in current folder
const secrets = await secretV2BridgeDAL.findByFolderId({ folderId: dto.folderId, projectId: dto.projectId }); const secrets = await secretV2BridgeDAL.findByFolderId({ folderId: dto.folderId });
await Promise.allSettled( await Promise.allSettled(
secrets.map(async (secret) => { secrets.map(async (secret) => {
@@ -397,7 +397,6 @@ export const secretQueueFactory = ({
// if no imports then return secrets in the current folder // if no imports then return secrets in the current folder
if (!secretImports.length) return content; if (!secretImports.length) return content;
const importedSecrets = await fnSecretsV2FromImports({ const importedSecrets = await fnSecretsV2FromImports({
projectId: dto.projectId,
decryptor: dto.decryptor, decryptor: dto.decryptor,
folderDAL, folderDAL,
secretDAL: secretV2BridgeDAL, secretDAL: secretV2BridgeDAL,