From c29cb667d76365032e207a839bdb27937d3949b8 Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Wed, 20 Mar 2024 12:49:47 +0100 Subject: [PATCH] Feat: Recursively get secrets from all nested secret paths --- backend/src/lib/api-docs/constants.ts | 1 + backend/src/server/routes/v3/secret-router.ts | 16 +++- backend/src/services/secret/secret-fns.ts | 69 +++++++++++++++- backend/src/services/secret/secret-service.ts | 81 ++++++++++++++++--- backend/src/services/secret/secret-types.ts | 1 + 5 files changed, 150 insertions(+), 18 deletions(-) diff --git a/backend/src/lib/api-docs/constants.ts b/backend/src/lib/api-docs/constants.ts index 8736e80c6..4a45dfb41 100644 --- a/backend/src/lib/api-docs/constants.ts +++ b/backend/src/lib/api-docs/constants.ts @@ -215,6 +215,7 @@ export const SECRETS = { export const RAW_SECRETS = { LIST: { + recursive: "Whether or not to fetch all secrets from the specified base path, and all of its subdirectories.", workspaceId: "The ID of the project to list secrets from.", workspaceSlug: "The slug of the project to list secrets from. This parameter is only usable by machine identities.", environment: "The slug of the environment to list secrets from.", diff --git a/backend/src/server/routes/v3/secret-router.ts b/backend/src/server/routes/v3/secret-router.ts index f69466328..8d3f08b61 100644 --- a/backend/src/server/routes/v3/secret-router.ts +++ b/backend/src/server/routes/v3/secret-router.ts @@ -157,6 +157,11 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => { workspaceSlug: z.string().trim().optional().describe(RAW_SECRETS.LIST.workspaceSlug), environment: z.string().trim().optional().describe(RAW_SECRETS.LIST.environment), secretPath: z.string().trim().default("/").transform(removeTrailingSlash).describe(RAW_SECRETS.LIST.secretPath), + recursive: z + .enum(["true", "false"]) + .default("false") + .transform((value) => value === "true") + .describe(RAW_SECRETS.LIST.recursive), include_imports: z .enum(["true", "false"]) .default("false") @@ -165,7 +170,13 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => { }), response: { 200: z.object({ - secrets: secretRawSchema.array(), + secrets: secretRawSchema + .merge( + z.object({ + secretPath: z.string().optional() + }) + ) + .array(), imports: z .object({ secretPath: z.string(), @@ -218,7 +229,8 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => { actorAuthMethod: req.permission.authMethod, projectId: workspaceId, path: secretPath, - includeImports: req.query.include_imports + includeImports: req.query.include_imports, + recursive: req.query.recursive }); await server.services.auditLog.createAuditLog({ diff --git a/backend/src/services/secret/secret-fns.ts b/backend/src/services/secret/secret-fns.ts index 212bb01f0..b499f9365 100644 --- a/backend/src/services/secret/secret-fns.ts +++ b/backend/src/services/secret/secret-fns.ts @@ -9,6 +9,7 @@ import { TSecretBlindIndexes, TSecrets } from "@app/db/schemas"; +import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service"; import { getConfig } from "@app/lib/config/env"; import { buildSecretBlindIndexFromName, @@ -17,8 +18,10 @@ import { } from "@app/lib/crypto"; import { BadRequestError } from "@app/lib/errors"; import { groupBy, unique } from "@app/lib/fn"; +import { logger } from "@app/lib/logger"; import { getBotKeyFnFactory } from "../project-bot/project-bot-fns"; +import { TProjectEnvDALFactory } from "../project-env/project-env-dal"; import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal"; import { TSecretDALFactory } from "./secret-dal"; import { @@ -45,6 +48,68 @@ export const generateSecretBlindIndexBySalt = async (secretName: string, secretB return secretBlindIndex; }; +type TRecursivelyFetchSecretsFromFoldersArg = { + permissionService: Pick; + projectEnvDAL: Pick; + folderDAL: Pick; +}; + +export const recursivelyGetSecretPaths = ({ folderDAL, projectEnvDAL }: TRecursivelyFetchSecretsFromFoldersArg) => { + const getPaths = async (projectId: string, environment: string, currentPath: string) => { + let secretPaths: string[] = []; + + // Get secrets in the current folder. + try { + const folder = await folderDAL.findBySecretPath(projectId, environment, currentPath); + + if (!folder) { + throw new Error(`Base directory '${currentPath}' not found.`); + } + + secretPaths.push(currentPath); + } catch (error) { + logger.error(error, "Error fetching secrets from base directory"); + throw error; + } + + // List all subfolders in the current folder. + try { + const env = await projectEnvDAL.findOne({ projectId, slug: environment }); + const parentFolder = await folderDAL.findBySecretPath(projectId, environment, currentPath); + + if (!env) { + throw new Error(`Environment with not found`); + } + + if (!parentFolder) { + throw new Error(`Parent folder not found`); + } + + const folders = await folderDAL.find({ envId: env.id, parentId: parentFolder.id }); + + // Use Promise.all to handle recursive calls concurrently for efficiency. + const secretsFromSubFolders = await Promise.all( + folders.map(async (folder) => { + // Ensure the path is correctly formatted for the next level. + const subFolderPath = `${currentPath}${currentPath !== "/" ? "/" : ""}${folder.name}`; + + return getPaths(projectId, environment, subFolderPath); + }) + ); + + // Flatten the array of arrays and concatenate with the current secrets array. + secretPaths = secretPaths.concat(...secretsFromSubFolders); + } catch (error) { + logger.error(error, "Error fetching secrets from subdirectories"); + throw error; + } + + return secretPaths; + }; + + return getPaths; +}; + type TInterpolateSecretArg = { projectId: string; secretEncKey: string; @@ -202,9 +267,7 @@ export const interpolateSecrets = ({ projectId, secretEncKey, secretDAL, folderD ); // eslint-disable-next-line - secrets[key].value = secrets[key].skipMultilineEncoding - ? expandedVal - : formatMultiValueEnv(expandedVal); + secrets[key].value = secrets[key].skipMultilineEncoding ? expandedVal : formatMultiValueEnv(expandedVal); } return secrets; diff --git a/backend/src/services/secret/secret-service.ts b/backend/src/services/secret/secret-service.ts index f8fed95bb..6b71f7988 100644 --- a/backend/src/services/secret/secret-service.ts +++ b/backend/src/services/secret/secret-service.ts @@ -13,13 +13,20 @@ import { logger } from "@app/lib/logger"; import { ActorType } from "../auth/auth-type"; import { TProjectDALFactory } from "../project/project-dal"; import { TProjectBotServiceFactory } from "../project-bot/project-bot-service"; +import { TProjectEnvDALFactory } from "../project-env/project-env-dal"; import { TSecretBlindIndexDALFactory } from "../secret-blind-index/secret-blind-index-dal"; import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal"; import { TSecretImportDALFactory } from "../secret-import/secret-import-dal"; import { fnSecretsFromImports } from "../secret-import/secret-import-fns"; import { TSecretTagDALFactory } from "../secret-tag/secret-tag-dal"; import { TSecretDALFactory } from "./secret-dal"; -import { decryptSecretRaw, fnSecretBlindIndexCheck, fnSecretBulkInsert, fnSecretBulkUpdate } from "./secret-fns"; +import { + decryptSecretRaw, + fnSecretBlindIndexCheck, + fnSecretBulkInsert, + fnSecretBulkUpdate, + recursivelyGetSecretPaths +} from "./secret-fns"; import { TSecretQueueFactory } from "./secret-queue"; import { TAttachSecretTagsDTO, @@ -47,8 +54,12 @@ type TSecretServiceFactoryDep = { secretDAL: TSecretDALFactory; secretTagDAL: TSecretTagDALFactory; secretVersionDAL: TSecretVersionDALFactory; - folderDAL: Pick; projectDAL: Pick; + projectEnvDAL: Pick; + folderDAL: Pick< + TSecretFolderDALFactory, + "findBySecretPath" | "updateById" | "findById" | "findByManySecretPath" | "find" + >; secretBlindIndexDAL: TSecretBlindIndexDALFactory; permissionService: Pick; snapshotService: Pick; @@ -61,6 +72,7 @@ type TSecretServiceFactoryDep = { export type TSecretServiceFactory = ReturnType; export const secretServiceFactory = ({ secretDAL, + projectEnvDAL, secretTagDAL, secretVersionDAL, folderDAL, @@ -789,21 +801,64 @@ export const secretServiceFactory = ({ actorOrgId, actorAuthMethod, environment, - includeImports + includeImports, + recursive }: TGetSecretsRawDTO) => { const botKey = await projectBotService.getBotKey(projectId); if (!botKey) throw new BadRequestError({ message: "Project bot not found", name: "bot_not_found_error" }); - const { secrets, imports } = await getSecrets({ - actorId, - projectId, - environment, - actor, - actorOrgId, - actorAuthMethod, - path, - includeImports - }); + let secrets: Awaited>["secrets"]; + let imports: Awaited>["imports"]; + + if (recursive) { + const getPaths = recursivelyGetSecretPaths({ + permissionService, + folderDAL, + projectEnvDAL + }); + + const paths = await getPaths(projectId, environment, path); + + const result = await Promise.all( + paths.map(async (currentPath) => { + const secs = await getSecrets({ + actorId, + projectId, + environment, + actor, + actorOrgId, + actorAuthMethod, + path: currentPath, + includeImports + }); + + return { + secrets: { + ...secs.secrets, + secretPath: currentPath + }, + imports: secs.imports + }; + }) + ); + + secrets = result.flatMap((el) => el.secrets); + imports = result.flatMap((el) => el.imports || []); + } else { + const result = await getSecrets({ + actorId, + projectId, + environment, + actor, + actorOrgId, + actorAuthMethod, + path, + includeImports + }); + + secrets = result.secrets; + imports = result.imports; + } return { secrets: secrets.map((el) => decryptSecretRaw(el, botKey)), diff --git a/backend/src/services/secret/secret-types.ts b/backend/src/services/secret/secret-types.ts index efd4f0f8b..c6cacb255 100644 --- a/backend/src/services/secret/secret-types.ts +++ b/backend/src/services/secret/secret-types.ts @@ -140,6 +140,7 @@ export type TGetSecretsRawDTO = { path: string; environment: string; includeImports?: boolean; + recursive?: boolean; } & TProjectPermission; export type TGetASecretRawDTO = {