From c29cb667d76365032e207a839bdb27937d3949b8 Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Wed, 20 Mar 2024 12:49:47 +0100 Subject: [PATCH 01/18] Feat: Recursively get secrets from all nested secret paths --- backend/src/lib/api-docs/constants.ts | 1 + backend/src/server/routes/v3/secret-router.ts | 16 +++- backend/src/services/secret/secret-fns.ts | 69 +++++++++++++++- backend/src/services/secret/secret-service.ts | 81 ++++++++++++++++--- backend/src/services/secret/secret-types.ts | 1 + 5 files changed, 150 insertions(+), 18 deletions(-) diff --git a/backend/src/lib/api-docs/constants.ts b/backend/src/lib/api-docs/constants.ts index 8736e80c6..4a45dfb41 100644 --- a/backend/src/lib/api-docs/constants.ts +++ b/backend/src/lib/api-docs/constants.ts @@ -215,6 +215,7 @@ export const SECRETS = { export const RAW_SECRETS = { LIST: { + recursive: "Whether or not to fetch all secrets from the specified base path, and all of its subdirectories.", workspaceId: "The ID of the project to list secrets from.", workspaceSlug: "The slug of the project to list secrets from. This parameter is only usable by machine identities.", environment: "The slug of the environment to list secrets from.", diff --git a/backend/src/server/routes/v3/secret-router.ts b/backend/src/server/routes/v3/secret-router.ts index f69466328..8d3f08b61 100644 --- a/backend/src/server/routes/v3/secret-router.ts +++ b/backend/src/server/routes/v3/secret-router.ts @@ -157,6 +157,11 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => { workspaceSlug: z.string().trim().optional().describe(RAW_SECRETS.LIST.workspaceSlug), environment: z.string().trim().optional().describe(RAW_SECRETS.LIST.environment), secretPath: z.string().trim().default("/").transform(removeTrailingSlash).describe(RAW_SECRETS.LIST.secretPath), + recursive: z + .enum(["true", "false"]) + .default("false") + .transform((value) => value === "true") + .describe(RAW_SECRETS.LIST.recursive), include_imports: z .enum(["true", "false"]) .default("false") @@ -165,7 +170,13 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => { }), response: { 200: z.object({ - secrets: secretRawSchema.array(), + secrets: secretRawSchema + .merge( + z.object({ + secretPath: z.string().optional() + }) + ) + .array(), imports: z .object({ secretPath: z.string(), @@ -218,7 +229,8 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => { actorAuthMethod: req.permission.authMethod, projectId: workspaceId, path: secretPath, - includeImports: req.query.include_imports + includeImports: req.query.include_imports, + recursive: req.query.recursive }); await server.services.auditLog.createAuditLog({ diff --git a/backend/src/services/secret/secret-fns.ts b/backend/src/services/secret/secret-fns.ts index 212bb01f0..b499f9365 100644 --- a/backend/src/services/secret/secret-fns.ts +++ b/backend/src/services/secret/secret-fns.ts @@ -9,6 +9,7 @@ import { TSecretBlindIndexes, TSecrets } from "@app/db/schemas"; +import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service"; import { getConfig } from "@app/lib/config/env"; import { buildSecretBlindIndexFromName, @@ -17,8 +18,10 @@ import { } from "@app/lib/crypto"; import { BadRequestError } from "@app/lib/errors"; import { groupBy, unique } from "@app/lib/fn"; +import { logger } from "@app/lib/logger"; import { getBotKeyFnFactory } from "../project-bot/project-bot-fns"; +import { TProjectEnvDALFactory } from "../project-env/project-env-dal"; import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal"; import { TSecretDALFactory } from "./secret-dal"; import { @@ -45,6 +48,68 @@ export const generateSecretBlindIndexBySalt = async (secretName: string, secretB return secretBlindIndex; }; +type TRecursivelyFetchSecretsFromFoldersArg = { + permissionService: Pick; + projectEnvDAL: Pick; + folderDAL: Pick; +}; + +export const recursivelyGetSecretPaths = ({ folderDAL, projectEnvDAL }: TRecursivelyFetchSecretsFromFoldersArg) => { + const getPaths = async (projectId: string, environment: string, currentPath: string) => { + let secretPaths: string[] = []; + + // Get secrets in the current folder. + try { + const folder = await folderDAL.findBySecretPath(projectId, environment, currentPath); + + if (!folder) { + throw new Error(`Base directory '${currentPath}' not found.`); + } + + secretPaths.push(currentPath); + } catch (error) { + logger.error(error, "Error fetching secrets from base directory"); + throw error; + } + + // List all subfolders in the current folder. + try { + const env = await projectEnvDAL.findOne({ projectId, slug: environment }); + const parentFolder = await folderDAL.findBySecretPath(projectId, environment, currentPath); + + if (!env) { + throw new Error(`Environment with not found`); + } + + if (!parentFolder) { + throw new Error(`Parent folder not found`); + } + + const folders = await folderDAL.find({ envId: env.id, parentId: parentFolder.id }); + + // Use Promise.all to handle recursive calls concurrently for efficiency. + const secretsFromSubFolders = await Promise.all( + folders.map(async (folder) => { + // Ensure the path is correctly formatted for the next level. + const subFolderPath = `${currentPath}${currentPath !== "/" ? "/" : ""}${folder.name}`; + + return getPaths(projectId, environment, subFolderPath); + }) + ); + + // Flatten the array of arrays and concatenate with the current secrets array. + secretPaths = secretPaths.concat(...secretsFromSubFolders); + } catch (error) { + logger.error(error, "Error fetching secrets from subdirectories"); + throw error; + } + + return secretPaths; + }; + + return getPaths; +}; + type TInterpolateSecretArg = { projectId: string; secretEncKey: string; @@ -202,9 +267,7 @@ export const interpolateSecrets = ({ projectId, secretEncKey, secretDAL, folderD ); // eslint-disable-next-line - secrets[key].value = secrets[key].skipMultilineEncoding - ? expandedVal - : formatMultiValueEnv(expandedVal); + secrets[key].value = secrets[key].skipMultilineEncoding ? expandedVal : formatMultiValueEnv(expandedVal); } return secrets; diff --git a/backend/src/services/secret/secret-service.ts b/backend/src/services/secret/secret-service.ts index f8fed95bb..6b71f7988 100644 --- a/backend/src/services/secret/secret-service.ts +++ b/backend/src/services/secret/secret-service.ts @@ -13,13 +13,20 @@ import { logger } from "@app/lib/logger"; import { ActorType } from "../auth/auth-type"; import { TProjectDALFactory } from "../project/project-dal"; import { TProjectBotServiceFactory } from "../project-bot/project-bot-service"; +import { TProjectEnvDALFactory } from "../project-env/project-env-dal"; import { TSecretBlindIndexDALFactory } from "../secret-blind-index/secret-blind-index-dal"; import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal"; import { TSecretImportDALFactory } from "../secret-import/secret-import-dal"; import { fnSecretsFromImports } from "../secret-import/secret-import-fns"; import { TSecretTagDALFactory } from "../secret-tag/secret-tag-dal"; import { TSecretDALFactory } from "./secret-dal"; -import { decryptSecretRaw, fnSecretBlindIndexCheck, fnSecretBulkInsert, fnSecretBulkUpdate } from "./secret-fns"; +import { + decryptSecretRaw, + fnSecretBlindIndexCheck, + fnSecretBulkInsert, + fnSecretBulkUpdate, + recursivelyGetSecretPaths +} from "./secret-fns"; import { TSecretQueueFactory } from "./secret-queue"; import { TAttachSecretTagsDTO, @@ -47,8 +54,12 @@ type TSecretServiceFactoryDep = { secretDAL: TSecretDALFactory; secretTagDAL: TSecretTagDALFactory; secretVersionDAL: TSecretVersionDALFactory; - folderDAL: Pick; projectDAL: Pick; + projectEnvDAL: Pick; + folderDAL: Pick< + TSecretFolderDALFactory, + "findBySecretPath" | "updateById" | "findById" | "findByManySecretPath" | "find" + >; secretBlindIndexDAL: TSecretBlindIndexDALFactory; permissionService: Pick; snapshotService: Pick; @@ -61,6 +72,7 @@ type TSecretServiceFactoryDep = { export type TSecretServiceFactory = ReturnType; export const secretServiceFactory = ({ secretDAL, + projectEnvDAL, secretTagDAL, secretVersionDAL, folderDAL, @@ -789,21 +801,64 @@ export const secretServiceFactory = ({ actorOrgId, actorAuthMethod, environment, - includeImports + includeImports, + recursive }: TGetSecretsRawDTO) => { const botKey = await projectBotService.getBotKey(projectId); if (!botKey) throw new BadRequestError({ message: "Project bot not found", name: "bot_not_found_error" }); - const { secrets, imports } = await getSecrets({ - actorId, - projectId, - environment, - actor, - actorOrgId, - actorAuthMethod, - path, - includeImports - }); + let secrets: Awaited>["secrets"]; + let imports: Awaited>["imports"]; + + if (recursive) { + const getPaths = recursivelyGetSecretPaths({ + permissionService, + folderDAL, + projectEnvDAL + }); + + const paths = await getPaths(projectId, environment, path); + + const result = await Promise.all( + paths.map(async (currentPath) => { + const secs = await getSecrets({ + actorId, + projectId, + environment, + actor, + actorOrgId, + actorAuthMethod, + path: currentPath, + includeImports + }); + + return { + secrets: { + ...secs.secrets, + secretPath: currentPath + }, + imports: secs.imports + }; + }) + ); + + secrets = result.flatMap((el) => el.secrets); + imports = result.flatMap((el) => el.imports || []); + } else { + const result = await getSecrets({ + actorId, + projectId, + environment, + actor, + actorOrgId, + actorAuthMethod, + path, + includeImports + }); + + secrets = result.secrets; + imports = result.imports; + } return { secrets: secrets.map((el) => decryptSecretRaw(el, botKey)), diff --git a/backend/src/services/secret/secret-types.ts b/backend/src/services/secret/secret-types.ts index efd4f0f8b..c6cacb255 100644 --- a/backend/src/services/secret/secret-types.ts +++ b/backend/src/services/secret/secret-types.ts @@ -140,6 +140,7 @@ export type TGetSecretsRawDTO = { path: string; environment: string; includeImports?: boolean; + recursive?: boolean; } & TProjectPermission; export type TGetASecretRawDTO = { From f91a6683c2984e6267cc7d9c70b2bf5d1f2166f1 Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Fri, 22 Mar 2024 16:10:56 +0100 Subject: [PATCH 02/18] Fix: Rename parameter --- backend/src/lib/api-docs/constants.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/lib/api-docs/constants.ts b/backend/src/lib/api-docs/constants.ts index 4a45dfb41..bb1b73a77 100644 --- a/backend/src/lib/api-docs/constants.ts +++ b/backend/src/lib/api-docs/constants.ts @@ -215,7 +215,7 @@ export const SECRETS = { export const RAW_SECRETS = { LIST: { - recursive: "Whether or not to fetch all secrets from the specified base path, and all of its subdirectories.", + deep: "Whether or not to fetch all secrets from the specified base path, and all of its subdirectories.", workspaceId: "The ID of the project to list secrets from.", workspaceSlug: "The slug of the project to list secrets from. This parameter is only usable by machine identities.", environment: "The slug of the environment to list secrets from.", From 9ff3210ed68415ce59c6937010abc253363535e6 Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Fri, 22 Mar 2024 16:11:36 +0100 Subject: [PATCH 03/18] Feat: Recursively get all secrets from inside path --- backend/src/server/routes/index.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/src/server/routes/index.ts b/backend/src/server/routes/index.ts index aea5baa94..6e59cd783 100644 --- a/backend/src/server/routes/index.ts +++ b/backend/src/server/routes/index.ts @@ -479,6 +479,7 @@ export const registerRoutes = async ( snapshotService, secretQueueService, secretImportDAL, + projectEnvDAL, projectBotService }); const sarService = secretApprovalRequestServiceFactory({ From 566f7e4c612231fb1704145f882b2615ff8bb084 Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Fri, 22 Mar 2024 16:12:22 +0100 Subject: [PATCH 04/18] Feat: Recursively get all secrets from inside path --- backend/src/server/routes/v3/secret-router.ts | 14 +- backend/src/services/secret/secret-fns.ts | 59 +++++- backend/src/services/secret/secret-service.ts | 193 +++++++++--------- 3 files changed, 166 insertions(+), 100 deletions(-) diff --git a/backend/src/server/routes/v3/secret-router.ts b/backend/src/server/routes/v3/secret-router.ts index 8d3f08b61..07dff0703 100644 --- a/backend/src/server/routes/v3/secret-router.ts +++ b/backend/src/server/routes/v3/secret-router.ts @@ -157,11 +157,11 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => { workspaceSlug: z.string().trim().optional().describe(RAW_SECRETS.LIST.workspaceSlug), environment: z.string().trim().optional().describe(RAW_SECRETS.LIST.environment), secretPath: z.string().trim().default("/").transform(removeTrailingSlash).describe(RAW_SECRETS.LIST.secretPath), - recursive: z + deep: z .enum(["true", "false"]) .default("false") .transform((value) => value === "true") - .describe(RAW_SECRETS.LIST.recursive), + .describe(RAW_SECRETS.LIST.deep), include_imports: z .enum(["true", "false"]) .default("false") @@ -230,7 +230,7 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => { projectId: workspaceId, path: secretPath, includeImports: req.query.include_imports, - recursive: req.query.recursive + deep: req.query.deep }); await server.services.auditLog.createAuditLog({ @@ -608,6 +608,10 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => { workspaceId: z.string().trim(), environment: z.string().trim(), secretPath: z.string().trim().default("/").transform(removeTrailingSlash), + deep: z + .enum(["true", "false"]) + .default("false") + .transform((value) => value === "true"), include_imports: z .enum(["true", "false"]) .default("false") @@ -621,6 +625,7 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => { _id: z.string(), workspace: z.string(), environment: z.string(), + secretPath: z.string().optional(), tags: SecretTagsSchema.pick({ id: true, slug: true, @@ -660,7 +665,8 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => { environment: req.query.environment, projectId: req.query.workspaceId, path: req.query.secretPath, - includeImports: req.query.include_imports + includeImports: req.query.include_imports, + deep: req.query.deep }); await server.services.auditLog.createAuditLog({ diff --git a/backend/src/services/secret/secret-fns.ts b/backend/src/services/secret/secret-fns.ts index b499f9365..961f3fef6 100644 --- a/backend/src/services/secret/secret-fns.ts +++ b/backend/src/services/secret/secret-fns.ts @@ -1,4 +1,5 @@ /* eslint-disable no-await-in-loop */ +import { subject } from "@casl/ability"; import path from "path"; import { @@ -10,6 +11,7 @@ import { TSecrets } from "@app/db/schemas"; import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service"; +import { ProjectPermissionActions, ProjectPermissionSub } from "@app/ee/services/permission/project-permission"; import { getConfig } from "@app/lib/config/env"; import { buildSecretBlindIndexFromName, @@ -20,6 +22,7 @@ import { BadRequestError } from "@app/lib/errors"; import { groupBy, unique } from "@app/lib/fn"; import { logger } from "@app/lib/logger"; +import { ActorAuthMethod, ActorType } from "../auth/auth-type"; import { getBotKeyFnFactory } from "../project-bot/project-bot-fns"; import { TProjectEnvDALFactory } from "../project-env/project-env-dal"; import { TSecretFolderDALFactory } from "../secret-folder/secret-folder-dal"; @@ -54,8 +57,25 @@ type TRecursivelyFetchSecretsFromFoldersArg = { folderDAL: Pick; }; -export const recursivelyGetSecretPaths = ({ folderDAL, projectEnvDAL }: TRecursivelyFetchSecretsFromFoldersArg) => { - const getPaths = async (projectId: string, environment: string, currentPath: string) => { +type TGetPathsDTO = { + projectId: string; + environment: string; + currentPath: string; + + auth: { + actor: ActorType; + actorId: string; + actorAuthMethod: ActorAuthMethod; + actorOrgId: string | undefined; + }; +}; + +export const recursivelyGetSecretPaths = ({ + folderDAL, + projectEnvDAL, + permissionService +}: TRecursivelyFetchSecretsFromFoldersArg) => { + const getPaths = async ({ projectId, environment, currentPath }: Omit) => { let secretPaths: string[] = []; // Get secrets in the current folder. @@ -93,7 +113,7 @@ export const recursivelyGetSecretPaths = ({ folderDAL, projectEnvDAL }: TRecursi // Ensure the path is correctly formatted for the next level. const subFolderPath = `${currentPath}${currentPath !== "/" ? "/" : ""}${folder.name}`; - return getPaths(projectId, environment, subFolderPath); + return getPaths({ projectId, environment, currentPath: subFolderPath }); }) ); @@ -107,7 +127,32 @@ export const recursivelyGetSecretPaths = ({ folderDAL, projectEnvDAL }: TRecursi return secretPaths; }; - return getPaths; + return async ({ projectId, environment, currentPath, auth }: TGetPathsDTO) => { + const paths = await getPaths({ projectId, environment, currentPath }); + + const { permission } = await permissionService.getProjectPermission( + auth.actor, + auth.actorId, + projectId, + auth.actorAuthMethod, + auth.actorOrgId + ); + + const allowedPaths = paths.filter((p) => + // if its service token allow full access over imported one + auth.actor === ActorType.SERVICE + ? true + : permission.can( + ProjectPermissionActions.Read, + subject(ProjectPermissionSub.Secrets, { + environment, + secretPath: p + }) + ) + ); + + return allowedPaths; + }; }; type TInterpolateSecretArg = { @@ -275,7 +320,10 @@ export const interpolateSecrets = ({ projectId, secretEncKey, secretDAL, folderD return expandSecrets; }; -export const decryptSecretRaw = (secret: TSecrets & { workspace: string; environment: string }, key: string) => { +export const decryptSecretRaw = ( + secret: TSecrets & { workspace: string; environment: string; secretPath?: string }, + key: string +) => { const secretKey = decryptSymmetric128BitHexKeyUTF8({ ciphertext: secret.secretKeyCiphertext, iv: secret.secretKeyIV, @@ -303,6 +351,7 @@ export const decryptSecretRaw = (secret: TSecrets & { workspace: string; environ return { secretKey, + secretPath: secret.secretPath, workspace: secret.workspace, environment: secret.environment, secretValue, diff --git a/backend/src/services/secret/secret-service.ts b/backend/src/services/secret/secret-service.ts index 6b71f7988..57ef22c56 100644 --- a/backend/src/services/secret/secret-service.ts +++ b/backend/src/services/secret/secret-service.ts @@ -1,3 +1,5 @@ +/* eslint-disable no-unreachable-loop */ +/* eslint-disable no-await-in-loop */ import { ForbiddenError, subject } from "@casl/ability"; import { SecretEncryptionAlgo, SecretKeyEncoding, SecretsSchema, SecretType } from "@app/db/schemas"; @@ -437,51 +439,98 @@ export const secretServiceFactory = ({ actor, actorOrgId, actorAuthMethod, - includeImports + includeImports, + deep }: TGetSecretsDTO) => { - const { permission } = await permissionService.getProjectPermission( - actor, - actorId, - projectId, - actorAuthMethod, - actorOrgId - ); - ForbiddenError.from(permission).throwUnlessCan( - ProjectPermissionActions.Read, - subject(ProjectPermissionSub.Secrets, { environment, secretPath: path }) - ); + const importsArray: Awaited> = []; + const secretsArray: (Awaited>[number] & { + secretPath: string; + environment: string; + workspace: string; + })[] = []; - const folder = await folderDAL.findBySecretPath(projectId, environment, path); - if (!folder) return { secrets: [], imports: [] }; - const folderId = folder.id; + let paths = [path]; - const secrets = await secretDAL.findByFolderId(folderId, actorId); + if (deep) { + const getPaths = recursivelyGetSecretPaths({ + permissionService, + folderDAL, + projectEnvDAL + }); + + const deepPaths = await getPaths({ + projectId, + environment, + currentPath: path, + auth: { + actor, + actorId, + actorAuthMethod, + actorOrgId + } + }); + + paths = deepPaths.length === 0 ? paths : deepPaths; + } + + for (const currentPath of paths) { + const { permission } = await permissionService.getProjectPermission( + actor, + actorId, + projectId, + actorAuthMethod, + actorOrgId + ); + ForbiddenError.from(permission).throwUnlessCan( + ProjectPermissionActions.Read, + subject(ProjectPermissionSub.Secrets, { environment, secretPath: currentPath }) + ); + + const folder = await folderDAL.findBySecretPath(projectId, environment, currentPath); + if (!folder) return { secrets: [], imports: [] }; + const folderId = folder.id; + + const secrets = await secretDAL.findByFolderId(folderId, actorId); + if (includeImports) { + const secretImports = await secretImportDAL.find({ folderId }); + const allowedImports = secretImports.filter(({ importEnv, importPath }) => + // if its service token allow full access over imported one + actor === ActorType.SERVICE + ? true + : permission.can( + ProjectPermissionActions.Read, + subject(ProjectPermissionSub.Secrets, { + environment: importEnv.slug, + secretPath: importPath + }) + ) + ); + const importedSecrets = await fnSecretsFromImports({ + allowedImports, + secretDAL, + folderDAL + }); + + secretsArray.push( + ...secrets.map((secret) => ({ ...secret, secretPath: currentPath, workspace: projectId, environment })) + ); + importsArray.push(...importedSecrets); + } + secretsArray.push( + ...secrets.map((secret) => ({ ...secret, secretPath: currentPath, workspace: projectId, environment })) + ); + } if (includeImports) { - const secretImports = await secretImportDAL.find({ folderId }); - const allowedImports = secretImports.filter(({ importEnv, importPath }) => - // if its service token allow full access over imported one - actor === ActorType.SERVICE - ? true - : permission.can( - ProjectPermissionActions.Read, - subject(ProjectPermissionSub.Secrets, { - environment: importEnv.slug, - secretPath: importPath - }) - ) - ); - const importedSecrets = await fnSecretsFromImports({ - allowedImports, - secretDAL, - folderDAL - }); return { - secrets: secrets.map((el) => ({ ...el, workspace: projectId, environment })), - imports: importedSecrets + secrets: secretsArray, + imports: importsArray }; } - return { secrets: secrets.map((el) => ({ ...el, workspace: projectId, environment })) }; + + return { + secrets: secretsArray + }; }; const getSecretByName = async ({ @@ -802,70 +851,32 @@ export const secretServiceFactory = ({ actorAuthMethod, environment, includeImports, - recursive + deep }: TGetSecretsRawDTO) => { const botKey = await projectBotService.getBotKey(projectId); if (!botKey) throw new BadRequestError({ message: "Project bot not found", name: "bot_not_found_error" }); - let secrets: Awaited>["secrets"]; - let imports: Awaited>["imports"]; - - if (recursive) { - const getPaths = recursivelyGetSecretPaths({ - permissionService, - folderDAL, - projectEnvDAL - }); - - const paths = await getPaths(projectId, environment, path); - - const result = await Promise.all( - paths.map(async (currentPath) => { - const secs = await getSecrets({ - actorId, - projectId, - environment, - actor, - actorOrgId, - actorAuthMethod, - path: currentPath, - includeImports - }); - - return { - secrets: { - ...secs.secrets, - secretPath: currentPath - }, - imports: secs.imports - }; - }) - ); - - secrets = result.flatMap((el) => el.secrets); - imports = result.flatMap((el) => el.imports || []); - } else { - const result = await getSecrets({ - actorId, - projectId, - environment, - actor, - actorOrgId, - actorAuthMethod, - path, - includeImports - }); - - secrets = result.secrets; - imports = result.imports; - } + const { secrets, imports } = await getSecrets({ + actorId, + projectId, + environment, + actor, + actorOrgId, + actorAuthMethod, + path, + includeImports, + deep + }); return { secrets: secrets.map((el) => decryptSecretRaw(el, botKey)), imports: (imports || [])?.map(({ secrets: importedSecrets, ...el }) => ({ ...el, secrets: importedSecrets.map((sec) => - decryptSecretRaw({ ...sec, environment: el.environment, workspace: projectId }, botKey) + decryptSecretRaw( + { ...sec, environment: el.environment, workspace: projectId, secretPath: el.secretPath }, + botKey + ) ) })) }; From 46a638cc6348183e06128da366c3ab3751d8e4a9 Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Fri, 22 Mar 2024 16:12:40 +0100 Subject: [PATCH 05/18] FIx: Rename parameter from recursive to deep --- backend/src/services/secret/secret-types.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/backend/src/services/secret/secret-types.ts b/backend/src/services/secret/secret-types.ts index c6cacb255..33e46b67f 100644 --- a/backend/src/services/secret/secret-types.ts +++ b/backend/src/services/secret/secret-types.ts @@ -74,6 +74,7 @@ export type TGetSecretsDTO = { path: string; environment: string; includeImports?: boolean; + deep?: boolean; } & TProjectPermission; export type TGetASecretDTO = { @@ -140,7 +141,7 @@ export type TGetSecretsRawDTO = { path: string; environment: string; includeImports?: boolean; - recursive?: boolean; + deep?: boolean; } & TProjectPermission; export type TGetASecretRawDTO = { From 88549f403042195e955cd4f86221b09bb50d1c4c Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Fri, 22 Mar 2024 16:13:05 +0100 Subject: [PATCH 06/18] Feat: Deep search support --- cli/packages/api/api.go | 4 ++++ cli/packages/api/model.go | 1 + cli/packages/cmd/agent.go | 2 +- cli/packages/cmd/run.go | 8 +++++++- cli/packages/cmd/secrets.go | 17 +++++++++++++++-- cli/packages/models/cli.go | 1 + cli/packages/util/secrets.go | 15 +++++++++------ 7 files changed, 38 insertions(+), 10 deletions(-) diff --git a/cli/packages/api/api.go b/cli/packages/api/api.go index 38d82a0a5..38af4630e 100644 --- a/cli/packages/api/api.go +++ b/cli/packages/api/api.go @@ -277,6 +277,10 @@ func CallGetSecretsV3(httpClient *resty.Client, request GetEncryptedSecretsV3Req SetQueryParam("environment", request.Environment). SetQueryParam("workspaceId", request.WorkspaceId) + if request.DeepSearch { + httpRequest.SetQueryParam("deep", "true") + } + if request.IncludeImport { httpRequest.SetQueryParam("include_imports", "true") } diff --git a/cli/packages/api/model.go b/cli/packages/api/model.go index b49cb1581..1b91c8b64 100644 --- a/cli/packages/api/model.go +++ b/cli/packages/api/model.go @@ -291,6 +291,7 @@ type GetEncryptedSecretsV3Request struct { WorkspaceId string `json:"workspaceId"` SecretPath string `json:"secretPath"` IncludeImport bool `json:"include_imports"` + DeepSearch bool `json:"deep"` } type GetFoldersV1Request struct { diff --git a/cli/packages/cmd/agent.go b/cli/packages/cmd/agent.go index ce3f7a8b0..80f321396 100644 --- a/cli/packages/cmd/agent.go +++ b/cli/packages/cmd/agent.go @@ -332,7 +332,7 @@ func ParseAgentConfig(configFile []byte) (*Config, error) { func secretTemplateFunction(accessToken string, existingEtag string, currentEtag *string) func(string, string, string) ([]models.SingleEnvironmentVariable, error) { return func(projectID, envSlug, secretPath string) ([]models.SingleEnvironmentVariable, error) { - res, err := util.GetPlainTextSecretsViaMachineIdentity(accessToken, projectID, envSlug, secretPath, false) + res, err := util.GetPlainTextSecretsViaMachineIdentity(accessToken, projectID, envSlug, secretPath, false, false) if err != nil { return nil, err } diff --git a/cli/packages/cmd/run.go b/cli/packages/cmd/run.go index d008af1ec..f6e675096 100644 --- a/cli/packages/cmd/run.go +++ b/cli/packages/cmd/run.go @@ -98,7 +98,12 @@ var runCmd = &cobra.Command{ util.HandleError(err, "Unable to parse flag") } - secrets, err := util.GetAllEnvironmentVariables(models.GetAllSecretsParameters{Environment: environmentName, InfisicalToken: infisicalToken, TagSlugs: tagSlugs, SecretsPath: secretsPath, IncludeImport: includeImports}, projectConfigDir) + deepSearch, err := cmd.Flags().GetBool("deep") + if err != nil { + util.HandleError(err, "Unable to parse flag") + } + + secrets, err := util.GetAllEnvironmentVariables(models.GetAllSecretsParameters{Environment: environmentName, InfisicalToken: infisicalToken, TagSlugs: tagSlugs, SecretsPath: secretsPath, IncludeImport: includeImports, DeepSearch: deepSearch}, projectConfigDir) if err != nil { util.HandleError(err, "Could not fetch secrets", "If you are using a service token to fetch secrets, please ensure it is valid") @@ -202,6 +207,7 @@ func init() { runCmd.Flags().StringP("env", "e", "dev", "Set the environment (dev, prod, etc.) from which your secrets should be pulled from") runCmd.Flags().Bool("expand", true, "Parse shell parameter expansions in your secrets") runCmd.Flags().Bool("include-imports", true, "Import linked secrets ") + runCmd.Flags().Bool("deep", false, "Fetch secrets from all sub-folders") runCmd.Flags().Bool("secret-overriding", true, "Prioritizes personal secrets, if any, with the same name over shared secrets") runCmd.Flags().StringP("command", "c", "", "chained commands to execute (e.g. \"npm install && npm run dev; echo ...\")") runCmd.Flags().StringP("tags", "t", "", "filter secrets by tag slugs ") diff --git a/cli/packages/cmd/secrets.go b/cli/packages/cmd/secrets.go index cf9c89b47..1db565b76 100644 --- a/cli/packages/cmd/secrets.go +++ b/cli/packages/cmd/secrets.go @@ -63,6 +63,13 @@ var secretsCmd = &cobra.Command{ util.HandleError(err) } + deepSearch, err := cmd.Flags().GetBool("deep") + if err != nil { + util.HandleError(err) + } + + fmt.Printf("Is deep search: %v\n", deepSearch) + tagSlugs, err := cmd.Flags().GetString("tags") if err != nil { util.HandleError(err, "Unable to parse flag") @@ -73,7 +80,7 @@ var secretsCmd = &cobra.Command{ util.HandleError(err, "Unable to parse flag") } - secrets, err := util.GetAllEnvironmentVariables(models.GetAllSecretsParameters{Environment: environmentName, InfisicalToken: infisicalToken, TagSlugs: tagSlugs, SecretsPath: secretsPath, IncludeImport: includeImports}, "") + secrets, err := util.GetAllEnvironmentVariables(models.GetAllSecretsParameters{Environment: environmentName, InfisicalToken: infisicalToken, TagSlugs: tagSlugs, SecretsPath: secretsPath, IncludeImport: includeImports, DeepSearch: deepSearch}, "") if err != nil { util.HandleError(err) } @@ -413,12 +420,17 @@ func getSecretsByNames(cmd *cobra.Command, args []string) { util.HandleError(err, "Unable to parse path flag") } + deepSearch, err := cmd.Flags().GetBool("deep") + if err != nil { + util.HandleError(err, "Unable to parse deep flag") + } + showOnlyValue, err := cmd.Flags().GetBool("raw-value") if err != nil { util.HandleError(err, "Unable to parse path flag") } - secrets, err := util.GetAllEnvironmentVariables(models.GetAllSecretsParameters{Environment: environmentName, InfisicalToken: infisicalToken, TagSlugs: tagSlugs, SecretsPath: secretsPath, IncludeImport: true}, "") + secrets, err := util.GetAllEnvironmentVariables(models.GetAllSecretsParameters{Environment: environmentName, InfisicalToken: infisicalToken, TagSlugs: tagSlugs, SecretsPath: secretsPath, IncludeImport: true, DeepSearch: deepSearch}, "") if err != nil { util.HandleError(err, "To fetch all secrets") } @@ -727,6 +739,7 @@ func init() { secretsCmd.PersistentFlags().String("env", "dev", "Used to select the environment name on which actions should be taken on") secretsCmd.Flags().Bool("expand", true, "Parse shell parameter expansions in your secrets") secretsCmd.Flags().Bool("include-imports", true, "Imported linked secrets ") + secretsCmd.Flags().Bool("deep", false, "Fetch secrets from all sub-folders") secretsCmd.PersistentFlags().StringP("tags", "t", "", "filter secrets by tag slugs") secretsCmd.Flags().String("path", "/", "get secrets within a folder path") rootCmd.AddCommand(secretsCmd) diff --git a/cli/packages/models/cli.go b/cli/packages/models/cli.go index 4a7dc782a..69319e2b2 100644 --- a/cli/packages/models/cli.go +++ b/cli/packages/models/cli.go @@ -93,6 +93,7 @@ type GetAllSecretsParameters struct { WorkspaceId string SecretsPath string IncludeImport bool + DeepSearch bool } type GetAllFoldersParameters struct { diff --git a/cli/packages/util/secrets.go b/cli/packages/util/secrets.go index 6f56bc84a..89d094f43 100644 --- a/cli/packages/util/secrets.go +++ b/cli/packages/util/secrets.go @@ -17,7 +17,7 @@ import ( "github.com/rs/zerolog/log" ) -func GetPlainTextSecretsViaServiceToken(fullServiceToken string, environment string, secretPath string, includeImports bool) ([]models.SingleEnvironmentVariable, api.GetServiceTokenDetailsResponse, error) { +func GetPlainTextSecretsViaServiceToken(fullServiceToken string, environment string, secretPath string, includeImports bool, deepSearch bool) ([]models.SingleEnvironmentVariable, api.GetServiceTokenDetailsResponse, error) { serviceTokenParts := strings.SplitN(fullServiceToken, ".", 4) if len(serviceTokenParts) < 4 { return nil, api.GetServiceTokenDetailsResponse{}, fmt.Errorf("invalid service token entered. Please double check your service token and try again") @@ -49,6 +49,7 @@ func GetPlainTextSecretsViaServiceToken(fullServiceToken string, environment str Environment: environment, SecretPath: secretPath, IncludeImport: includeImports, + DeepSearch: deepSearch, }) if err != nil { @@ -80,7 +81,7 @@ func GetPlainTextSecretsViaServiceToken(fullServiceToken string, environment str return plainTextSecrets, serviceTokenDetails, nil } -func GetPlainTextSecretsViaJTW(JTWToken string, receiversPrivateKey string, workspaceId string, environmentName string, tagSlugs string, secretsPath string, includeImports bool) ([]models.SingleEnvironmentVariable, error) { +func GetPlainTextSecretsViaJTW(JTWToken string, receiversPrivateKey string, workspaceId string, environmentName string, tagSlugs string, secretsPath string, includeImports bool, deepSearch bool) ([]models.SingleEnvironmentVariable, error) { httpClient := resty.New() httpClient.SetAuthToken(JTWToken). SetHeader("Accept", "application/json") @@ -125,6 +126,7 @@ func GetPlainTextSecretsViaJTW(JTWToken string, receiversPrivateKey string, work WorkspaceId: workspaceId, Environment: environmentName, IncludeImport: includeImports, + DeepSearch: deepSearch, // TagSlugs: tagSlugs, } @@ -152,7 +154,7 @@ func GetPlainTextSecretsViaJTW(JTWToken string, receiversPrivateKey string, work return plainTextSecrets, nil } -func GetPlainTextSecretsViaMachineIdentity(accessToken string, workspaceId string, environmentName string, secretsPath string, includeImports bool) (models.PlaintextSecretResult, error) { +func GetPlainTextSecretsViaMachineIdentity(accessToken string, workspaceId string, environmentName string, secretsPath string, includeImports bool, deepSearch bool) (models.PlaintextSecretResult, error) { httpClient := resty.New() httpClient.SetAuthToken(accessToken). SetHeader("Accept", "application/json") @@ -161,6 +163,7 @@ func GetPlainTextSecretsViaMachineIdentity(accessToken string, workspaceId strin WorkspaceId: workspaceId, Environment: environmentName, IncludeImport: includeImports, + DeepSearch: deepSearch, // TagSlugs: tagSlugs, } @@ -329,7 +332,7 @@ func GetAllEnvironmentVariables(params models.GetAllSecretsParameters, projectCo } secretsToReturn, errorToReturn = GetPlainTextSecretsViaJTW(loggedInUserDetails.UserCredentials.JTWToken, loggedInUserDetails.UserCredentials.PrivateKey, infisicalDotJson.WorkspaceId, - params.Environment, params.TagSlugs, params.SecretsPath, params.IncludeImport) + params.Environment, params.TagSlugs, params.SecretsPath, params.IncludeImport, params.DeepSearch) log.Debug().Msgf("GetAllEnvironmentVariables: Trying to fetch secrets JTW token [err=%s]", errorToReturn) backupSecretsEncryptionKey := []byte(loggedInUserDetails.UserCredentials.PrivateKey)[0:32] @@ -350,10 +353,10 @@ func GetAllEnvironmentVariables(params models.GetAllSecretsParameters, projectCo } else { if params.InfisicalToken != "" { log.Debug().Msg("Trying to fetch secrets using service token") - secretsToReturn, _, errorToReturn = GetPlainTextSecretsViaServiceToken(params.InfisicalToken, params.Environment, params.SecretsPath, params.IncludeImport) + secretsToReturn, _, errorToReturn = GetPlainTextSecretsViaServiceToken(params.InfisicalToken, params.Environment, params.SecretsPath, params.IncludeImport, params.DeepSearch) } else if params.UniversalAuthAccessToken != "" { log.Debug().Msg("Trying to fetch secrets using universal auth") - res, err := GetPlainTextSecretsViaMachineIdentity(params.UniversalAuthAccessToken, params.WorkspaceId, params.Environment, params.SecretsPath, params.IncludeImport) + res, err := GetPlainTextSecretsViaMachineIdentity(params.UniversalAuthAccessToken, params.WorkspaceId, params.Environment, params.SecretsPath, params.IncludeImport, params.DeepSearch) errorToReturn = err secretsToReturn = res.Secrets From 3befd90723a7212e83d359a5d4e115b4da0ad242 Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Sun, 24 Mar 2024 17:10:22 +0100 Subject: [PATCH 07/18] Fix: Refactor to in-memory approach --- backend/src/services/secret/secret-fns.ts | 117 ++++++++++++---------- 1 file changed, 62 insertions(+), 55 deletions(-) diff --git a/backend/src/services/secret/secret-fns.ts b/backend/src/services/secret/secret-fns.ts index 961f3fef6..3c7c9680c 100644 --- a/backend/src/services/secret/secret-fns.ts +++ b/backend/src/services/secret/secret-fns.ts @@ -8,6 +8,7 @@ import { SecretType, TableName, TSecretBlindIndexes, + TSecretFolders, TSecrets } from "@app/db/schemas"; import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service"; @@ -20,7 +21,6 @@ import { } from "@app/lib/crypto"; import { BadRequestError } from "@app/lib/errors"; import { groupBy, unique } from "@app/lib/fn"; -import { logger } from "@app/lib/logger"; import { ActorAuthMethod, ActorType } from "../auth/auth-type"; import { getBotKeyFnFactory } from "../project-bot/project-bot-fns"; @@ -53,8 +53,8 @@ export const generateSecretBlindIndexBySalt = async (secretName: string, secretB type TRecursivelyFetchSecretsFromFoldersArg = { permissionService: Pick; - projectEnvDAL: Pick; folderDAL: Pick; + projectEnvDAL: Pick; }; type TGetPathsDTO = { @@ -70,65 +70,73 @@ type TGetPathsDTO = { }; }; +// Introduce a new interface for mapping parent IDs to their children +interface FolderMap { + [parentId: string]: TSecretFolders[]; +} + export const recursivelyGetSecretPaths = ({ folderDAL, projectEnvDAL, permissionService }: TRecursivelyFetchSecretsFromFoldersArg) => { - const getPaths = async ({ projectId, environment, currentPath }: Omit) => { - let secretPaths: string[] = []; + const buildHierarchy = (folders: TSecretFolders[]): FolderMap => { + const map: FolderMap = {}; + map.null = []; // Initialize mapping for root directory - // Get secrets in the current folder. - try { - const folder = await folderDAL.findBySecretPath(projectId, environment, currentPath); - - if (!folder) { - throw new Error(`Base directory '${currentPath}' not found.`); + folders.forEach((folder) => { + const parentId = folder.parentId || "null"; + if (!map[parentId]) { + map[parentId] = []; } + map[parentId].push(folder); + }); - secretPaths.push(currentPath); - } catch (error) { - logger.error(error, "Error fetching secrets from base directory"); - throw error; - } + return map; + }; - // List all subfolders in the current folder. - try { - const env = await projectEnvDAL.findOne({ projectId, slug: environment }); - const parentFolder = await folderDAL.findBySecretPath(projectId, environment, currentPath); + const generatePaths = (map: FolderMap, parentId: string = "null", basePath: string = ""): string[] => { + const children = map[parentId || "null"] || []; + let paths: string[] = []; - if (!env) { - throw new Error(`Environment with not found`); - } + children.forEach((child) => { + // Determine if this is the root folder of the environment. If no parentId is present and the name is root, it's the root folder + const isRootFolder = child.name === "root" && !child.parentId; - if (!parentFolder) { - throw new Error(`Parent folder not found`); - } + // Form the current path based on the base path and the current child + // eslint-disable-next-line no-nested-ternary + const currPath = basePath === "" ? (isRootFolder ? "/" : `/${child.name}`) : `${basePath}/${child.name}`; - const folders = await folderDAL.find({ envId: env.id, parentId: parentFolder.id }); + paths.push(currPath); // Add the current path - // Use Promise.all to handle recursive calls concurrently for efficiency. - const secretsFromSubFolders = await Promise.all( - folders.map(async (folder) => { - // Ensure the path is correctly formatted for the next level. - const subFolderPath = `${currentPath}${currentPath !== "/" ? "/" : ""}${folder.name}`; + // Recursively generate paths for children, passing down the formatted pathh + const childPaths = generatePaths(map, child.id, currPath); + paths = paths.concat(childPaths); + }); - return getPaths({ projectId, environment, currentPath: subFolderPath }); - }) - ); - - // Flatten the array of arrays and concatenate with the current secrets array. - secretPaths = secretPaths.concat(...secretsFromSubFolders); - } catch (error) { - logger.error(error, "Error fetching secrets from subdirectories"); - throw error; - } - - return secretPaths; + return paths; }; return async ({ projectId, environment, currentPath, auth }: TGetPathsDTO) => { - const paths = await getPaths({ projectId, environment, currentPath }); + const env = await projectEnvDAL.findOne({ + projectId, + slug: environment + }); + + if (!env) { + throw new Error(`'${environment}' environment not found in project with ID ${projectId}`); + } + + // Fetch all folders in env once with a single query + const folders = await folderDAL.find({ + envId: env.id + }); + + // Build the folder hierarchy map + const folderMap = buildHierarchy(folders); + + // Generate the paths paths and normalize the root path toe / + const paths = generatePaths(folderMap).map((p) => (p === "/" ? p : p.substring(1))); const { permission } = await permissionService.getProjectPermission( auth.actor, @@ -138,17 +146,16 @@ export const recursivelyGetSecretPaths = ({ auth.actorOrgId ); - const allowedPaths = paths.filter((p) => - // if its service token allow full access over imported one - auth.actor === ActorType.SERVICE - ? true - : permission.can( - ProjectPermissionActions.Read, - subject(ProjectPermissionSub.Secrets, { - environment, - secretPath: p - }) - ) + // Filter out paths that the user does not have permission to access, and paths that are not in the current path + const allowedPaths = paths.filter( + (p) => + permission.can( + ProjectPermissionActions.Read, + subject(ProjectPermissionSub.Secrets, { + environment, + secretPath: p + }) + ) && p.startsWith(currentPath === "/" ? "" : currentPath) ); return allowedPaths; From 1e1b5d655ec73659bd73a251d0b782e4fb90707a Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Tue, 26 Mar 2024 19:15:29 +0100 Subject: [PATCH 08/18] Fix: Refactored secret fetching to be more performant --- backend/src/services/secret/secret-service.ts | 108 +++++++++--------- 1 file changed, 53 insertions(+), 55 deletions(-) diff --git a/backend/src/services/secret/secret-service.ts b/backend/src/services/secret/secret-service.ts index 57ef22c56..0b2d6262e 100644 --- a/backend/src/services/secret/secret-service.ts +++ b/backend/src/services/secret/secret-service.ts @@ -67,7 +67,7 @@ type TSecretServiceFactoryDep = { snapshotService: Pick; secretQueueService: Pick; projectBotService: Pick; - secretImportDAL: Pick; + secretImportDAL: Pick; secretVersionTagDAL: Pick; }; @@ -442,14 +442,15 @@ export const secretServiceFactory = ({ includeImports, deep }: TGetSecretsDTO) => { - const importsArray: Awaited> = []; - const secretsArray: (Awaited>[number] & { - secretPath: string; - environment: string; - workspace: string; - })[] = []; + const { permission } = await permissionService.getProjectPermission( + actor, + actorId, + projectId, + actorAuthMethod, + actorOrgId + ); - let paths = [path]; + let paths: { folderId: string; path: string }[] = []; if (deep) { const getPaths = recursivelyGetSecretPaths({ @@ -471,65 +472,62 @@ export const secretServiceFactory = ({ }); paths = deepPaths.length === 0 ? paths : deepPaths; - } + } else { + const folder = await folderDAL.findBySecretPath(projectId, environment, path); + if (!folder) return { secrets: [], imports: [] }; + paths = [{ folderId: folder.id, path }]; - for (const currentPath of paths) { - const { permission } = await permissionService.getProjectPermission( - actor, - actorId, - projectId, - actorAuthMethod, - actorOrgId - ); ForbiddenError.from(permission).throwUnlessCan( ProjectPermissionActions.Read, - subject(ProjectPermissionSub.Secrets, { environment, secretPath: currentPath }) - ); - - const folder = await folderDAL.findBySecretPath(projectId, environment, currentPath); - if (!folder) return { secrets: [], imports: [] }; - const folderId = folder.id; - - const secrets = await secretDAL.findByFolderId(folderId, actorId); - if (includeImports) { - const secretImports = await secretImportDAL.find({ folderId }); - const allowedImports = secretImports.filter(({ importEnv, importPath }) => - // if its service token allow full access over imported one - actor === ActorType.SERVICE - ? true - : permission.can( - ProjectPermissionActions.Read, - subject(ProjectPermissionSub.Secrets, { - environment: importEnv.slug, - secretPath: importPath - }) - ) - ); - const importedSecrets = await fnSecretsFromImports({ - allowedImports, - secretDAL, - folderDAL - }); - - secretsArray.push( - ...secrets.map((secret) => ({ ...secret, secretPath: currentPath, workspace: projectId, environment })) - ); - importsArray.push(...importedSecrets); - } - secretsArray.push( - ...secrets.map((secret) => ({ ...secret, secretPath: currentPath, workspace: projectId, environment })) + subject(ProjectPermissionSub.Secrets, { environment, secretPath: folder.path }) ); } + const groupedPaths = groupBy(paths, (p) => p.folderId); + + const secrets = await secretDAL.findByFolderIds( + paths.map((p) => p.folderId), + actorId + ); + if (includeImports) { + const secretImports = await secretImportDAL.findByFolderIds(paths.map((p) => p.folderId)); + const allowedImports = secretImports.filter(({ importEnv, importPath }) => + // if its service token allow full access over imported one + actor === ActorType.SERVICE + ? true + : permission.can( + ProjectPermissionActions.Read, + subject(ProjectPermissionSub.Secrets, { + environment: importEnv.slug, + secretPath: importPath + }) + ) + ); + const importedSecrets = await fnSecretsFromImports({ + allowedImports, + secretDAL, + folderDAL + }); + return { - secrets: secretsArray, - imports: importsArray + secrets: secrets.map((secret) => ({ + ...secret, + workspace: projectId, + environment, + secretPath: groupedPaths[secret.folderId][0].path + })), + imports: importedSecrets }; } return { - secrets: secretsArray + secrets: secrets.map((secret) => ({ + ...secret, + workspace: projectId, + environment, + secretPath: groupedPaths[secret.folderId][0].path + })) }; }; From 50082e192cd56c203a0597256c664bc7e467b4c6 Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Tue, 26 Mar 2024 19:26:30 +0100 Subject: [PATCH 09/18] Feat: Recursively get all secrets, findByFolderIds DLA --- .../secret-import/secret-import-dal.ts | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/backend/src/services/secret-import/secret-import-dal.ts b/backend/src/services/secret-import/secret-import-dal.ts index f9c6f1be7..cbc8936e1 100644 --- a/backend/src/services/secret-import/secret-import-dal.ts +++ b/backend/src/services/secret-import/secret-import-dal.ts @@ -70,9 +70,31 @@ export const secretImportDALFactory = (db: TDbClient) => { } }; + const findByFolderIds = async (folderIds: string[], tx?: Knex) => { + try { + const docs = await (tx || db)(TableName.SecretImport) + .whereIn("folderId", folderIds) + .join(TableName.Environment, `${TableName.SecretImport}.importEnv`, `${TableName.Environment}.id`) + .select( + db.ref("*").withSchema(TableName.SecretImport) as unknown as keyof TSecretImports, + db.ref("slug").withSchema(TableName.Environment), + db.ref("name").withSchema(TableName.Environment), + db.ref("id").withSchema(TableName.Environment).as("envId") + ) + .orderBy("position", "asc"); + return docs.map(({ envId, slug, name, ...el }) => ({ + ...el, + importEnv: { id: envId, slug, name } + })); + } catch (error) { + throw new DatabaseError({ error, name: "Find secret imports" }); + } + }; + return { ...secretImportOrm, find, + findByFolderIds, findLastImportPosition, updateAllPosition }; From 812fe5cf313baa0f0361fbbd036e05ed61481db7 Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Tue, 26 Mar 2024 19:26:55 +0100 Subject: [PATCH 10/18] Feat: Recursively get all secrets --- backend/src/services/secret/secret-dal.ts | 45 +++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/backend/src/services/secret/secret-dal.ts b/backend/src/services/secret/secret-dal.ts index 504174765..8a5970b83 100644 --- a/backend/src/services/secret/secret-dal.ts +++ b/backend/src/services/secret/secret-dal.ts @@ -171,6 +171,50 @@ export const secretDALFactory = (db: TDbClient) => { } }; + const findByFolderIds = async (folderIds: string[], userId?: string, tx?: Knex) => { + try { + // check if not uui then userId id is null (corner case because service token's ID is not UUI in effort to keep backwards compatibility from mongo) + if (userId && !uuidValidate(userId)) { + // eslint-disable-next-line no-param-reassign + userId = undefined; + } + + const secs = await (tx || db)(TableName.Secret) + .whereIn("folderId", folderIds) + .where((bd) => { + void bd.whereNull("userId").orWhere({ userId: userId || null }); + }) + .leftJoin(TableName.JnSecretTag, `${TableName.Secret}.id`, `${TableName.JnSecretTag}.${TableName.Secret}Id`) + .leftJoin(TableName.SecretTag, `${TableName.JnSecretTag}.${TableName.SecretTag}Id`, `${TableName.SecretTag}.id`) + .select(selectAllTableCols(TableName.Secret)) + .select(db.ref("id").withSchema(TableName.SecretTag).as("tagId")) + .select(db.ref("color").withSchema(TableName.SecretTag).as("tagColor")) + .select(db.ref("slug").withSchema(TableName.SecretTag).as("tagSlug")) + .select(db.ref("name").withSchema(TableName.SecretTag).as("tagName")) + .orderBy("id", "asc"); + const data = sqlNestRelationships({ + data: secs, + key: "id", + parentMapper: (el) => ({ _id: el.id, ...SecretsSchema.parse(el) }), + childrenMapper: [ + { + key: "tagId", + label: "tags" as const, + mapper: ({ tagId: id, tagColor: color, tagSlug: slug, tagName: name }) => ({ + id, + color, + slug, + name + }) + } + ] + }); + return data; + } catch (error) { + throw new DatabaseError({ error, name: "get all secret" }); + } + }; + const findByBlindIndexes = async ( folderId: string, blindIndexes: Array<{ blindIndex: string; type: SecretType }>, @@ -207,6 +251,7 @@ export const secretDALFactory = (db: TDbClient) => { bulkUpdateNoVersionIncrement, getSecretTags, findByFolderId, + findByFolderIds, findByBlindIndexes }; }; From e5136c9ef5cdef2c7e1edf598f13006cb9eaede0 Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Tue, 26 Mar 2024 19:27:11 +0100 Subject: [PATCH 11/18] Feat: Recursively get all secrets --- backend/src/services/secret/secret-fns.ts | 33 ++++++++++++++++------- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/backend/src/services/secret/secret-fns.ts b/backend/src/services/secret/secret-fns.ts index 3c7c9680c..e7e7374d0 100644 --- a/backend/src/services/secret/secret-fns.ts +++ b/backend/src/services/secret/secret-fns.ts @@ -95,9 +95,13 @@ export const recursivelyGetSecretPaths = ({ return map; }; - const generatePaths = (map: FolderMap, parentId: string = "null", basePath: string = ""): string[] => { + const generatePaths = ( + map: FolderMap, + parentId: string = "null", + basePath: string = "" + ): { path: string; folderId: string }[] => { const children = map[parentId || "null"] || []; - let paths: string[] = []; + let paths: { path: string; folderId: string }[] = []; children.forEach((child) => { // Determine if this is the root folder of the environment. If no parentId is present and the name is root, it's the root folder @@ -107,11 +111,19 @@ export const recursivelyGetSecretPaths = ({ // eslint-disable-next-line no-nested-ternary const currPath = basePath === "" ? (isRootFolder ? "/" : `/${child.name}`) : `${basePath}/${child.name}`; - paths.push(currPath); // Add the current path + paths.push({ + path: currPath, + folderId: child.id + }); // Add the current path // Recursively generate paths for children, passing down the formatted pathh const childPaths = generatePaths(map, child.id, currPath); - paths = paths.concat(childPaths); + paths = paths.concat( + childPaths.map((p) => ({ + path: p.path, + folderId: p.folderId + })) + ); }); return paths; @@ -135,8 +147,11 @@ export const recursivelyGetSecretPaths = ({ // Build the folder hierarchy map const folderMap = buildHierarchy(folders); - // Generate the paths paths and normalize the root path toe / - const paths = generatePaths(folderMap).map((p) => (p === "/" ? p : p.substring(1))); + // Generate the paths paths and normalize the root path to / + const paths = generatePaths(folderMap).map((p) => ({ + path: p.path === "/" ? p.path : p.path.substring(1), + folderId: p.folderId + })); const { permission } = await permissionService.getProjectPermission( auth.actor, @@ -148,14 +163,14 @@ export const recursivelyGetSecretPaths = ({ // Filter out paths that the user does not have permission to access, and paths that are not in the current path const allowedPaths = paths.filter( - (p) => + (folder) => permission.can( ProjectPermissionActions.Read, subject(ProjectPermissionSub.Secrets, { environment, - secretPath: p + secretPath: folder.path }) - ) && p.startsWith(currentPath === "/" ? "" : currentPath) + ) && folder.path.startsWith(currentPath === "/" ? "" : currentPath) ); return allowedPaths; From 2e8680c5d403ed68f33648ecde7b6d918ad42b14 Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Tue, 26 Mar 2024 19:30:56 +0100 Subject: [PATCH 12/18] Update secret-service.ts --- backend/src/services/secret/secret-service.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/services/secret/secret-service.ts b/backend/src/services/secret/secret-service.ts index 0b2d6262e..f030ecee1 100644 --- a/backend/src/services/secret/secret-service.ts +++ b/backend/src/services/secret/secret-service.ts @@ -471,7 +471,7 @@ export const secretServiceFactory = ({ } }); - paths = deepPaths.length === 0 ? paths : deepPaths; + if (!deepPaths) return { secrets: [], imports: [] }; } else { const folder = await folderDAL.findBySecretPath(projectId, environment, path); if (!folder) return { secrets: [], imports: [] }; From 4f05e4ce932e6b188b3dbc3305ff5b9dbf594b4d Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Tue, 26 Mar 2024 19:41:48 +0100 Subject: [PATCH 13/18] Fix test case --- backend/src/services/secret/secret-service.ts | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/backend/src/services/secret/secret-service.ts b/backend/src/services/secret/secret-service.ts index f030ecee1..7c002de64 100644 --- a/backend/src/services/secret/secret-service.ts +++ b/backend/src/services/secret/secret-service.ts @@ -473,14 +473,15 @@ export const secretServiceFactory = ({ if (!deepPaths) return { secrets: [], imports: [] }; } else { - const folder = await folderDAL.findBySecretPath(projectId, environment, path); - if (!folder) return { secrets: [], imports: [] }; - paths = [{ folderId: folder.id, path }]; - ForbiddenError.from(permission).throwUnlessCan( ProjectPermissionActions.Read, - subject(ProjectPermissionSub.Secrets, { environment, secretPath: folder.path }) + subject(ProjectPermissionSub.Secrets, { environment, secretPath: path }) ); + + const folder = await folderDAL.findBySecretPath(projectId, environment, path); + if (!folder) return { secrets: [], imports: [] }; + + paths = [{ folderId: folder.id, path }]; } const groupedPaths = groupBy(paths, (p) => p.folderId); From 4c496d5e3d341f8cb5e8c3db25db3d20001c305c Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Sat, 30 Mar 2024 08:40:43 +0100 Subject: [PATCH 14/18] Update secret-service.ts --- backend/src/services/secret/secret-service.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/backend/src/services/secret/secret-service.ts b/backend/src/services/secret/secret-service.ts index 7c002de64..fb880e116 100644 --- a/backend/src/services/secret/secret-service.ts +++ b/backend/src/services/secret/secret-service.ts @@ -472,6 +472,8 @@ export const secretServiceFactory = ({ }); if (!deepPaths) return { secrets: [], imports: [] }; + + paths = deepPaths.map(({ folderId, path: p }) => ({ folderId, path: p })); } else { ForbiddenError.from(permission).throwUnlessCan( ProjectPermissionActions.Read, From b9482966cffc9d0281f41c1b10911f762160a068 Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Mon, 1 Apr 2024 09:52:49 -0700 Subject: [PATCH 15/18] Fix: Replaced merge with extend as proposed by Akhil --- backend/src/server/routes/v3/secret-router.ts | 34 ++++++++----------- 1 file changed, 15 insertions(+), 19 deletions(-) diff --git a/backend/src/server/routes/v3/secret-router.ts b/backend/src/server/routes/v3/secret-router.ts index 07dff0703..8d69f009c 100644 --- a/backend/src/server/routes/v3/secret-router.ts +++ b/backend/src/server/routes/v3/secret-router.ts @@ -171,11 +171,9 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => { response: { 200: z.object({ secrets: secretRawSchema - .merge( - z.object({ - secretPath: z.string().optional() - }) - ) + .extend({ + secretPath: z.string().optional() + }) .array(), imports: z .object({ @@ -620,20 +618,18 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => { response: { 200: z.object({ secrets: SecretsSchema.omit({ secretBlindIndex: true }) - .merge( - z.object({ - _id: z.string(), - workspace: z.string(), - environment: z.string(), - secretPath: z.string().optional(), - tags: SecretTagsSchema.pick({ - id: true, - slug: true, - name: true, - color: true - }).array() - }) - ) + .extend({ + _id: z.string(), + workspace: z.string(), + environment: z.string(), + secretPath: z.string().optional(), + tags: SecretTagsSchema.pick({ + id: true, + slug: true, + name: true, + color: true + }).array() + }) .array(), imports: z .object({ From da2dcb347af0cd004a58a5d01a66875e80f7d669 Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Mon, 1 Apr 2024 09:58:13 -0700 Subject: [PATCH 16/18] Fix: Restructured recursive path functions as suggested by Akhil --- backend/src/services/secret/secret-fns.ts | 101 +++++++++++----------- 1 file changed, 51 insertions(+), 50 deletions(-) diff --git a/backend/src/services/secret/secret-fns.ts b/backend/src/services/secret/secret-fns.ts index e7e7374d0..24c2bd811 100644 --- a/backend/src/services/secret/secret-fns.ts +++ b/backend/src/services/secret/secret-fns.ts @@ -74,62 +74,61 @@ type TGetPathsDTO = { interface FolderMap { [parentId: string]: TSecretFolders[]; } +const buildHierarchy = (folders: TSecretFolders[]): FolderMap => { + const map: FolderMap = {}; + map.null = []; // Initialize mapping for root directory + + folders.forEach((folder) => { + const parentId = folder.parentId || "null"; + if (!map[parentId]) { + map[parentId] = []; + } + map[parentId].push(folder); + }); + + return map; +}; + +const generatePaths = ( + map: FolderMap, + parentId: string = "null", + basePath: string = "" +): { path: string; folderId: string }[] => { + const children = map[parentId || "null"] || []; + let paths: { path: string; folderId: string }[] = []; + + children.forEach((child) => { + // Determine if this is the root folder of the environment. If no parentId is present and the name is root, it's the root folder + const isRootFolder = child.name === "root" && !child.parentId; + + // Form the current path based on the base path and the current child + // eslint-disable-next-line no-nested-ternary + const currPath = basePath === "" ? (isRootFolder ? "/" : `/${child.name}`) : `${basePath}/${child.name}`; + + paths.push({ + path: currPath, + folderId: child.id + }); // Add the current path + + // Recursively generate paths for children, passing down the formatted pathh + const childPaths = generatePaths(map, child.id, currPath); + paths = paths.concat( + childPaths.map((p) => ({ + path: p.path, + folderId: p.folderId + })) + ); + }); + + return paths; +}; export const recursivelyGetSecretPaths = ({ folderDAL, projectEnvDAL, permissionService }: TRecursivelyFetchSecretsFromFoldersArg) => { - const buildHierarchy = (folders: TSecretFolders[]): FolderMap => { - const map: FolderMap = {}; - map.null = []; // Initialize mapping for root directory - - folders.forEach((folder) => { - const parentId = folder.parentId || "null"; - if (!map[parentId]) { - map[parentId] = []; - } - map[parentId].push(folder); - }); - - return map; - }; - - const generatePaths = ( - map: FolderMap, - parentId: string = "null", - basePath: string = "" - ): { path: string; folderId: string }[] => { - const children = map[parentId || "null"] || []; - let paths: { path: string; folderId: string }[] = []; - - children.forEach((child) => { - // Determine if this is the root folder of the environment. If no parentId is present and the name is root, it's the root folder - const isRootFolder = child.name === "root" && !child.parentId; - - // Form the current path based on the base path and the current child - // eslint-disable-next-line no-nested-ternary - const currPath = basePath === "" ? (isRootFolder ? "/" : `/${child.name}`) : `${basePath}/${child.name}`; - - paths.push({ - path: currPath, - folderId: child.id - }); // Add the current path - - // Recursively generate paths for children, passing down the formatted pathh - const childPaths = generatePaths(map, child.id, currPath); - paths = paths.concat( - childPaths.map((p) => ({ - path: p.path, - folderId: p.folderId - })) - ); - }); - - return paths; - }; - - return async ({ projectId, environment, currentPath, auth }: TGetPathsDTO) => { + const getPaths = async ({ projectId, environment, currentPath, auth }: TGetPathsDTO) => { const env = await projectEnvDAL.findOne({ projectId, slug: environment @@ -175,6 +174,8 @@ export const recursivelyGetSecretPaths = ({ return allowedPaths; }; + + return getPaths; }; type TInterpolateSecretArg = { From 42fb7329557fba27b1fd35094166d3af2b1e3d26 Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Mon, 1 Apr 2024 10:10:34 -0700 Subject: [PATCH 17/18] Fix: Renamed deep parameter to recursive --- cli/packages/api/api.go | 4 ++-- cli/packages/api/model.go | 4 ++-- cli/packages/cmd/run.go | 6 +++--- cli/packages/cmd/secrets.go | 14 ++++++-------- cli/packages/models/cli.go | 2 +- cli/packages/util/secrets.go | 18 +++++++++--------- 6 files changed, 23 insertions(+), 25 deletions(-) diff --git a/cli/packages/api/api.go b/cli/packages/api/api.go index 38af4630e..d45a42db4 100644 --- a/cli/packages/api/api.go +++ b/cli/packages/api/api.go @@ -277,8 +277,8 @@ func CallGetSecretsV3(httpClient *resty.Client, request GetEncryptedSecretsV3Req SetQueryParam("environment", request.Environment). SetQueryParam("workspaceId", request.WorkspaceId) - if request.DeepSearch { - httpRequest.SetQueryParam("deep", "true") + if request.Recursive { + httpRequest.SetQueryParam("recursive", "true") } if request.IncludeImport { diff --git a/cli/packages/api/model.go b/cli/packages/api/model.go index 1b91c8b64..ed9a1d629 100644 --- a/cli/packages/api/model.go +++ b/cli/packages/api/model.go @@ -291,7 +291,7 @@ type GetEncryptedSecretsV3Request struct { WorkspaceId string `json:"workspaceId"` SecretPath string `json:"secretPath"` IncludeImport bool `json:"include_imports"` - DeepSearch bool `json:"deep"` + Recursive bool `json:"recursive"` } type GetFoldersV1Request struct { @@ -511,7 +511,7 @@ type CreateDynamicSecretLeaseV1Request struct { type CreateDynamicSecretLeaseV1Response struct { Lease struct { - Id string `json:"id"` + Id string `json:"id"` ExpireAt time.Time `json:"expireAt"` } `json:"lease"` DynamicSecret struct { diff --git a/cli/packages/cmd/run.go b/cli/packages/cmd/run.go index f6e675096..cb44e3d7e 100644 --- a/cli/packages/cmd/run.go +++ b/cli/packages/cmd/run.go @@ -98,12 +98,12 @@ var runCmd = &cobra.Command{ util.HandleError(err, "Unable to parse flag") } - deepSearch, err := cmd.Flags().GetBool("deep") + recursive, err := cmd.Flags().GetBool("recursive") if err != nil { util.HandleError(err, "Unable to parse flag") } - secrets, err := util.GetAllEnvironmentVariables(models.GetAllSecretsParameters{Environment: environmentName, InfisicalToken: infisicalToken, TagSlugs: tagSlugs, SecretsPath: secretsPath, IncludeImport: includeImports, DeepSearch: deepSearch}, projectConfigDir) + secrets, err := util.GetAllEnvironmentVariables(models.GetAllSecretsParameters{Environment: environmentName, InfisicalToken: infisicalToken, TagSlugs: tagSlugs, SecretsPath: secretsPath, IncludeImport: includeImports, Recursive: recursive}, projectConfigDir) if err != nil { util.HandleError(err, "Could not fetch secrets", "If you are using a service token to fetch secrets, please ensure it is valid") @@ -207,7 +207,7 @@ func init() { runCmd.Flags().StringP("env", "e", "dev", "Set the environment (dev, prod, etc.) from which your secrets should be pulled from") runCmd.Flags().Bool("expand", true, "Parse shell parameter expansions in your secrets") runCmd.Flags().Bool("include-imports", true, "Import linked secrets ") - runCmd.Flags().Bool("deep", false, "Fetch secrets from all sub-folders") + runCmd.Flags().Bool("recursive", false, "Fetch secrets from all sub-folders") runCmd.Flags().Bool("secret-overriding", true, "Prioritizes personal secrets, if any, with the same name over shared secrets") runCmd.Flags().StringP("command", "c", "", "chained commands to execute (e.g. \"npm install && npm run dev; echo ...\")") runCmd.Flags().StringP("tags", "t", "", "filter secrets by tag slugs ") diff --git a/cli/packages/cmd/secrets.go b/cli/packages/cmd/secrets.go index 1db565b76..bbee424c4 100644 --- a/cli/packages/cmd/secrets.go +++ b/cli/packages/cmd/secrets.go @@ -63,13 +63,11 @@ var secretsCmd = &cobra.Command{ util.HandleError(err) } - deepSearch, err := cmd.Flags().GetBool("deep") + recursive, err := cmd.Flags().GetBool("recursive") if err != nil { util.HandleError(err) } - fmt.Printf("Is deep search: %v\n", deepSearch) - tagSlugs, err := cmd.Flags().GetString("tags") if err != nil { util.HandleError(err, "Unable to parse flag") @@ -80,7 +78,7 @@ var secretsCmd = &cobra.Command{ util.HandleError(err, "Unable to parse flag") } - secrets, err := util.GetAllEnvironmentVariables(models.GetAllSecretsParameters{Environment: environmentName, InfisicalToken: infisicalToken, TagSlugs: tagSlugs, SecretsPath: secretsPath, IncludeImport: includeImports, DeepSearch: deepSearch}, "") + secrets, err := util.GetAllEnvironmentVariables(models.GetAllSecretsParameters{Environment: environmentName, InfisicalToken: infisicalToken, TagSlugs: tagSlugs, SecretsPath: secretsPath, IncludeImport: includeImports, Recursive: recursive}, "") if err != nil { util.HandleError(err) } @@ -420,9 +418,9 @@ func getSecretsByNames(cmd *cobra.Command, args []string) { util.HandleError(err, "Unable to parse path flag") } - deepSearch, err := cmd.Flags().GetBool("deep") + recursive, err := cmd.Flags().GetBool("recursive") if err != nil { - util.HandleError(err, "Unable to parse deep flag") + util.HandleError(err, "Unable to parse recursive flag") } showOnlyValue, err := cmd.Flags().GetBool("raw-value") @@ -430,7 +428,7 @@ func getSecretsByNames(cmd *cobra.Command, args []string) { util.HandleError(err, "Unable to parse path flag") } - secrets, err := util.GetAllEnvironmentVariables(models.GetAllSecretsParameters{Environment: environmentName, InfisicalToken: infisicalToken, TagSlugs: tagSlugs, SecretsPath: secretsPath, IncludeImport: true, DeepSearch: deepSearch}, "") + secrets, err := util.GetAllEnvironmentVariables(models.GetAllSecretsParameters{Environment: environmentName, InfisicalToken: infisicalToken, TagSlugs: tagSlugs, SecretsPath: secretsPath, IncludeImport: true, Recursive: recursive}, "") if err != nil { util.HandleError(err, "To fetch all secrets") } @@ -739,7 +737,7 @@ func init() { secretsCmd.PersistentFlags().String("env", "dev", "Used to select the environment name on which actions should be taken on") secretsCmd.Flags().Bool("expand", true, "Parse shell parameter expansions in your secrets") secretsCmd.Flags().Bool("include-imports", true, "Imported linked secrets ") - secretsCmd.Flags().Bool("deep", false, "Fetch secrets from all sub-folders") + secretsCmd.Flags().Bool("recursive", false, "Fetch secrets from all sub-folders") secretsCmd.PersistentFlags().StringP("tags", "t", "", "filter secrets by tag slugs") secretsCmd.Flags().String("path", "/", "get secrets within a folder path") rootCmd.AddCommand(secretsCmd) diff --git a/cli/packages/models/cli.go b/cli/packages/models/cli.go index 69319e2b2..e197218e0 100644 --- a/cli/packages/models/cli.go +++ b/cli/packages/models/cli.go @@ -93,7 +93,7 @@ type GetAllSecretsParameters struct { WorkspaceId string SecretsPath string IncludeImport bool - DeepSearch bool + Recursive bool } type GetAllFoldersParameters struct { diff --git a/cli/packages/util/secrets.go b/cli/packages/util/secrets.go index 89d094f43..23e95db64 100644 --- a/cli/packages/util/secrets.go +++ b/cli/packages/util/secrets.go @@ -17,7 +17,7 @@ import ( "github.com/rs/zerolog/log" ) -func GetPlainTextSecretsViaServiceToken(fullServiceToken string, environment string, secretPath string, includeImports bool, deepSearch bool) ([]models.SingleEnvironmentVariable, api.GetServiceTokenDetailsResponse, error) { +func GetPlainTextSecretsViaServiceToken(fullServiceToken string, environment string, secretPath string, includeImports bool, recursive bool) ([]models.SingleEnvironmentVariable, api.GetServiceTokenDetailsResponse, error) { serviceTokenParts := strings.SplitN(fullServiceToken, ".", 4) if len(serviceTokenParts) < 4 { return nil, api.GetServiceTokenDetailsResponse{}, fmt.Errorf("invalid service token entered. Please double check your service token and try again") @@ -49,7 +49,7 @@ func GetPlainTextSecretsViaServiceToken(fullServiceToken string, environment str Environment: environment, SecretPath: secretPath, IncludeImport: includeImports, - DeepSearch: deepSearch, + Recursive: recursive, }) if err != nil { @@ -81,7 +81,7 @@ func GetPlainTextSecretsViaServiceToken(fullServiceToken string, environment str return plainTextSecrets, serviceTokenDetails, nil } -func GetPlainTextSecretsViaJTW(JTWToken string, receiversPrivateKey string, workspaceId string, environmentName string, tagSlugs string, secretsPath string, includeImports bool, deepSearch bool) ([]models.SingleEnvironmentVariable, error) { +func GetPlainTextSecretsViaJTW(JTWToken string, receiversPrivateKey string, workspaceId string, environmentName string, tagSlugs string, secretsPath string, includeImports bool, recursive bool) ([]models.SingleEnvironmentVariable, error) { httpClient := resty.New() httpClient.SetAuthToken(JTWToken). SetHeader("Accept", "application/json") @@ -126,7 +126,7 @@ func GetPlainTextSecretsViaJTW(JTWToken string, receiversPrivateKey string, work WorkspaceId: workspaceId, Environment: environmentName, IncludeImport: includeImports, - DeepSearch: deepSearch, + Recursive: recursive, // TagSlugs: tagSlugs, } @@ -154,7 +154,7 @@ func GetPlainTextSecretsViaJTW(JTWToken string, receiversPrivateKey string, work return plainTextSecrets, nil } -func GetPlainTextSecretsViaMachineIdentity(accessToken string, workspaceId string, environmentName string, secretsPath string, includeImports bool, deepSearch bool) (models.PlaintextSecretResult, error) { +func GetPlainTextSecretsViaMachineIdentity(accessToken string, workspaceId string, environmentName string, secretsPath string, includeImports bool, recursive bool) (models.PlaintextSecretResult, error) { httpClient := resty.New() httpClient.SetAuthToken(accessToken). SetHeader("Accept", "application/json") @@ -163,7 +163,7 @@ func GetPlainTextSecretsViaMachineIdentity(accessToken string, workspaceId strin WorkspaceId: workspaceId, Environment: environmentName, IncludeImport: includeImports, - DeepSearch: deepSearch, + Recursive: recursive, // TagSlugs: tagSlugs, } @@ -332,7 +332,7 @@ func GetAllEnvironmentVariables(params models.GetAllSecretsParameters, projectCo } secretsToReturn, errorToReturn = GetPlainTextSecretsViaJTW(loggedInUserDetails.UserCredentials.JTWToken, loggedInUserDetails.UserCredentials.PrivateKey, infisicalDotJson.WorkspaceId, - params.Environment, params.TagSlugs, params.SecretsPath, params.IncludeImport, params.DeepSearch) + params.Environment, params.TagSlugs, params.SecretsPath, params.IncludeImport, params.Recursive) log.Debug().Msgf("GetAllEnvironmentVariables: Trying to fetch secrets JTW token [err=%s]", errorToReturn) backupSecretsEncryptionKey := []byte(loggedInUserDetails.UserCredentials.PrivateKey)[0:32] @@ -353,10 +353,10 @@ func GetAllEnvironmentVariables(params models.GetAllSecretsParameters, projectCo } else { if params.InfisicalToken != "" { log.Debug().Msg("Trying to fetch secrets using service token") - secretsToReturn, _, errorToReturn = GetPlainTextSecretsViaServiceToken(params.InfisicalToken, params.Environment, params.SecretsPath, params.IncludeImport, params.DeepSearch) + secretsToReturn, _, errorToReturn = GetPlainTextSecretsViaServiceToken(params.InfisicalToken, params.Environment, params.SecretsPath, params.IncludeImport, params.Recursive) } else if params.UniversalAuthAccessToken != "" { log.Debug().Msg("Trying to fetch secrets using universal auth") - res, err := GetPlainTextSecretsViaMachineIdentity(params.UniversalAuthAccessToken, params.WorkspaceId, params.Environment, params.SecretsPath, params.IncludeImport, params.DeepSearch) + res, err := GetPlainTextSecretsViaMachineIdentity(params.UniversalAuthAccessToken, params.WorkspaceId, params.Environment, params.SecretsPath, params.IncludeImport, params.Recursive) errorToReturn = err secretsToReturn = res.Secrets From 36bf1b2abc46b4425b839feeb57730f6b9d0c8ba Mon Sep 17 00:00:00 2001 From: Daniel Hougaard <62331820+DanielHougaard@users.noreply.github.com> Date: Mon, 1 Apr 2024 10:10:49 -0700 Subject: [PATCH 18/18] Fix: Renamed deep parameter to recursive --- backend/src/lib/api-docs/constants.ts | 2 +- backend/src/server/routes/v3/secret-router.ts | 10 +++++----- backend/src/services/secret/secret-service.ts | 8 ++++---- backend/src/services/secret/secret-types.ts | 4 ++-- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/backend/src/lib/api-docs/constants.ts b/backend/src/lib/api-docs/constants.ts index bb1b73a77..4a45dfb41 100644 --- a/backend/src/lib/api-docs/constants.ts +++ b/backend/src/lib/api-docs/constants.ts @@ -215,7 +215,7 @@ export const SECRETS = { export const RAW_SECRETS = { LIST: { - deep: "Whether or not to fetch all secrets from the specified base path, and all of its subdirectories.", + recursive: "Whether or not to fetch all secrets from the specified base path, and all of its subdirectories.", workspaceId: "The ID of the project to list secrets from.", workspaceSlug: "The slug of the project to list secrets from. This parameter is only usable by machine identities.", environment: "The slug of the environment to list secrets from.", diff --git a/backend/src/server/routes/v3/secret-router.ts b/backend/src/server/routes/v3/secret-router.ts index 8d69f009c..5344be43d 100644 --- a/backend/src/server/routes/v3/secret-router.ts +++ b/backend/src/server/routes/v3/secret-router.ts @@ -157,11 +157,11 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => { workspaceSlug: z.string().trim().optional().describe(RAW_SECRETS.LIST.workspaceSlug), environment: z.string().trim().optional().describe(RAW_SECRETS.LIST.environment), secretPath: z.string().trim().default("/").transform(removeTrailingSlash).describe(RAW_SECRETS.LIST.secretPath), - deep: z + recursive: z .enum(["true", "false"]) .default("false") .transform((value) => value === "true") - .describe(RAW_SECRETS.LIST.deep), + .describe(RAW_SECRETS.LIST.recursive), include_imports: z .enum(["true", "false"]) .default("false") @@ -228,7 +228,7 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => { projectId: workspaceId, path: secretPath, includeImports: req.query.include_imports, - deep: req.query.deep + recursive: req.query.recursive }); await server.services.auditLog.createAuditLog({ @@ -606,7 +606,7 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => { workspaceId: z.string().trim(), environment: z.string().trim(), secretPath: z.string().trim().default("/").transform(removeTrailingSlash), - deep: z + recursive: z .enum(["true", "false"]) .default("false") .transform((value) => value === "true"), @@ -662,7 +662,7 @@ export const registerSecretRouter = async (server: FastifyZodProvider) => { projectId: req.query.workspaceId, path: req.query.secretPath, includeImports: req.query.include_imports, - deep: req.query.deep + recursive: req.query.recursive }); await server.services.auditLog.createAuditLog({ diff --git a/backend/src/services/secret/secret-service.ts b/backend/src/services/secret/secret-service.ts index fb880e116..5b8c3c934 100644 --- a/backend/src/services/secret/secret-service.ts +++ b/backend/src/services/secret/secret-service.ts @@ -440,7 +440,7 @@ export const secretServiceFactory = ({ actorOrgId, actorAuthMethod, includeImports, - deep + recursive }: TGetSecretsDTO) => { const { permission } = await permissionService.getProjectPermission( actor, @@ -452,7 +452,7 @@ export const secretServiceFactory = ({ let paths: { folderId: string; path: string }[] = []; - if (deep) { + if (recursive) { const getPaths = recursivelyGetSecretPaths({ permissionService, folderDAL, @@ -852,7 +852,7 @@ export const secretServiceFactory = ({ actorAuthMethod, environment, includeImports, - deep + recursive }: TGetSecretsRawDTO) => { const botKey = await projectBotService.getBotKey(projectId); if (!botKey) throw new BadRequestError({ message: "Project bot not found", name: "bot_not_found_error" }); @@ -866,7 +866,7 @@ export const secretServiceFactory = ({ actorAuthMethod, path, includeImports, - deep + recursive }); return { diff --git a/backend/src/services/secret/secret-types.ts b/backend/src/services/secret/secret-types.ts index 33e46b67f..22347de4e 100644 --- a/backend/src/services/secret/secret-types.ts +++ b/backend/src/services/secret/secret-types.ts @@ -74,7 +74,7 @@ export type TGetSecretsDTO = { path: string; environment: string; includeImports?: boolean; - deep?: boolean; + recursive?: boolean; } & TProjectPermission; export type TGetASecretDTO = { @@ -141,7 +141,7 @@ export type TGetSecretsRawDTO = { path: string; environment: string; includeImports?: boolean; - deep?: boolean; + recursive?: boolean; } & TProjectPermission; export type TGetASecretRawDTO = {