From 7138b392f2e8993d8d1df7d4d298aff9b47223d6 Mon Sep 17 00:00:00 2001 From: Scott Wilson Date: Tue, 12 Nov 2024 10:21:07 -0800 Subject: [PATCH 1/4] Feature: add ability to paste .env, .yml or .json secrets for upload and also fix upload when keys conflict but are not on current page --- .../src/server/routes/v1/dashboard-router.ts | 87 ++++++++++ .../secret-v2-bridge/secret-v2-bridge-dal.ts | 4 + .../secret-v2-bridge-types.ts | 2 + backend/src/services/secret/secret-types.ts | 1 + .../src/components/utilities/parseDotEnv.ts | 2 +- .../src/components/utilities/parseJson.ts | 11 ++ frontend/src/hooks/api/dashboard/queries.tsx | 19 +++ frontend/src/hooks/api/dashboard/types.ts | 11 ++ .../views/SecretMainPage/SecretMainPage.tsx | 1 - .../SecretDropzone/CopySecretsFromBoard.tsx | 2 + .../SecretDropzone/PasteSecretEnvModal.tsx | 132 ++++++++++++++++ .../SecretDropzone/SecretDropzone.tsx | 149 +++++++++++------- 12 files changed, 364 insertions(+), 57 deletions(-) create mode 100644 frontend/src/components/utilities/parseJson.ts create mode 100644 frontend/src/views/SecretMainPage/components/SecretDropzone/PasteSecretEnvModal.tsx diff --git a/backend/src/server/routes/v1/dashboard-router.ts b/backend/src/server/routes/v1/dashboard-router.ts index 8213cf666..d3975bbc2 100644 --- a/backend/src/server/routes/v1/dashboard-router.ts +++ b/backend/src/server/routes/v1/dashboard-router.ts @@ -840,4 +840,91 @@ export const registerDashboardRouter = async (server: FastifyZodProvider) => { }; } }); + + server.route({ + method: "GET", + url: "/secrets-by-keys", + config: { + rateLimit: secretsLimit + }, + schema: { + security: [ + { + bearerAuth: [] + } + ], + querystring: z.object({ + projectId: z.string().trim(), + environment: z.string().trim(), + secretPath: z.string().trim().default("/").transform(removeTrailingSlash), + keys: z.string().trim().transform(decodeURIComponent) + }), + response: { + 200: z.object({ + secrets: secretRawSchema + .extend({ + secretPath: z.string().optional(), + tags: SecretTagsSchema.pick({ + id: true, + slug: true, + color: true + }) + .extend({ name: z.string() }) + .array() + .optional() + }) + .array() + .optional() + }) + } + }, + onRequest: verifyAuth([AuthMode.JWT]), + handler: async (req) => { + const { secretPath, projectId, environment } = req.query; + + const keys = req.query.keys?.split(",").filter((key) => Boolean(key.trim())) ?? []; + if (!keys.length) throw new BadRequestError({ message: "One or more keys required" }); + + const { secrets } = await server.services.secret.getSecretsRaw({ + actorId: req.permission.id, + actor: req.permission.type, + actorOrgId: req.permission.orgId, + environment, + actorAuthMethod: req.permission.authMethod, + projectId, + path: secretPath, + keys + }); + + await server.services.auditLog.createAuditLog({ + projectId, + ...req.auditLogInfo, + event: { + type: EventType.GET_SECRETS, + metadata: { + environment, + secretPath, + numberOfSecrets: secrets.length + } + } + }); + + if (getUserAgentType(req.headers["user-agent"]) !== UserAgentType.K8_OPERATOR) { + await server.services.telemetry.sendPostHogEvents({ + event: PostHogEventTypes.SecretPulled, + distinctId: getTelemetryDistinctId(req), + properties: { + numberOfSecrets: secrets.length, + workspaceId: projectId, + environment, + secretPath, + channel: getUserAgentType(req.headers["user-agent"]), + ...req.auditLogInfo + } + }); + } + + return { secrets }; + } + }); }; diff --git a/backend/src/services/secret-v2-bridge/secret-v2-bridge-dal.ts b/backend/src/services/secret-v2-bridge/secret-v2-bridge-dal.ts index 9ca3e87d3..3bdd5783f 100644 --- a/backend/src/services/secret-v2-bridge/secret-v2-bridge-dal.ts +++ b/backend/src/services/secret-v2-bridge/secret-v2-bridge-dal.ts @@ -361,6 +361,10 @@ export const secretV2BridgeDALFactory = (db: TDbClient) => { void bd.whereILike(`${TableName.SecretV2}.key`, `%${filters?.search}%`); } } + + if (filters?.keys) { + void bd.whereIn(`${TableName.SecretV2}.key`, filters.keys); + } }) .where((bd) => { void bd.whereNull(`${TableName.SecretV2}.userId`).orWhere({ userId: userId || null }); diff --git a/backend/src/services/secret-v2-bridge/secret-v2-bridge-types.ts b/backend/src/services/secret-v2-bridge/secret-v2-bridge-types.ts index e621f8edb..7216989ff 100644 --- a/backend/src/services/secret-v2-bridge/secret-v2-bridge-types.ts +++ b/backend/src/services/secret-v2-bridge/secret-v2-bridge-types.ts @@ -33,6 +33,7 @@ export type TGetSecretsDTO = { offset?: number; limit?: number; search?: string; + keys?: string[]; } & TProjectPermission; export type TGetASecretDTO = { @@ -294,6 +295,7 @@ export type TFindSecretsByFolderIdsFilter = { search?: string; tagSlugs?: string[]; includeTagsInSearch?: boolean; + keys?: string[]; }; export type TGetSecretsRawByFolderMappingsDTO = { diff --git a/backend/src/services/secret/secret-types.ts b/backend/src/services/secret/secret-types.ts index 7c09c9349..ca5c5a74b 100644 --- a/backend/src/services/secret/secret-types.ts +++ b/backend/src/services/secret/secret-types.ts @@ -185,6 +185,7 @@ export type TGetSecretsRawDTO = { offset?: number; limit?: number; search?: string; + keys?: string[]; } & TProjectPermission; export type TGetASecretRawDTO = { diff --git a/frontend/src/components/utilities/parseDotEnv.ts b/frontend/src/components/utilities/parseDotEnv.ts index bec3a506b..ac9c616b5 100644 --- a/frontend/src/components/utilities/parseDotEnv.ts +++ b/frontend/src/components/utilities/parseDotEnv.ts @@ -6,7 +6,7 @@ const LINE = * @param {ArrayBuffer} src - source buffer * @returns {String} text - text of buffer */ -export function parseDotEnv(src: ArrayBuffer) { +export function parseDotEnv(src: ArrayBuffer | string) { const object: { [key: string]: { value: string; comments: string[] }; } = {}; diff --git a/frontend/src/components/utilities/parseJson.ts b/frontend/src/components/utilities/parseJson.ts new file mode 100644 index 000000000..e6eb36f80 --- /dev/null +++ b/frontend/src/components/utilities/parseJson.ts @@ -0,0 +1,11 @@ +export const parseJson = (src: ArrayBuffer | string) => { + const file = src.toString(); + const formatedData: Record = JSON.parse(file); + const env: Record = {}; + Object.keys(formatedData).forEach((key) => { + if (typeof formatedData[key] === "string") { + env[key] = { value: formatedData[key], comments: [] }; + } + }); + return env; +}; diff --git a/frontend/src/hooks/api/dashboard/queries.tsx b/frontend/src/hooks/api/dashboard/queries.tsx index 7654ec07f..adff8bb0e 100644 --- a/frontend/src/hooks/api/dashboard/queries.tsx +++ b/frontend/src/hooks/api/dashboard/queries.tsx @@ -5,6 +5,7 @@ import axios from "axios"; import { createNotification } from "@app/components/notifications"; import { apiRequest } from "@app/config/request"; import { + DashboardProjectSecretsByKeys, DashboardProjectSecretsDetails, DashboardProjectSecretsDetailsResponse, DashboardProjectSecretsOverview, @@ -12,6 +13,7 @@ import { DashboardSecretsOrderBy, TDashboardProjectSecretsQuickSearch, TDashboardProjectSecretsQuickSearchResponse, + TGetDashboardProjectSecretsByKeys, TGetDashboardProjectSecretsDetailsDTO, TGetDashboardProjectSecretsOverviewDTO, TGetDashboardProjectSecretsQuickSearchDTO @@ -101,6 +103,23 @@ export const fetchProjectSecretsDetails = async ({ return data; }; +export const fetchDashboardProjectSecretsByKeys = async ({ + keys, + ...params +}: TGetDashboardProjectSecretsByKeys) => { + const { data } = await apiRequest.get( + "/api/v1/dashboard/secrets-by-keys", + { + params: { + ...params, + keys: encodeURIComponent(keys.join(",")) + } + } + ); + + return data; +}; + export const useGetProjectSecretsOverview = ( { projectId, diff --git a/frontend/src/hooks/api/dashboard/types.ts b/frontend/src/hooks/api/dashboard/types.ts index da121a349..444614b82 100644 --- a/frontend/src/hooks/api/dashboard/types.ts +++ b/frontend/src/hooks/api/dashboard/types.ts @@ -29,6 +29,10 @@ export type DashboardProjectSecretsDetailsResponse = { totalCount: number; }; +export type DashboardProjectSecretsByKeys = { + secrets: SecretV3Raw[]; +}; + export type DashboardProjectSecretsOverview = Omit< DashboardProjectSecretsOverviewResponse, "secrets" @@ -89,3 +93,10 @@ export type TGetDashboardProjectSecretsQuickSearchDTO = { search: string; environments: string[]; }; + +export type TGetDashboardProjectSecretsByKeys = { + projectId: string; + secretPath: string; + environment: string; + keys: string[]; +}; diff --git a/frontend/src/views/SecretMainPage/SecretMainPage.tsx b/frontend/src/views/SecretMainPage/SecretMainPage.tsx index 31635884c..85d25a56f 100644 --- a/frontend/src/views/SecretMainPage/SecretMainPage.tsx +++ b/frontend/src/views/SecretMainPage/SecretMainPage.tsx @@ -552,7 +552,6 @@ const SecretMainPageContent = () => { {(isAllowed) => (