Feature: add ability to paste .env, .yml or .json secrets for upload and also fix upload when keys conflict but are not on current page

backend/src/server/routes/v1/dashboard-router.ts

@@ -840,4 +840,91 @@ export const registerDashboardRouter = async (server: FastifyZodProvider) => {
       };
     }
   });
+
+  server.route({
+    method: "GET",
+    url: "/secrets-by-keys",
+    config: {
+      rateLimit: secretsLimit
+    },
+    schema: {
+      security: [
+        {
+          bearerAuth: []
+        }
+      ],
+      querystring: z.object({
+        projectId: z.string().trim(),
+        environment: z.string().trim(),
+        secretPath: z.string().trim().default("/").transform(removeTrailingSlash),
+        keys: z.string().trim().transform(decodeURIComponent)
+      }),
+      response: {
+        200: z.object({
+          secrets: secretRawSchema
+            .extend({
+              secretPath: z.string().optional(),
+              tags: SecretTagsSchema.pick({
+                id: true,
+                slug: true,
+                color: true
+              })
+                .extend({ name: z.string() })
+                .array()
+                .optional()
+            })
+            .array()
+            .optional()
+        })
+      }
+    },
+    onRequest: verifyAuth([AuthMode.JWT]),
+    handler: async (req) => {
+      const { secretPath, projectId, environment } = req.query;
+
+      const keys = req.query.keys?.split(",").filter((key) => Boolean(key.trim())) ?? [];
+      if (!keys.length) throw new BadRequestError({ message: "One or more keys required" });
+
+      const { secrets } = await server.services.secret.getSecretsRaw({
+        actorId: req.permission.id,
+        actor: req.permission.type,
+        actorOrgId: req.permission.orgId,
+        environment,
+        actorAuthMethod: req.permission.authMethod,
+        projectId,
+        path: secretPath,
+        keys
+      });
+
+      await server.services.auditLog.createAuditLog({
+        projectId,
+        ...req.auditLogInfo,
+        event: {
+          type: EventType.GET_SECRETS,
+          metadata: {
+            environment,
+            secretPath,
+            numberOfSecrets: secrets.length
+          }
+        }
+      });
+
+      if (getUserAgentType(req.headers["user-agent"]) !== UserAgentType.K8_OPERATOR) {
+        await server.services.telemetry.sendPostHogEvents({
+          event: PostHogEventTypes.SecretPulled,
+          distinctId: getTelemetryDistinctId(req),
+          properties: {
+            numberOfSecrets: secrets.length,
+            workspaceId: projectId,
+            environment,
+            secretPath,
+            channel: getUserAgentType(req.headers["user-agent"]),
+            ...req.auditLogInfo
+          }
+        });
+      }
+
+      return { secrets };
+    }
+  });
 };

backend/src/services/secret-v2-bridge/secret-v2-bridge-dal.ts

@@ -361,6 +361,10 @@ export const secretV2BridgeDALFactory = (db: TDbClient) => {
               void bd.whereILike(`${TableName.SecretV2}.key`, `%${filters?.search}%`);
             }
           }
+
+          if (filters?.keys) {
+            void bd.whereIn(`${TableName.SecretV2}.key`, filters.keys);
+          }
         })
         .where((bd) => {
           void bd.whereNull(`${TableName.SecretV2}.userId`).orWhere({ userId: userId || null });

backend/src/services/secret-v2-bridge/secret-v2-bridge-types.ts

@@ -33,6 +33,7 @@ export type TGetSecretsDTO = {
   offset?: number;
   limit?: number;
   search?: string;
+  keys?: string[];
 } & TProjectPermission;
 
 export type TGetASecretDTO = {
@@ -294,6 +295,7 @@ export type TFindSecretsByFolderIdsFilter = {
   search?: string;
   tagSlugs?: string[];
   includeTagsInSearch?: boolean;
+  keys?: string[];
 };
 
 export type TGetSecretsRawByFolderMappingsDTO = {

backend/src/services/secret/secret-types.ts

@@ -185,6 +185,7 @@ export type TGetSecretsRawDTO = {
   offset?: number;
   limit?: number;
   search?: string;
+  keys?: string[];
 } & TProjectPermission;
 
 export type TGetASecretRawDTO = {

frontend/src/components/utilities/parseDotEnv.ts

@@ -6,7 +6,7 @@ const LINE =
  * @param {ArrayBuffer} src - source buffer
  * @returns {String} text - text of buffer
  */
-export function parseDotEnv(src: ArrayBuffer) {
+export function parseDotEnv(src: ArrayBuffer | string) {
   const object: {
     [key: string]: { value: string; comments: string[] };
   } = {};

frontend/src/components/utilities/parseJson.ts

@@ -0,0 +1,11 @@
+export const parseJson = (src: ArrayBuffer | string) => {
+  const file = src.toString();
+  const formatedData: Record<string, string> = JSON.parse(file);
+  const env: Record<string, { value: string; comments: string[] }> = {};
+  Object.keys(formatedData).forEach((key) => {
+    if (typeof formatedData[key] === "string") {
+      env[key] = { value: formatedData[key], comments: [] };
+    }
+  });
+  return env;
+};

frontend/src/hooks/api/dashboard/queries.tsx

@@ -5,6 +5,7 @@ import axios from "axios";
 import { createNotification } from "@app/components/notifications";
 import { apiRequest } from "@app/config/request";
 import {
+  DashboardProjectSecretsByKeys,
   DashboardProjectSecretsDetails,
   DashboardProjectSecretsDetailsResponse,
   DashboardProjectSecretsOverview,
@@ -12,6 +13,7 @@ import {
   DashboardSecretsOrderBy,
   TDashboardProjectSecretsQuickSearch,
   TDashboardProjectSecretsQuickSearchResponse,
+  TGetDashboardProjectSecretsByKeys,
   TGetDashboardProjectSecretsDetailsDTO,
   TGetDashboardProjectSecretsOverviewDTO,
   TGetDashboardProjectSecretsQuickSearchDTO
@@ -101,6 +103,23 @@ export const fetchProjectSecretsDetails = async ({
   return data;
 };
 
+export const fetchDashboardProjectSecretsByKeys = async ({
+  keys,
+  ...params
+}: TGetDashboardProjectSecretsByKeys) => {
+  const { data } = await apiRequest.get<DashboardProjectSecretsByKeys>(
+    "/api/v1/dashboard/secrets-by-keys",
+    {
+      params: {
+        ...params,
+        keys: encodeURIComponent(keys.join(","))
+      }
+    }
+  );
+
+  return data;
+};
+
 export const useGetProjectSecretsOverview = (
   {
     projectId,

frontend/src/hooks/api/dashboard/types.ts

@@ -29,6 +29,10 @@ export type DashboardProjectSecretsDetailsResponse = {
   totalCount: number;
 };
 
+export type DashboardProjectSecretsByKeys = {
+  secrets: SecretV3Raw[];
+};
+
 export type DashboardProjectSecretsOverview = Omit<
   DashboardProjectSecretsOverviewResponse,
   "secrets"
@@ -89,3 +93,10 @@ export type TGetDashboardProjectSecretsQuickSearchDTO = {
   search: string;
   environments: string[];
 };
+
+export type TGetDashboardProjectSecretsByKeys = {
+  projectId: string;
+  secretPath: string;
+  environment: string;
+  keys: string[];
+};

frontend/src/views/SecretMainPage/SecretMainPage.tsx

@@ -552,7 +552,6 @@ const SecretMainPageContent = () => {
             </ModalContent>
           </Modal>
           <SecretDropzone
-            secrets={secrets}
             environment={environment}
             workspaceId={workspaceId}
             secretPath={secretPath}

frontend/src/views/SecretMainPage/components/SecretDropzone/CopySecretsFromBoard.tsx

@@ -3,6 +3,7 @@ import { Controller, useForm } from "react-hook-form";
 import { subject } from "@casl/ability";
 import {
   faClone,
+  faFileImport,
   faKey,
   faSearch,
   faSquareCheck,
@@ -151,6 +152,7 @@ export const CopySecretsFromBoard = ({
           >
             {(isAllowed) => (
               <Button
+                leftIcon={<FontAwesomeIcon icon={faFileImport} />}
                 onClick={() => onToggle(true)}
                 isDisabled={!isAllowed}
                 variant="star"

frontend/src/views/SecretMainPage/components/SecretDropzone/PasteSecretEnvModal.tsx

@@ -0,0 +1,132 @@
+import { useForm } from "react-hook-form";
+import { subject } from "@casl/ability";
+import { faPaste } from "@fortawesome/free-solid-svg-icons";
+import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
+import { zodResolver } from "@hookform/resolvers/zod";
+import { z } from "zod";
+
+import { ProjectPermissionCan } from "@app/components/permissions";
+import { parseDotEnv } from "@app/components/utilities/parseDotEnv";
+import { parseJson } from "@app/components/utilities/parseJson";
+import {
+  Button,
+  FormControl,
+  Modal,
+  ModalContent,
+  ModalTrigger,
+  TextArea
+} from "@app/components/v2";
+import { ProjectPermissionActions, ProjectPermissionSub } from "@app/context";
+
+type Props = {
+  isOpen?: boolean;
+  isSmaller?: boolean;
+  onToggle: (isOpen: boolean) => void;
+  onParsedEnv: (env: Record<string, { value: string; comments: string[] }>) => void;
+  environment: string;
+  secretPath: string;
+};
+
+const formSchema = z.object({
+  value: z.string().trim()
+});
+
+type TForm = z.infer<typeof formSchema>;
+
+const PasteEnvForm = ({ onParsedEnv }: Pick<Props, "onParsedEnv">) => {
+  const {
+    handleSubmit,
+    register,
+    formState: { isDirty, errors },
+    setError,
+    setFocus
+  } = useForm<TForm>({ defaultValues: { value: "" }, resolver: zodResolver(formSchema) });
+
+  const onSubmit = ({ value }: TForm) => {
+    let env: Record<string, { value: string; comments: string[] }>;
+    try {
+      env = parseJson(value);
+    } catch (e) {
+      // not json, parse as env
+      env = parseDotEnv(value);
+    }
+
+    if (!Object.keys(env).length) {
+      setError("value", { message: "No secrets found." });
+      setFocus("value");
+      return;
+    }
+
+    onParsedEnv(env);
+  };
+
+  return (
+    <form onSubmit={handleSubmit(onSubmit)}>
+      <FormControl
+        label="Secret Values"
+        isError={Boolean(errors.value)}
+        errorText={errors.value?.message}
+      >
+        <TextArea
+          {...register("value")}
+          placeholder="Paste secrets in .json, .yml or .env format..."
+          className="h-[60vh] !resize-none"
+        />
+      </FormControl>
+      <Button isDisabled={!isDirty} type="submit">
+        Import Secrets
+      </Button>
+    </form>
+  );
+};
+
+export const PasteSecretEnvModal = ({
+  isSmaller,
+  isOpen,
+  onParsedEnv,
+  onToggle,
+  environment,
+  secretPath
+}: Props) => {
+  return (
+    <Modal isOpen={isOpen} onOpenChange={onToggle}>
+      <ModalTrigger asChild>
+        <div>
+          <ProjectPermissionCan
+            I={ProjectPermissionActions.Create}
+            a={subject(ProjectPermissionSub.Secrets, {
+              environment,
+              secretPath,
+              secretName: "*",
+              secretTags: ["*"]
+            })}
+          >
+            {(isAllowed) => (
+              <Button
+                leftIcon={<FontAwesomeIcon icon={faPaste} />}
+                onClick={() => onToggle(true)}
+                isDisabled={!isAllowed}
+                variant="star"
+                size={isSmaller ? "xs" : "sm"}
+              >
+                Paste Secret Values
+              </Button>
+            )}
+          </ProjectPermissionCan>
+        </div>
+      </ModalTrigger>
+      <ModalContent
+        className="max-w-2xl"
+        title="Past Secret Values"
+        subTitle="Paste values in .env, .json or .yml format"
+      >
+        <PasteEnvForm
+          onParsedEnv={(value) => {
+            onToggle(false);
+            onParsedEnv(value);
+          }}
+        />
+      </ModalContent>
+    </Modal>
+  );
+};

frontend/src/views/SecretMainPage/components/SecretDropzone/SecretDropzone.tsx

@@ -1,7 +1,7 @@
 import { ChangeEvent, DragEvent } from "react";
 import { useTranslation } from "react-i18next";
 import { subject } from "@casl/ability";
-import { faUpload } from "@fortawesome/free-solid-svg-icons";
+import { faPlus, faUpload } from "@fortawesome/free-solid-svg-icons";
 import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
 import { useQueryClient } from "@tanstack/react-query";
 import { twMerge } from "tailwind-merge";
@@ -10,29 +10,22 @@ import { createNotification } from "@app/components/notifications";
 import { ProjectPermissionCan } from "@app/components/permissions";
 // TODO:(akhilmhdh) convert all the util functions like this into a lib folder grouped by functionality
 import { parseDotEnv } from "@app/components/utilities/parseDotEnv";
+import { parseJson } from "@app/components/utilities/parseJson";
 import { Button, Modal, ModalContent } from "@app/components/v2";
 import { ProjectPermissionActions, ProjectPermissionSub } from "@app/context";
 import { usePopUp, useToggle } from "@app/hooks";
 import { useCreateSecretBatch, useUpdateSecretBatch } from "@app/hooks/api";
-import { dashboardKeys } from "@app/hooks/api/dashboard/queries";
+import {
+  dashboardKeys,
+  fetchDashboardProjectSecretsByKeys
+} from "@app/hooks/api/dashboard/queries";
 import { secretApprovalRequestKeys } from "@app/hooks/api/secretApprovalRequest/queries";
 import { secretKeys } from "@app/hooks/api/secrets/queries";
-import { SecretType, SecretV3RawSanitized } from "@app/hooks/api/types";
+import { SecretType } from "@app/hooks/api/types";
 
 import { PopUpNames, usePopUpAction } from "../../SecretMainPage.store";
 import { CopySecretsFromBoard } from "./CopySecretsFromBoard";
-
+import { PasteSecretEnvModal } from "./PasteSecretEnvModal";
-const parseJson = (src: ArrayBuffer) => {
-  const file = src.toString();
-  const formatedData: Record<string, string> = JSON.parse(file);
-  const env: Record<string, { value: string; comments: string[] }> = {};
-  Object.keys(formatedData).forEach((key) => {
-    if (typeof formatedData[key] === "string") {
-      env[key] = { value: formatedData[key], comments: [] };
-    }
-  });
-  return env;
-};
 
 type TParsedEnv = Record<string, { value: string; comments: string[] }>;
 type TSecOverwriteOpt = { update: TParsedEnv; create: TParsedEnv };
@@ -43,7 +36,6 @@ type Props = {
   workspaceId: string;
   environment: string;
   secretPath: string;
-  secrets?: SecretV3RawSanitized[];
   isProtectedBranch?: boolean;
 };
 
@@ -53,7 +45,6 @@ export const SecretDropzone = ({
   workspaceId,
   environment,
   secretPath,
-  secrets = [],
   isProtectedBranch = false
 }: Props): JSX.Element => {
   const { t } = useTranslation();
@@ -62,7 +53,8 @@ export const SecretDropzone = ({
 
   const { popUp, handlePopUpToggle, handlePopUpOpen, handlePopUpClose } = usePopUp([
     "importSecEnv",
-    "overlapKeyWarning"
+    "confirmUpload",
+    "pasteSecEnv"
   ] as const);
   const queryClient = useQueryClient();
   const { openPopUp } = usePopUpAction();
@@ -86,20 +78,10 @@ export const SecretDropzone = ({
     }
   };
 
-  const handleParsedEnv = (env: TParsedEnv) => {
+  const handleParsedEnv = async (env: TParsedEnv) => {
-    const secretsGroupedByKey = secrets?.reduce<Record<string, boolean>>(
+    const envSecretKeys = Object.keys(env);
-      (prev, curr) => ({ ...prev, [curr.key]: true }),
-      {}
-    );
-    const overlappedSecrets = Object.keys(env)
-      .filter((secKey) => secretsGroupedByKey?.[secKey])
-      .reduce<TParsedEnv>((prev, curr) => ({ ...prev, [curr]: env[curr] }), {});
 
-    const nonOverlappedSecrets = Object.keys(env)
+    if (!envSecretKeys.length) {
-      .filter((secKey) => !secretsGroupedByKey?.[secKey])
-      .reduce<TParsedEnv>((prev, curr) => ({ ...prev, [curr]: env[curr] }), {});
-
-    if (!Object.keys(overlappedSecrets).length && !Object.keys(nonOverlappedSecrets).length) {
       createNotification({
         type: "error",
         text: "Failed to find secrets"
@@ -107,10 +89,42 @@ export const SecretDropzone = ({
       return;
     }
 
-    handlePopUpOpen("overlapKeyWarning", {
+    try {
-      update: overlappedSecrets,
+      setIsLoading.on();
-      create: nonOverlappedSecrets
+      const { secrets: existingSecrets } = await fetchDashboardProjectSecretsByKeys({
-    });
+        secretPath,
+        environment,
+        projectId: workspaceId,
+        keys: envSecretKeys
+      });
+
+      const secretsGroupedByKey = existingSecrets.reduce<Record<string, boolean>>(
+        (prev, curr) => ({ ...prev, [curr.secretKey]: true }),
+        {}
+      );
+
+      const updateSecrets = Object.keys(env)
+        .filter((secKey) => secretsGroupedByKey[secKey])
+        .reduce<TParsedEnv>((prev, curr) => ({ ...prev, [curr]: env[curr] }), {});
+
+      const createSecrets = Object.keys(env)
+        .filter((secKey) => !secretsGroupedByKey[secKey])
+        .reduce<TParsedEnv>((prev, curr) => ({ ...prev, [curr]: env[curr] }), {});
+
+      handlePopUpOpen("confirmUpload", {
+        update: updateSecrets,
+        create: createSecrets
+      });
+    } catch (e) {
+      console.error(e);
+      createNotification({
+        text: "Failed to check for secret conflicts",
+        type: "error"
+      });
+      handlePopUpClose("confirmUpload");
+    } finally {
+      setIsLoading.off();
+    }
   };
 
   const parseFile = (file?: File, isJson?: boolean) => {
@@ -160,7 +174,7 @@ export const SecretDropzone = ({
   };
 
   const handleSaveSecrets = async () => {
-    const { update, create } = popUp?.overlapKeyWarning?.data as TSecOverwriteOpt;
+    const { update, create } = popUp?.confirmUpload?.data as TSecOverwriteOpt;
     try {
       if (Object.keys(create || {}).length) {
         await createSecretBatch({
@@ -195,7 +209,7 @@ export const SecretDropzone = ({
         dashboardKeys.getDashboardSecrets({ projectId: workspaceId, secretPath })
       );
       queryClient.invalidateQueries(secretApprovalRequestKeys.count({ workspaceId }));
-      handlePopUpClose("overlapKeyWarning");
+      handlePopUpClose("confirmUpload");
       createNotification({
         type: "success",
         text: isProtectedBranch
@@ -211,10 +225,16 @@ export const SecretDropzone = ({
     }
   };
 
-  const isUploadedDuplicateSecretsEmpty = !Object.keys(
+  const createSecretCount = Object.keys(
-    (popUp.overlapKeyWarning?.data as TSecOverwriteOpt)?.update || {}
+    (popUp.confirmUpload?.data as TSecOverwriteOpt)?.create || {}
   ).length;
 
+  const updateSecretCount = Object.keys(
+    (popUp.confirmUpload?.data as TSecOverwriteOpt)?.update || {}
+  ).length;
+
+  const isNonConflictingUpload = !updateSecretCount;
+
   return (
     <div>
       <div
@@ -278,7 +298,15 @@ export const SecretDropzone = ({
               <p className="mx-4 text-xs text-mineshaft-400">OR</p>
               <div className="w-1/5 border-t border-mineshaft-700" />
             </div>
-            <div className="flex items-center justify-center space-x-8">
+            <div className="flex flex-col items-center justify-center gap-4 lg:flex-row">
+              <PasteSecretEnvModal
+                isOpen={popUp.pasteSecEnv.isOpen}
+                onToggle={(isOpen) => handlePopUpToggle("pasteSecEnv", isOpen)}
+                onParsedEnv={handleParsedEnv}
+                environment={environment}
+                secretPath={secretPath}
+                isSmaller={isSmaller}
+              />
               <CopySecretsFromBoard
                 isOpen={popUp.importSecEnv.isOpen}
                 onToggle={(isOpen) => handlePopUpToggle("importSecEnv", isOpen)}
@@ -301,11 +329,12 @@ export const SecretDropzone = ({
                 >
                   {(isAllowed) => (
                     <Button
+                      leftIcon={<FontAwesomeIcon icon={faPlus} />}
                       onClick={() => openPopUp(PopUpNames.CreateSecretForm)}
                       variant="star"
                       isDisabled={!isAllowed}
                     >
-                      Add a new secret
+                      Add a New Secret
                     </Button>
                   )}
                 </ProjectPermissionCan>
@@ -315,25 +344,25 @@ export const SecretDropzone = ({
         )}
       </div>
       <Modal
-        isOpen={popUp?.overlapKeyWarning?.isOpen}
+        isOpen={popUp?.confirmUpload?.isOpen}
-        onOpenChange={(open) => handlePopUpToggle("overlapKeyWarning", open)}
+        onOpenChange={(open) => handlePopUpToggle("confirmUpload", open)}
       >
         <ModalContent
-          title={isUploadedDuplicateSecretsEmpty ? "Confirmation" : "Duplicate Secrets!!"}
+          title="Confirm Secret Upload"
           footerContent={[
             <Button
               isLoading={isSubmitting}
               isDisabled={isSubmitting}
-              colorSchema={isUploadedDuplicateSecretsEmpty ? "primary" : "danger"}
+              colorSchema={isNonConflictingUpload ? "primary" : "danger"}
               key="overwrite-btn"
               onClick={handleSaveSecrets}
             >
-              {isUploadedDuplicateSecretsEmpty ? "Upload" : "Overwrite"}
+              {isNonConflictingUpload ? "Upload" : "Overwrite"}
             </Button>,
             <Button
               key="keep-old-btn"
-              className="mr-4"
+              className="ml-4"
-              onClick={() => handlePopUpClose("overlapKeyWarning")}
+              onClick={() => handlePopUpClose("confirmUpload")}
               variant="outline_bg"
               isDisabled={isSubmitting}
             >
@@ -341,17 +370,27 @@ export const SecretDropzone = ({
             </Button>
           ]}
         >
-          {isUploadedDuplicateSecretsEmpty ? (
+          {isNonConflictingUpload ? (
-            <div>Upload secrets from this file</div>
+            <div>
+              Are you sure you want to import {createSecretCount} secret
+              {createSecretCount > 1 ? "s" : ""} to this environment?
+            </div>
           ) : (
-            <div className="flex flex-col space-y-2 text-gray-300">
+            <div className="flex flex-col text-gray-300">
-              <div>Your file contains following duplicate secrets</div>
+              <div>Your project already contains the following {updateSecretCount} secrets:</div>
-              <div className="text-sm text-gray-400">
+              <div className="mt-2 text-sm text-gray-400">
-                {Object.keys((popUp?.overlapKeyWarning?.data as TSecOverwriteOpt)?.update || {})
+                {Object.keys((popUp?.confirmUpload?.data as TSecOverwriteOpt)?.update || {})
                   ?.map((key) => key)
                   .join(", ")}
               </div>
-              <div>Are you sure you want to overwrite these secrets and create other ones?</div>
+              <div className="mt-6">
+                Are you sure you want to overwrite these secrets
+                {createSecretCount > 0
+                  ? ` and import ${createSecretCount} new
+                one${createSecretCount > 1 ? "s" : ""}`
+                  : ""}
+                ?
+              </div>
             </div>
           )}
         </ModalContent>