Merge pull request #188 from mocherfaoui/import-export-secrets

add ability to import/export secrets with comments

frontend/components/dashboard/DropZone.tsx

@@ -3,10 +3,11 @@ import Image from "next/image";
 import { useTranslation } from "next-i18next";
 import { faUpload } from "@fortawesome/free-solid-svg-icons";
 import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
+import { parseDocument, Scalar, YAMLMap } from 'yaml';
 
 import Button from "../basic/buttons/Button";
 import Error from "../basic/Error";
-import parse from "../utilities/file";
+import { parseDotEnv } from '../utilities/parseDotEnv';
 import guidGenerator from "../utilities/randomId";
 
 interface DropZoneProps {
@@ -51,6 +52,53 @@ const DropZone = ({
 
   const [loading, setLoading] = useState(false);
 
+  const getSecrets = (file: ArrayBuffer, fileType: string) => {
+    let secrets;
+    switch (fileType) {
+      case 'env': {
+        const keyPairs = parseDotEnv(file);
+        secrets = Object.keys(keyPairs).map((key, index) => {
+          return {
+            id: guidGenerator(),
+            pos: numCurrentRows + index,
+            key: key,
+            value: keyPairs[key as keyof typeof keyPairs].value,
+            comment: keyPairs[key as keyof typeof keyPairs].comments.join('\n'),
+            type: 'shared',
+          };
+        });
+        break;
+      }
+      case 'yml': {
+        const parsedFile = parseDocument(file.toString());
+        const keyPairs = parsedFile.contents!.toJSON();
+
+        secrets = Object.keys(keyPairs).map((key, index) => {
+          const fileContent = parsedFile.contents as YAMLMap<Scalar, Scalar>;
+          const comment =
+            fileContent!.items
+              .find((item) => item.key.value === key)
+              ?.key?.commentBefore?.split('\n')
+              .map((comment) => comment.trim())
+              .join('\n') ?? '';
+          return {
+            id: guidGenerator(),
+            pos: numCurrentRows + index,
+            key: key,
+            value: keyPairs[key as keyof typeof keyPairs]?.toString() ?? '',
+            comment,
+            type: 'shared',
+          };
+        });
+        break;
+      }
+      default:
+        secrets = '';
+        break;
+    }
+    return secrets;
+  };
+
   // This function function immediately parses the file after it is dropped
   const handleDrop = async (e: DragEvent) => {
     setLoading(true);
@@ -61,20 +109,12 @@ const DropZone = ({
 
     const file = e.dataTransfer.files[0];
     const reader = new FileReader();
+    const fileType = file.name.split('.')[1];
 
     reader.onload = (event) => {
       if (event.target === null || event.target.result === null) return;
       // parse function's argument looks like to be ArrayBuffer
-      const keyPairs = parse(event.target.result as Buffer);
-      const newData = Object.keys(keyPairs).map((key, index) => {
-        return {
-          id: guidGenerator(),
-          pos: numCurrentRows + index,
-          key: key,
-          value: keyPairs[key as keyof typeof keyPairs],
-          type: "shared",
-        };
-      });
+      const newData = getSecrets(event.target.result as ArrayBuffer, fileType);
       setData(newData);
       setButtonReady(true);
     };
@@ -95,25 +135,14 @@ const DropZone = ({
     setTimeout(() => setLoading(false), 5000);
     if (e.currentTarget.files === null) return;
     const file = e.currentTarget.files[0];
+    const fileType = file.name.split('.')[1];
     const reader = new FileReader();
     reader.onload = (event) => {
       if (event.target === null || event.target.result === null) return;
       const { result } = event.target;
-      if (typeof result === "string") {
-        const newData = result
-          .split("\n")
-          .map((line: string, index: number) => {
-            return {
-              id: guidGenerator(),
-              pos: numCurrentRows + index,
-              key: line.split("=")[0],
-              value: line.split("=").slice(1, line.split("=").length).join("="),
-              type: "shared",
-            };
-          });
-        setData(newData);
-        setButtonReady(true);
-      }
+      const newData = getSecrets(result as ArrayBuffer, fileType);
+      setData(newData);
+      setButtonReady(true);
     };
     reader.readAsText(file);
   };
@@ -139,7 +168,7 @@ const DropZone = ({
         id="fileSelect"
         type="file"
         className="opacity-0 absolute w-full h-full"
-        accept=".txt,.env"
+        accept=".txt,.env,.yml"
         onChange={handleFileSelect}
       />
       {errorDragAndDrop ? (
@@ -176,7 +205,7 @@ const DropZone = ({
         id="fileSelect"
         type="file"
         className="opacity-0 absolute w-full h-full"
-        accept=".txt,.env"
+        accept=".txt,.env,.yml"
         onChange={handleFileSelect}
       />
       <div className="flex flex-row w-full items-center justify-center mb-6 mt-5">

frontend/components/utilities/file.ts

@@ -1,47 +0,0 @@
-const LINE =
-  /(?:^|^)\s*(?:export\s+)?([\w.-]+)(?:\s*=\s*?|:\s+?)(\s*'(?:\\'|[^'])*'|\s*"(?:\\"|[^"])*"|\s*`(?:\\`|[^`])*`|[^#\r\n]+)?\s*(?:#.*)?(?:$|$)/gm;
-
-/**
- * Return text that is the buffer parsed
- * @param {Buffer} src - source buffer
- * @returns {String} text - text of buffer
- */
-function parse(src: Buffer) {
-  const obj: Record<string, string> = {};
-
-  // Convert buffer to string
-  let lines = src.toString();
-
-  // Convert line breaks to same format
-  lines = lines.replace(/\r\n?/gm, '\n');
-
-  let match;
-  while ((match = LINE.exec(lines)) != null) {
-    const key = match[1];
-
-    // Default undefined or null to empty string
-    let value = match[2] || '';
-
-    // Remove whitespace
-    value = value.trim();
-
-    // Check if double quoted
-    const maybeQuote = value[0];
-
-    // Remove surrounding quotes
-    value = value.replace(/^(['"`])([\s\S]*)\1$/gm, '$2');
-
-    // Expand newlines if double quoted
-    if (maybeQuote === '"') {
-      value = value.replace(/\\n/g, '\n');
-      value = value.replace(/\\r/g, '\r');
-    }
-
-    // Add to object
-    obj[key] = value;
-  }
-
-  return obj;
-}
-
-export default parse;

frontend/components/utilities/parseDotEnv.ts

@@ -0,0 +1,66 @@
+const LINE =
+  /(?:^|^)\s*(?:export\s+)?([\w.-]+)(?:\s*=\s*?|:\s+?)(\s*'(?:\\'|[^'])*'|\s*"(?:\\"|[^"])*"|\s*`(?:\\`|[^`])*`|[^#\r\n]+)?\s*(?:#.*)?(?:$|$)/gm;
+
+/**
+ * Return text that is the buffer parsed
+ * @param {ArrayBuffer} src - source buffer
+ * @returns {String} text - text of buffer
+ */
+export function parseDotEnv(src: ArrayBuffer) {
+  const object: {
+    [key: string]: { value: string; comments: string[] };
+  } = {};
+
+  // Convert buffer to string
+  let lines = src.toString();
+
+  // Convert line breaks to same format
+  lines = lines.replace(/\r\n?/gm, '\n');
+
+  let comments: string[] = [];
+
+  lines
+    .split('\n')
+    .map((line) => {
+      // collect comments of each env variable
+      if (line.startsWith('#')) {
+        comments.push(line.replace('#', '').trim());
+      } else if (line) {
+        let match;
+        let item: [string, string, string[]] | [] = [];
+
+        while ((match = LINE.exec(line)) !== null) {
+          const key = match[1];
+
+          // Default undefined or null to empty string
+          let value = match[2] || '';
+
+          // Remove whitespace
+          value = value.trim();
+
+          // Check if double quoted
+          const maybeQuote = value[0];
+
+          // Remove surrounding quotes
+          value = value.replace(/^(['"`])([\s\S]*)\1$/gm, '$2');
+
+          // Expand newlines if double quoted
+          if (maybeQuote === '"') {
+            value = value.replace(/\\n/g, '\n');
+            value = value.replace(/\\r/g, '\r');
+          }
+          item = [key, value, comments];
+        }
+        comments = [];
+        return item;
+      }
+      return [];
+    })
+    .filter((line) => line.length > 1)
+    .forEach((line) => {
+      const [key, value, comments] = line;
+      object[key as string] = { value, comments };
+    });
+
+  return object;
+}

frontend/components/utilities/telemetry/Telemetry.js

@@ -29,7 +29,7 @@ class Capturer {
 
 }
 
-class Telemetry {
+export default class Telemetry {
   constructor() {
       if (!Telemetry.instance) {
         Telemetry.instance = new Capturer();
@@ -40,5 +40,3 @@ class Telemetry {
       return Telemetry.instance;
   }
 }
-
-module.exports = Telemetry;

frontend/package.json

@@ -53,7 +53,8 @@
     "tweetnacl": "^1.0.3",
     "tweetnacl-util": "^0.15.1",
     "uuid": "^8.3.2",
-    "uuidv4": "^6.2.13"
+    "uuidv4": "^6.2.13",
+    "yaml": "^2.2.0"
   },
   "devDependencies": {
     "@tailwindcss/typography": "^0.5.4",

frontend/pages/dashboard/[id].tsx

@@ -17,8 +17,10 @@ import {
   faPlus,
 } from '@fortawesome/free-solid-svg-icons';
 import { FontAwesomeIcon } from '@fortawesome/react-fontawesome';
+import { Menu, Transition } from '@headlessui/react';
 import getProjectSercetSnapshotsCount from 'ee/api/secrets/GetProjectSercetSnapshotsCount';
 import PITRecoverySidebar from 'ee/components/PITRecoverySidebar';
+import { Document, YAMLSeq } from 'yaml';
 
 import Button from '~/components/basic/buttons/Button';
 import ListBox from '~/components/basic/Listbox';
@@ -425,16 +427,75 @@ export default function Dashboard() {
     setData(sortedData);
   };
 
+  // check if there are secrets with an override
+  const checkOverrides = (data: SecretDataProps[]) => {
+    let secrets : SecretDataProps[] = data!.map((secret) => Object.create(secret));
+    const overridenSecrets = data!.filter(
+      (secret) => secret.type === 'personal'
+    );
+    if (overridenSecrets.length) {
+      overridenSecrets.forEach((secret) => {
+        const index = secrets!.findIndex(
+          (_secret) => _secret.key === secret.key && _secret.type === 'shared'
+        );
+        secrets![index].value = secret.value;
+      });
+      secrets = secrets!.filter((secret) => secret.type === 'shared');
+    }
+    return secrets;
+  };
   // This function downloads the secrets as a .env file
-  const download = () => {
-    const file = data!
-      .map((item: SecretDataProps) => [item.key, item.value].join('='))
+  const downloadDotEnv = () => {
+    if (!data) return;
+    const secrets = checkOverrides(data)
+
+    const file = secrets!
+      .map(
+        (item: SecretDataProps) =>
+          `${
+            item.comment
+              ? item.comment
+                  .split('\n')
+                  .map((comment) => '# '.concat(comment))
+                  .join('\n') + '\n'
+              : ''
+          }` + [item.key, item.value].join('=')
+      )
       .join('\n');
+
+        const blob = new Blob([file]);
+        const fileDownloadUrl = URL.createObjectURL(blob);
+        const alink = document.createElement('a');
+        alink.href = fileDownloadUrl;
+        alink.download = envMapping[env] + '.env';
+        alink.click();
+  };
+
+  // This function downloads the secrets as a .yml file
+  const downloadYaml = () => {
+    if (!data) return;
+    const doc = new Document(new YAMLSeq());
+    const secrets = checkOverrides(data);
+    secrets.forEach((secret) => {
+      const pair = doc.createNode({ [secret.key]: secret.value });
+      pair.commentBefore = secret.comment
+        .split('\n')
+        .map((line) => (line ? ' '.concat(line) : ''))
+        .join('\n');
+      doc.add(pair);
+    });
+
+    const file = doc
+      .toString()
+      .split('\n')
+      .map((line) => (line.startsWith('-') ? line.replace('- ', '') : line))
+      .join('\n');
+
     const blob = new Blob([file]);
     const fileDownloadUrl = URL.createObjectURL(blob);
     const alink = document.createElement('a');
     alink.href = fileDownloadUrl;
-    alink.download = envMapping[env] + '.env';
+    alink.download = envMapping[env] + '.yml';
     alink.click();
   };
 
@@ -614,12 +675,50 @@ export default function Dashboard() {
                       />
                     </div>}
                     {!snapshotData && <div className="ml-2 min-w-max flex flex-row items-start justify-start">
-                      <Button
-                        onButtonPressed={download}
-                        color="mineshaft"
-                        size="icon-md"
-                        icon={faDownload}
-                      />
+                      <Menu
+                        as="div"
+                        className="relative inline-block text-left"
+                      >
+                        <Menu.Button
+                          as="div"
+                          className="inline-flex w-full justify-center  text-sm font-medium text-gray-200 rounded-md hover:bg-white/10 duration-200 focus:outline-none focus-visible:ring-2 focus-visible:ring-white focus-visible:ring-opacity-75"
+                        >
+                          <Button
+                            color="mineshaft"
+                            size="icon-md"
+                            icon={faDownload}
+                            onButtonPressed={() => {}}
+                          />
+                        </Menu.Button>
+                        <Transition
+                          as={Fragment}
+                          enter="transition ease-out duration-100"
+                          enterFrom="transform opacity-0 scale-95"
+                          enterTo="transform opacity-100 scale-100"
+                          leave="transition ease-in duration-75"
+                          leaveFrom="transform opacity-100 scale-100"
+                          leaveTo="transform opacity-0 scale-95"
+                        >
+                          <Menu.Items className="absolute z-50 drop-shadow-xl right-0 mt-0.5 w-[20rem] origin-top-right rounded-md bg-bunker border border-mineshaft-500 shadow-lg ring-1 ring-black ring-opacity-5 focus:outline-none p-2 space-y-2">
+                            <Menu.Item>
+                              <Button
+                                color="mineshaft"
+                                onButtonPressed={downloadDotEnv}
+                                size="md"
+                                text="Download as .env"
+                              />
+                            </Menu.Item>
+                            <Menu.Item>
+                              <Button
+                                color="mineshaft"
+                                onButtonPressed={downloadYaml}
+                                size="md"
+                                text="Download as .yml"
+                              />
+                            </Menu.Item>
+                          </Menu.Items>
+                        </Transition>
+                      </Menu>
                     </div>}
                     <div className="ml-2 min-w-max flex flex-row items-start justify-start">
                       <Button
@@ -672,7 +771,9 @@ export default function Dashboard() {
                         modifyValue={listenChangeValue}
                         modifyKey={listenChangeKey}
                         isBlurred={blurred}
-                        isDuplicate={findDuplicates(data?.map((item) => item.key + item.type))?.includes(keyPair.key + keyPair.type)}
+                        isDuplicate={findDuplicates(
+                          data?.map((item) => item.key + item.type)
+                        )?.includes(keyPair.key + keyPair.type)}
                         toggleSidebar={toggleSidebar}
                         sidebarSecretId={sidebarSecretId}
                         isSnapshot={false}
@@ -694,7 +795,9 @@ export default function Dashboard() {
                         modifyValue={listenChangeValue}
                         modifyKey={listenChangeKey}
                         isBlurred={blurred}
-                        isDuplicate={findDuplicates(data?.map((item) => item.key + item.type))?.includes(keyPair.key + keyPair.type)}
+                        isDuplicate={findDuplicates(
+                          data?.map((item) => item.key + item.type)
+                        )?.includes(keyPair.key + keyPair.type)}
                         toggleSidebar={toggleSidebar}
                         sidebarSecretId={sidebarSecretId}
                         isSnapshot={true}