Merge pull request #2582 from Infisical/daniel/stream-upload

fix: env-key large file uploads

backend/package-lock.json

@@ -21,6 +21,7 @@
         "@fastify/etag": "^5.1.0",
         "@fastify/formbody": "^7.4.0",
         "@fastify/helmet": "^11.1.1",
+        "@fastify/multipart": "8.3.0",
         "@fastify/passport": "^2.4.0",
         "@fastify/rate-limit": "^9.0.0",
         "@fastify/session": "^10.7.0",
@@ -4311,6 +4312,15 @@
         "fast-uri": "^2.0.0"
       }
     },
+    "node_modules/@fastify/busboy": {
+      "version": "2.1.1",
+      "resolved": "https://registry.npmjs.org/@fastify/busboy/-/busboy-2.1.1.tgz",
+      "integrity": "sha512-vBZP4NlzfOlerQTnba4aqZoMhE/a9HY7HRqoOPaETQcSQuWEIyZMHGfVu6w9wGtGK5fED5qRs2DteVCjOH60sA==",
+      "license": "MIT",
+      "engines": {
+        "node": ">=14"
+      }
+    },
     "node_modules/@fastify/cookie": {
       "version": "9.3.1",
       "resolved": "https://registry.npmjs.org/@fastify/cookie/-/cookie-9.3.1.tgz",
@@ -4381,6 +4391,20 @@
         "helmet": "^7.0.0"
       }
     },
+    "node_modules/@fastify/multipart": {
+      "version": "8.3.0",
+      "resolved": "https://registry.npmjs.org/@fastify/multipart/-/multipart-8.3.0.tgz",
+      "integrity": "sha512-A8h80TTyqUzaMVH0Cr9Qcm6RxSkVqmhK/MVBYHYeRRSUbUYv08WecjWKSlG2aSnD4aGI841pVxAjC+G1GafUeQ==",
+      "license": "MIT",
+      "dependencies": {
+        "@fastify/busboy": "^2.1.0",
+        "@fastify/deepmerge": "^1.0.0",
+        "@fastify/error": "^3.0.0",
+        "fastify-plugin": "^4.0.0",
+        "secure-json-parse": "^2.4.0",
+        "stream-wormhole": "^1.1.0"
+      }
+    },
     "node_modules/@fastify/passport": {
       "version": "2.4.0",
       "resolved": "https://registry.npmjs.org/@fastify/passport/-/passport-2.4.0.tgz",
@@ -16604,6 +16628,15 @@
       "resolved": "https://registry.npmjs.org/stream-shift/-/stream-shift-1.0.3.tgz",
       "integrity": "sha512-76ORR0DO1o1hlKwTbi/DM3EXWGf3ZJYO8cXX5RJwnul2DEg2oyoZyjLNoQM8WsvZiFKCRfC1O0J7iCvie3RZmQ=="
     },
+    "node_modules/stream-wormhole": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/stream-wormhole/-/stream-wormhole-1.1.0.tgz",
+      "integrity": "sha512-gHFfL3px0Kctd6Po0M8TzEvt3De/xu6cnRrjlfYNhwbhLPLwigI2t1nc6jrzNuaYg5C4YF78PPFuQPzRiqn9ew==",
+      "license": "MIT",
+      "engines": {
+        "node": ">=4.0.0"
+      }
+    },
     "node_modules/string_decoder": {
       "version": "1.3.0",
       "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz",

backend/package.json

@@ -125,6 +125,7 @@
     "@fastify/etag": "^5.1.0",
     "@fastify/formbody": "^7.4.0",
     "@fastify/helmet": "^11.1.1",
+    "@fastify/multipart": "8.3.0",
     "@fastify/passport": "^2.4.0",
     "@fastify/rate-limit": "^9.0.0",
     "@fastify/session": "^10.7.0",

backend/src/server/routes/v3/external-migration-router.ts

@@ -1,5 +1,6 @@
-import { z } from "zod";
+import fastifyMultipart from "@fastify/multipart";
 
+import { BadRequestError } from "@app/lib/errors";
 import { readLimit } from "@app/server/config/rateLimiter";
 import { verifyAuth } from "@app/server/plugins/auth/verify-auth";
 import { AuthMode } from "@app/services/auth/auth-type";
@@ -7,6 +8,8 @@ import { AuthMode } from "@app/services/auth/auth-type";
 const MB25_IN_BYTES = 26214400;
 
 export const registerExternalMigrationRouter = async (server: FastifyZodProvider) => {
+  await server.register(fastifyMultipart);
+
   server.route({
     method: "POST",
     bodyLimit: MB25_IN_BYTES,
@@ -14,20 +17,34 @@ export const registerExternalMigrationRouter = async (server: FastifyZodProvider
     config: {
       rateLimit: readLimit
     },
-    schema: {
-      body: z.object({
-        decryptionKey: z.string().trim().min(1),
-        encryptedJson: z.object({
-          nonce: z.string().trim().min(1),
-          data: z.string().trim().min(1)
-        })
-      })
-    },
     onRequest: verifyAuth([AuthMode.JWT, AuthMode.IDENTITY_ACCESS_TOKEN]),
     handler: async (req) => {
+      const data = await req.file({
+        limits: {
+          fileSize: MB25_IN_BYTES
+        }
+      });
+
+      if (!data) {
+        throw new BadRequestError({ message: "No file provided" });
+      }
+
+      const fullFile = Buffer.from(await data.toBuffer()).toString("utf8");
+      const parsedJsonFile = JSON.parse(fullFile) as { nonce: string; data: string };
+
+      const decryptionKey = (data.fields.decryptionKey as { value: string }).value;
+
+      if (!parsedJsonFile.nonce || !parsedJsonFile.data) {
+        throw new BadRequestError({ message: "Invalid file format. Nonce or data missing." });
+      }
+
+      if (!decryptionKey) {
+        throw new BadRequestError({ message: "Decryption key is required" });
+      }
+
       await server.services.migration.importEnvKeyData({
-        decryptionKey: req.body.decryptionKey,
+        decryptionKey,
-        encryptedJson: req.body.encryptedJson,
+        encryptedJson: parsedJsonFile,
         actorId: req.permission.id,
         actor: req.permission.type,
         actorOrgId: req.permission.orgId,

frontend/src/hooks/api/migration/mutations.tsx

@@ -8,20 +8,27 @@ export const useImportEnvKey = () => {
   const queryClient = useQueryClient();
 
   return useMutation({
-    mutationFn: async ({
+    mutationFn: async ({ file, decryptionKey }: { file: File; decryptionKey: string }) => {
-      encryptedJson,
+      const formData = new FormData();
-      decryptionKey
+
-    }: {
+      formData.append("decryptionKey", decryptionKey);
-      encryptedJson: {
+      formData.append("file", file);
-        nonce: string;
+
-        data: string;
+      try {
-      };
+        const response = await apiRequest.post("/api/v3/migrate/env-key/", formData, {
-      decryptionKey: string;
+          headers: {
-    }) => {
+            "Content-Type": "multipart/form-data"
-      await apiRequest.post("/api/v3/migrate/env-key/", {
+          },
-        encryptedJson,
+          onUploadProgress: (progressEvent) => {
-        decryptionKey
+            const percentCompleted = Math.round((progressEvent.loaded * 100) / progressEvent.total);
-      });
+            console.log(`Upload Progress: ${percentCompleted}%`);
+          }
+        });
+
+        console.log("Upload successful:", response.data);
+      } catch (error) {
+        console.error("Upload failed:", error);
+      }
     },
     onSuccess: () => {
       queryClient.invalidateQueries(workspaceKeys.getAllUserWorkspace);

frontend/src/views/Settings/OrgSettingsPage/components/ImportTab/components/EnvKeyPlatformModal.tsx

@@ -13,17 +13,13 @@ type Props = {
   onClose: () => void;
 };
 
-const formSchema = z.object({
-  encryptionKey: z.string().min(1),
-  encryptedJson: z.object({
-    nonce: z.string().min(1),
-    data: z.string().min(1)
-  })
-});
-
-type TFormData = z.infer<typeof formSchema>;
-
 export const EnvKeyPlatformModal = ({ onClose }: Props) => {
+  const formSchema = z.object({
+    encryptionKey: z.string().min(1),
+    file: z.instanceof(File)
+  });
+  type TFormData = z.infer<typeof formSchema>;
+
   const fileUploadRef = useRef<HTMLInputElement>(null);
 
   const { mutateAsync: importEnvKey } = useImportEnvKey();
@@ -40,8 +36,8 @@ export const EnvKeyPlatformModal = ({ onClose }: Props) => {
   });
 
   const onSubmit = async (data: TFormData) => {
-    if (!data.encryptedJson) {
+    if (!data.file) {
-      setError("encryptedJson", {
+      setError("file", {
         type: "required",
         message: "File is required"
       });
@@ -50,7 +46,7 @@ export const EnvKeyPlatformModal = ({ onClose }: Props) => {
 
     try {
       await importEnvKey({
-        encryptedJson: data.encryptedJson,
+        file: data.file,
         decryptionKey: data.encryptionKey
       });
       createNotification({
@@ -71,7 +67,6 @@ export const EnvKeyPlatformModal = ({ onClose }: Props) => {
   };
 
   const onImportFileDrop = (file?: File) => {
-    const reader = new FileReader();
     if (!file) {
       createNotification({
         text: "No file selected.",
@@ -79,40 +74,8 @@ export const EnvKeyPlatformModal = ({ onClose }: Props) => {
       });
       return;
     }
-    reader.onload = (event) => {
-      if (!event?.target?.result) return;
 
-      const droppedFile = event.target.result.toString();
+    setValue("file", file, { shouldDirty: true, shouldValidate: true });
-      const formattedData: Record<string, string> = JSON.parse(droppedFile);
-      if (
-        Object.keys(formattedData).includes("nonce") &&
-        Object.keys(formattedData).includes("data")
-      ) {
-        const data = {
-          nonce: formattedData.nonce,
-          data: formattedData.data
-        };
-        setValue("encryptedJson", data, { shouldDirty: true, shouldValidate: true });
-      } else {
-        setValue(
-          "encryptedJson",
-          {
-            nonce: "",
-            data: ""
-          },
-          { shouldDirty: true, shouldValidate: true }
-        );
-
-        if (fileUploadRef.current) {
-          fileUploadRef.current.value = "";
-        }
-        createNotification({
-          text: "Improper file format, please upload the EnvKey export.",
-          type: "error"
-        });
-      }
-    };
-    reader.readAsText(file);
   };
 
   return (