feat(infisical-pg): idempotent folder creation

backend-pg/package-lock.json

@@ -58,6 +58,7 @@
         "smee-client": "^2.0.0",
         "tweetnacl": "^1.0.3",
         "tweetnacl-util": "^0.15.1",
+        "uuid": "^9.0.1",
         "zod": "^3.22.4",
         "zod-to-json-schema": "^3.22.0"
       },

backend-pg/package.json

@@ -115,6 +115,7 @@
     "smee-client": "^2.0.0",
     "tweetnacl": "^1.0.3",
     "tweetnacl-util": "^0.15.1",
+    "uuid": "^9.0.1",
     "zod": "^3.22.4",
     "zod-to-json-schema": "^3.22.0"
   }

backend-pg/src/services/secret-folder/secret-folder-dal.ts

@@ -252,6 +252,27 @@ export const secretFolderDalFactory = (db: TDbClient) => {
     }
   };
 
+  // used in folder creation
+  // even if its the original given /path1/path2
+  // it will stop automatically at /path2
+  const findClosestFolder = async (
+    projectId: string,
+    environment: string,
+    path: string,
+    tx?: Knex
+  ) => {
+    try {
+      const folder = await sqlFindFolderByPathQuery(tx || db, projectId, environment, path)
+        .orderBy("depth", "desc")
+        .first();
+      if (!folder) return;
+      const { envId: id, envName: name, envSlug: slug, ...el } = folder;
+      return { ...el, envId: id, environment: { id, name, slug } };
+    } catch (error) {
+      throw new DatabaseError({ error, name: "Find by secret path" });
+    }
+  };
+
   const findByManySecretPath = async (
     query: Array<{ envId: string; secretPath: string }>,
     tx?: Knex
@@ -329,6 +350,7 @@ export const secretFolderDalFactory = (db: TDbClient) => {
     findBySecretPath,
     findById,
     findByManySecretPath,
-    findSecretPathByFolderIds
+    findSecretPathByFolderIds,
+    findClosestFolder
   };
 };

backend-pg/src/services/secret-folder/secret-folder-service.ts

@@ -1,5 +1,8 @@
 import { ForbiddenError, subject } from "@casl/ability";
+import path from "path";
+import { v4 as uuidv4 } from "uuid";
 
+import { TSecretFoldersInsert } from "@app/db/schemas";
 import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service";
 import {
   ProjectPermissionActions,
@@ -41,12 +44,12 @@ export const secretFolderServiceFactory = ({
     actorId,
     name,
     environment,
-    path
+    path: secretPath
   }: TCreateFolderDTO) => {
     const { permission } = await permissionService.getProjectPermission(actor, actorId, projectId);
     ForbiddenError.from(permission).throwUnlessCan(
       ProjectPermissionActions.Create,
-      subject(ProjectPermissionSub.Secrets, { environment, secretPath: path })
+      subject(ProjectPermissionSub.Secrets, { environment, secretPath })
     );
 
     const env = await projectEnvDal.findOne({ projectId, slug: environment });
@@ -54,10 +57,66 @@ export const secretFolderServiceFactory = ({
       throw new BadRequestError({ message: "Environment not found", name: "Create folder" });
 
     const folder = await folderDal.transaction(async (tx) => {
-      const parentFolder = await folderDal.findBySecretPath(projectId, environment, path, tx);
+      // the logic is simple we need to avoid creating same folder in same path multiple times
+      // that is this request must be idempotent
+      // so we do a tricky move. we try to find the to be created folder path if that is exactly match return that
+      // else we get some path before that then we will start creating remaining folder
+      const pathWithFolder = path.join(secretPath, name);
+      const parentFolder = await folderDal.findClosestFolder(
+        projectId,
+        environment,
+        pathWithFolder,
+        tx
+      );
+      // no folder found is not possible root should be their
       if (!parentFolder) throw new BadRequestError({ message: "Secret path not found" });
+      // exact folder
+      if (parentFolder.path === pathWithFolder) return parentFolder;
+
+      let parentFolderId = parentFolder.id;
+      if (parentFolder.path !== secretPath) {
+        // this is upsert folder in a path
+        // we are not taking snapshots of this because
+        // snapshot will be removed from automatic for all commits to user click or cron based
+        const missingSegment = secretPath
+          .substring(parentFolder.path.length)
+          .split("/")
+          .filter(Boolean);
+        if (missingSegment.length) {
+          const newFolders: Array<TSecretFoldersInsert & { id: string }> = missingSegment.map(
+            (segment, i) =>
+              i === 0
+                ? {
+                    name: segment,
+                    parentId: parentFolder.id,
+                    id: uuidv4(),
+                    envId: env.id,
+                    version: 1
+                  }
+                : {
+                    name: segment,
+                    parentId: newFolders[i - 1].id,
+                    id: uuidv4(),
+                    envId: env.id,
+                    version: 1
+                  }
+          );
+          parentFolderId = newFolders.at(-1)?.id as string;
+          const docs = await folderDal.insertMany(newFolders, tx);
+          await folderVersionDal.insertMany(
+            docs.map((doc) => ({
+              name: doc.name,
+              envId: doc.envId,
+              version: doc.version,
+              folderId: doc.id
+            })),
+            tx
+          );
+        }
+      }
+
       const doc = await folderDal.create(
-        { name, envId: env.id, version: 1, parentId: parentFolder.id },
+        { name, envId: env.id, version: 1, parentId: parentFolderId },
         tx
       );
       await folderVersionDal.create(
@@ -82,16 +141,16 @@ export const secretFolderServiceFactory = ({
     actorId,
     name,
     environment,
-    path,
+    path: secretPath,
     id
   }: TUpdateFolderDTO) => {
     const { permission } = await permissionService.getProjectPermission(actor, actorId, projectId);
     ForbiddenError.from(permission).throwUnlessCan(
       ProjectPermissionActions.Edit,
-      subject(ProjectPermissionSub.Secrets, { environment, secretPath: path })
+      subject(ProjectPermissionSub.Secrets, { environment, secretPath })
     );
 
-    const parentFolder = await folderDal.findBySecretPath(projectId, environment, path);
+    const parentFolder = await folderDal.findBySecretPath(projectId, environment, secretPath);
     if (!parentFolder) throw new BadRequestError({ message: "Secret path not found" });
 
     const env = await projectEnvDal.findOne({ projectId, slug: environment });
@@ -133,13 +192,13 @@ export const secretFolderServiceFactory = ({
     actor,
     actorId,
     environment,
-    path,
+    path: secretPath,
     id
   }: TDeleteFolderDTO) => {
     const { permission } = await permissionService.getProjectPermission(actor, actorId, projectId);
     ForbiddenError.from(permission).throwUnlessCan(
       ProjectPermissionActions.Delete,
-      subject(ProjectPermissionSub.Secrets, { environment, secretPath: path })
+      subject(ProjectPermissionSub.Secrets, { environment, secretPath })
     );
 
     const env = await projectEnvDal.findOne({ projectId, slug: environment });
@@ -147,7 +206,7 @@ export const secretFolderServiceFactory = ({
       throw new BadRequestError({ message: "Environment not found", name: "Create folder" });
 
     const folder = await folderDal.transaction(async (tx) => {
-      const parentFolder = await folderDal.findBySecretPath(projectId, environment, path, tx);
+      const parentFolder = await folderDal.findBySecretPath(projectId, environment, secretPath, tx);
       if (!parentFolder) throw new BadRequestError({ message: "Secret path not found" });
 
       const [doc] = await folderDal.delete({ envId: env.id, id, parentId: parentFolder.id }, tx);
@@ -159,7 +218,13 @@ export const secretFolderServiceFactory = ({
     return folder;
   };
 
-  const getFolders = async ({ projectId, actor, actorId, environment, path }: TGetFolderDTO) => {
+  const getFolders = async ({
+    projectId,
+    actor,
+    actorId,
+    environment,
+    path: secretPath
+  }: TGetFolderDTO) => {
     // folder list is allowed to be read by anyone
     // permission to check does user has access
     await permissionService.getProjectPermission(actor, actorId, projectId);
@@ -167,8 +232,8 @@ export const secretFolderServiceFactory = ({
     const env = await projectEnvDal.findOne({ projectId, slug: environment });
     if (!env) throw new BadRequestError({ message: "Environment not found", name: "get folders" });
 
-    const parentFolder = await folderDal.findBySecretPath(projectId, environment, path);
+    const parentFolder = await folderDal.findBySecretPath(projectId, environment, secretPath);
-    if (!parentFolder) throw new BadRequestError({ message: "Secret path not found" });
+    if (!parentFolder) return [];
 
     const folders = await folderDal.find({ envId: env.id, parentId: parentFolder.id });
     return folders;

backend-pg/src/services/secret-import/secret-import-service.ts

@@ -58,7 +58,10 @@ export const secretImportServiceFactory = ({
     // check if user has permission to import from target path
     ForbiddenError.from(permission).throwUnlessCan(
       ProjectPermissionActions.Create,
-      subject(ProjectPermissionSub.Secrets, { environment: data.environment, secretPath: data.path })
+      subject(ProjectPermissionSub.Secrets, {
+        environment: data.environment,
+        secretPath: data.path
+      })
     );
 
     const folder = await folderDal.findBySecretPath(projectId, environment, path);
@@ -196,7 +199,7 @@ export const secretImportServiceFactory = ({
       subject(ProjectPermissionSub.Secrets, { environment, secretPath: path })
     );
     const folder = await folderDal.findBySecretPath(projectId, environment, path);
-    if (!folder) throw new BadRequestError({ message: "Folder not found", name: "Get imports" });
+    if (!folder) return [];
     // this will already order by position
     // so anything based on this order will also be in right position
     const secretImports = await secretImportDal.find({ folderId: folder.id });

backend-pg/src/services/secret/secret-service.ts

@@ -499,7 +499,7 @@ export const secretServiceFactory = ({
     );
 
     const folder = await folderDal.findBySecretPath(projectId, environment, path);
-    if (!folder) throw new BadRequestError({ message: "Folder not  found", name: "Create secret" });
+    if (!folder) return { secrets: [], imports: [] };
     const folderId = folder.id;
 
     const secrets = await secretDal.findByFolderId(folderId, actorId);

frontend/src/views/SecretOverviewPage/SecretOverviewPage.tsx

@@ -120,7 +120,7 @@ export const SecretOverviewPage = () => {
         if (folderName && parentPath) {
           await createFolder({
             projectId: workspaceId,
-            path: secretPath,
+            path: parentPath,
             environment: env,
             name: folderName
           });
@@ -211,14 +211,15 @@ export const SecretOverviewPage = () => {
 
   const handleExploreEnvClick = async (slug: string) => {
     if (secretPath !== "/") {
-      const path = secretPath.split("/");
+      const pathSegment = secretPath.split("/").filter(Boolean);
-      const directory = path.slice(0, -1).join("/");
+      const parentPath = `/${pathSegment.slice(0, -1).join("/")}`;
-      const folderName = path.at(-1);
+      const folderName = pathSegment.at(-1);
-      if (folderName && directory) {
+      console.log(folderName, parentPath);
+      if (folderName && parentPath) {
         await createFolder({
           projectId: workspaceId,
           environment: slug,
-          path: secretPath,
+          path: parentPath,
           name: folderName
         });
       }
@@ -270,15 +271,16 @@ export const SecretOverviewPage = () => {
         <p className="text-md text-bunker-300">
           Inject your secrets using
           <a
-            className="ml-1 text-mineshaft-300 underline underline-offset-4 decoration-primary-800 hover:decoration-primary-600 hover:text-mineshaft-100 duration-200"
+            className="ml-1 text-mineshaft-300 underline decoration-primary-800 underline-offset-4 duration-200 hover:text-mineshaft-100 hover:decoration-primary-600"
             href="https://infisical.com/docs/cli/overview"
             target="_blank"
             rel="noopener noreferrer"
           >
             Infisical CLI
-          </a>,
+          </a>
+          ,
           <a
-            className="ml-1 text-mineshaft-300 underline underline-offset-4 decoration-primary-800 hover:decoration-primary-600 hover:text-mineshaft-100 duration-200"
+            className="ml-1 text-mineshaft-300 underline decoration-primary-800 underline-offset-4 duration-200 hover:text-mineshaft-100 hover:decoration-primary-600"
             href="https://infisical.com/docs/documentation/getting-started/api"
             target="_blank"
             rel="noopener noreferrer"
@@ -287,7 +289,7 @@ export const SecretOverviewPage = () => {
           </a>
           ,
           <a
-            className="ml-1 text-mineshaft-300 underline underline-offset-4 decoration-primary-800 hover:decoration-primary-600 hover:text-mineshaft-100 duration-200"
+            className="ml-1 text-mineshaft-300 underline decoration-primary-800 underline-offset-4 duration-200 hover:text-mineshaft-100 hover:decoration-primary-600"
             href="https://infisical.com/docs/sdks/overview"
             target="_blank"
             rel="noopener noreferrer"
@@ -296,13 +298,14 @@ export const SecretOverviewPage = () => {
           </a>
           , and
           <a
-            className="ml-1 text-mineshaft-300 underline underline-offset-4 decoration-primary-800 hover:decoration-primary-600 hover:text-mineshaft-100 duration-200"
+            className="ml-1 text-mineshaft-300 underline decoration-primary-800 underline-offset-4 duration-200 hover:text-mineshaft-100 hover:decoration-primary-600"
             href="https://infisical.com/docs/documentation/getting-started/introduction"
             target="_blank"
             rel="noopener noreferrer"
           >
             more
-          </a>.
+          </a>
+          .
         </p>
       </div>
       <div className="mt-8 flex items-center justify-between">
@@ -399,7 +402,8 @@ export const SecretOverviewPage = () => {
                   </Td>
                 </Tr>
               )}
-              {!isTableLoading && filteredFolderNames.map((folderName, index) => (
+              {!isTableLoading &&
+                filteredFolderNames.map((folderName, index) => (
                   <SecretOverviewFolderRow
                     folderName={folderName}
                     isFolderPresentInEnv={isFolderPresentInEnv}
@@ -408,7 +412,8 @@ export const SecretOverviewPage = () => {
                     onClick={handleFolderClick}
                   />
                 ))}
-              {!isTableLoading && (userAvailableEnvs?.length > 0 ? (
+              {!isTableLoading &&
+                (userAvailableEnvs?.length > 0 ? (
                   filteredSecretNames.map((key, index) => (
                     <SecretOverviewTableRow
                       secretPath={secretPath}

package-lock.json

@@ -7,6 +7,7 @@
 			"name": "infisical",
 			"license": "ISC",
 			"devDependencies": {
+				"@types/uuid": "^9.0.7",
 				"eslint": "^8.29.0",
 				"husky": "^8.0.3"
 			}
@@ -102,6 +103,12 @@
 				"node": ">= 8"
 			}
 		},
+		"node_modules/@types/uuid": {
+			"version": "9.0.7",
+			"resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-9.0.7.tgz",
+			"integrity": "sha512-WUtIVRUZ9i5dYXefDEAI7sh9/O7jGvHg7Df/5O/gtH3Yabe5odI3UWopVR1qbPXQtvOxWu3mM4XxlYeZtMWF4g==",
+			"dev": true
+		},
 		"node_modules/acorn": {
 			"version": "8.8.1",
 			"resolved": "https://registry.npmjs.org/acorn/-/acorn-8.8.1.tgz",
@@ -1202,6 +1209,12 @@
 				"fastq": "^1.6.0"
 			}
 		},
+		"@types/uuid": {
+			"version": "9.0.7",
+			"resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-9.0.7.tgz",
+			"integrity": "sha512-WUtIVRUZ9i5dYXefDEAI7sh9/O7jGvHg7Df/5O/gtH3Yabe5odI3UWopVR1qbPXQtvOxWu3mM4XxlYeZtMWF4g==",
+			"dev": true
+		},
 		"acorn": {
 			"version": "8.8.1",
 			"resolved": "https://registry.npmjs.org/acorn/-/acorn-8.8.1.tgz",

package.json

@@ -19,6 +19,7 @@
 		]
 	},
 	"devDependencies": {
+		"@types/uuid": "^9.0.7",
 		"eslint": "^8.29.0",
 		"husky": "^8.0.3"
 	}