From 4031f4a559261c47f7acf59986f0091fb5551486 Mon Sep 17 00:00:00 2001 From: Akhil Mohan Date: Thu, 18 Jan 2024 20:17:20 +0530 Subject: [PATCH] feat(infisical-pg): idempotent folder creation --- backend-pg/package-lock.json | 1 + backend-pg/package.json | 1 + .../secret-folder/secret-folder-dal.ts | 24 ++++- .../secret-folder/secret-folder-service.ts | 91 ++++++++++++++++--- .../secret-import/secret-import-service.ts | 9 +- .../src/services/secret/secret-service.ts | 2 +- .../SecretOverviewPage/SecretOverviewPage.tsx | 79 ++++++++-------- package-lock.json | 13 +++ package.json | 1 + 9 files changed, 166 insertions(+), 55 deletions(-) diff --git a/backend-pg/package-lock.json b/backend-pg/package-lock.json index a282ac60e..d105e8109 100644 --- a/backend-pg/package-lock.json +++ b/backend-pg/package-lock.json @@ -58,6 +58,7 @@ "smee-client": "^2.0.0", "tweetnacl": "^1.0.3", "tweetnacl-util": "^0.15.1", + "uuid": "^9.0.1", "zod": "^3.22.4", "zod-to-json-schema": "^3.22.0" }, diff --git a/backend-pg/package.json b/backend-pg/package.json index db4d2ca2a..506d632cd 100644 --- a/backend-pg/package.json +++ b/backend-pg/package.json @@ -115,6 +115,7 @@ "smee-client": "^2.0.0", "tweetnacl": "^1.0.3", "tweetnacl-util": "^0.15.1", + "uuid": "^9.0.1", "zod": "^3.22.4", "zod-to-json-schema": "^3.22.0" } diff --git a/backend-pg/src/services/secret-folder/secret-folder-dal.ts b/backend-pg/src/services/secret-folder/secret-folder-dal.ts index b11a57cc2..06ef7cfb6 100644 --- a/backend-pg/src/services/secret-folder/secret-folder-dal.ts +++ b/backend-pg/src/services/secret-folder/secret-folder-dal.ts @@ -252,6 +252,27 @@ export const secretFolderDalFactory = (db: TDbClient) => { } }; + // used in folder creation + // even if its the original given /path1/path2 + // it will stop automatically at /path2 + const findClosestFolder = async ( + projectId: string, + environment: string, + path: string, + tx?: Knex + ) => { + try { + const folder = await sqlFindFolderByPathQuery(tx || db, projectId, environment, path) + .orderBy("depth", "desc") + .first(); + if (!folder) return; + const { envId: id, envName: name, envSlug: slug, ...el } = folder; + return { ...el, envId: id, environment: { id, name, slug } }; + } catch (error) { + throw new DatabaseError({ error, name: "Find by secret path" }); + } + }; + const findByManySecretPath = async ( query: Array<{ envId: string; secretPath: string }>, tx?: Knex @@ -329,6 +350,7 @@ export const secretFolderDalFactory = (db: TDbClient) => { findBySecretPath, findById, findByManySecretPath, - findSecretPathByFolderIds + findSecretPathByFolderIds, + findClosestFolder }; }; diff --git a/backend-pg/src/services/secret-folder/secret-folder-service.ts b/backend-pg/src/services/secret-folder/secret-folder-service.ts index 080a5303f..55432d480 100644 --- a/backend-pg/src/services/secret-folder/secret-folder-service.ts +++ b/backend-pg/src/services/secret-folder/secret-folder-service.ts @@ -1,5 +1,8 @@ import { ForbiddenError, subject } from "@casl/ability"; +import path from "path"; +import { v4 as uuidv4 } from "uuid"; +import { TSecretFoldersInsert } from "@app/db/schemas"; import { TPermissionServiceFactory } from "@app/ee/services/permission/permission-service"; import { ProjectPermissionActions, @@ -41,12 +44,12 @@ export const secretFolderServiceFactory = ({ actorId, name, environment, - path + path: secretPath }: TCreateFolderDTO) => { const { permission } = await permissionService.getProjectPermission(actor, actorId, projectId); ForbiddenError.from(permission).throwUnlessCan( ProjectPermissionActions.Create, - subject(ProjectPermissionSub.Secrets, { environment, secretPath: path }) + subject(ProjectPermissionSub.Secrets, { environment, secretPath }) ); const env = await projectEnvDal.findOne({ projectId, slug: environment }); @@ -54,10 +57,66 @@ export const secretFolderServiceFactory = ({ throw new BadRequestError({ message: "Environment not found", name: "Create folder" }); const folder = await folderDal.transaction(async (tx) => { - const parentFolder = await folderDal.findBySecretPath(projectId, environment, path, tx); + // the logic is simple we need to avoid creating same folder in same path multiple times + // that is this request must be idempotent + // so we do a tricky move. we try to find the to be created folder path if that is exactly match return that + // else we get some path before that then we will start creating remaining folder + const pathWithFolder = path.join(secretPath, name); + const parentFolder = await folderDal.findClosestFolder( + projectId, + environment, + pathWithFolder, + tx + ); + // no folder found is not possible root should be their if (!parentFolder) throw new BadRequestError({ message: "Secret path not found" }); + // exact folder + if (parentFolder.path === pathWithFolder) return parentFolder; + + let parentFolderId = parentFolder.id; + if (parentFolder.path !== secretPath) { + // this is upsert folder in a path + // we are not taking snapshots of this because + // snapshot will be removed from automatic for all commits to user click or cron based + const missingSegment = secretPath + .substring(parentFolder.path.length) + .split("/") + .filter(Boolean); + if (missingSegment.length) { + const newFolders: Array = missingSegment.map( + (segment, i) => + i === 0 + ? { + name: segment, + parentId: parentFolder.id, + id: uuidv4(), + envId: env.id, + version: 1 + } + : { + name: segment, + parentId: newFolders[i - 1].id, + id: uuidv4(), + envId: env.id, + version: 1 + } + ); + parentFolderId = newFolders.at(-1)?.id as string; + const docs = await folderDal.insertMany(newFolders, tx); + await folderVersionDal.insertMany( + docs.map((doc) => ({ + name: doc.name, + envId: doc.envId, + version: doc.version, + folderId: doc.id + })), + tx + ); + } + } + const doc = await folderDal.create( - { name, envId: env.id, version: 1, parentId: parentFolder.id }, + { name, envId: env.id, version: 1, parentId: parentFolderId }, tx ); await folderVersionDal.create( @@ -82,16 +141,16 @@ export const secretFolderServiceFactory = ({ actorId, name, environment, - path, + path: secretPath, id }: TUpdateFolderDTO) => { const { permission } = await permissionService.getProjectPermission(actor, actorId, projectId); ForbiddenError.from(permission).throwUnlessCan( ProjectPermissionActions.Edit, - subject(ProjectPermissionSub.Secrets, { environment, secretPath: path }) + subject(ProjectPermissionSub.Secrets, { environment, secretPath }) ); - const parentFolder = await folderDal.findBySecretPath(projectId, environment, path); + const parentFolder = await folderDal.findBySecretPath(projectId, environment, secretPath); if (!parentFolder) throw new BadRequestError({ message: "Secret path not found" }); const env = await projectEnvDal.findOne({ projectId, slug: environment }); @@ -133,13 +192,13 @@ export const secretFolderServiceFactory = ({ actor, actorId, environment, - path, + path: secretPath, id }: TDeleteFolderDTO) => { const { permission } = await permissionService.getProjectPermission(actor, actorId, projectId); ForbiddenError.from(permission).throwUnlessCan( ProjectPermissionActions.Delete, - subject(ProjectPermissionSub.Secrets, { environment, secretPath: path }) + subject(ProjectPermissionSub.Secrets, { environment, secretPath }) ); const env = await projectEnvDal.findOne({ projectId, slug: environment }); @@ -147,7 +206,7 @@ export const secretFolderServiceFactory = ({ throw new BadRequestError({ message: "Environment not found", name: "Create folder" }); const folder = await folderDal.transaction(async (tx) => { - const parentFolder = await folderDal.findBySecretPath(projectId, environment, path, tx); + const parentFolder = await folderDal.findBySecretPath(projectId, environment, secretPath, tx); if (!parentFolder) throw new BadRequestError({ message: "Secret path not found" }); const [doc] = await folderDal.delete({ envId: env.id, id, parentId: parentFolder.id }, tx); @@ -159,7 +218,13 @@ export const secretFolderServiceFactory = ({ return folder; }; - const getFolders = async ({ projectId, actor, actorId, environment, path }: TGetFolderDTO) => { + const getFolders = async ({ + projectId, + actor, + actorId, + environment, + path: secretPath + }: TGetFolderDTO) => { // folder list is allowed to be read by anyone // permission to check does user has access await permissionService.getProjectPermission(actor, actorId, projectId); @@ -167,8 +232,8 @@ export const secretFolderServiceFactory = ({ const env = await projectEnvDal.findOne({ projectId, slug: environment }); if (!env) throw new BadRequestError({ message: "Environment not found", name: "get folders" }); - const parentFolder = await folderDal.findBySecretPath(projectId, environment, path); - if (!parentFolder) throw new BadRequestError({ message: "Secret path not found" }); + const parentFolder = await folderDal.findBySecretPath(projectId, environment, secretPath); + if (!parentFolder) return []; const folders = await folderDal.find({ envId: env.id, parentId: parentFolder.id }); return folders; diff --git a/backend-pg/src/services/secret-import/secret-import-service.ts b/backend-pg/src/services/secret-import/secret-import-service.ts index 9021fc744..e63ffd6a8 100644 --- a/backend-pg/src/services/secret-import/secret-import-service.ts +++ b/backend-pg/src/services/secret-import/secret-import-service.ts @@ -48,7 +48,7 @@ export const secretImportServiceFactory = ({ path }: TCreateSecretImportDTO) => { const { permission } = await permissionService.getProjectPermission(actor, actorId, projectId); - + // check if user has permission to import into destination path ForbiddenError.from(permission).throwUnlessCan( ProjectPermissionActions.Create, @@ -58,7 +58,10 @@ export const secretImportServiceFactory = ({ // check if user has permission to import from target path ForbiddenError.from(permission).throwUnlessCan( ProjectPermissionActions.Create, - subject(ProjectPermissionSub.Secrets, { environment: data.environment, secretPath: data.path }) + subject(ProjectPermissionSub.Secrets, { + environment: data.environment, + secretPath: data.path + }) ); const folder = await folderDal.findBySecretPath(projectId, environment, path); @@ -196,7 +199,7 @@ export const secretImportServiceFactory = ({ subject(ProjectPermissionSub.Secrets, { environment, secretPath: path }) ); const folder = await folderDal.findBySecretPath(projectId, environment, path); - if (!folder) throw new BadRequestError({ message: "Folder not found", name: "Get imports" }); + if (!folder) return []; // this will already order by position // so anything based on this order will also be in right position const secretImports = await secretImportDal.find({ folderId: folder.id }); diff --git a/backend-pg/src/services/secret/secret-service.ts b/backend-pg/src/services/secret/secret-service.ts index 2f539a8dd..a4113d979 100644 --- a/backend-pg/src/services/secret/secret-service.ts +++ b/backend-pg/src/services/secret/secret-service.ts @@ -499,7 +499,7 @@ export const secretServiceFactory = ({ ); const folder = await folderDal.findBySecretPath(projectId, environment, path); - if (!folder) throw new BadRequestError({ message: "Folder not found", name: "Create secret" }); + if (!folder) return { secrets: [], imports: [] }; const folderId = folder.id; const secrets = await secretDal.findByFolderId(folderId, actorId); diff --git a/frontend/src/views/SecretOverviewPage/SecretOverviewPage.tsx b/frontend/src/views/SecretOverviewPage/SecretOverviewPage.tsx index 68a6a0b70..8e013c11f 100644 --- a/frontend/src/views/SecretOverviewPage/SecretOverviewPage.tsx +++ b/frontend/src/views/SecretOverviewPage/SecretOverviewPage.tsx @@ -120,7 +120,7 @@ export const SecretOverviewPage = () => { if (folderName && parentPath) { await createFolder({ projectId: workspaceId, - path: secretPath, + path: parentPath, environment: env, name: folderName }); @@ -211,14 +211,15 @@ export const SecretOverviewPage = () => { const handleExploreEnvClick = async (slug: string) => { if (secretPath !== "/") { - const path = secretPath.split("/"); - const directory = path.slice(0, -1).join("/"); - const folderName = path.at(-1); - if (folderName && directory) { + const pathSegment = secretPath.split("/").filter(Boolean); + const parentPath = `/${pathSegment.slice(0, -1).join("/")}`; + const folderName = pathSegment.at(-1); + console.log(folderName, parentPath); + if (folderName && parentPath) { await createFolder({ projectId: workspaceId, environment: slug, - path: secretPath, + path: parentPath, name: folderName }); } @@ -270,15 +271,16 @@ export const SecretOverviewPage = () => {

Inject your secrets using Infisical CLI - , + + , { , Infisical SDKs - , and + , and more - . + + .

@@ -399,32 +402,34 @@ export const SecretOverviewPage = () => { )} - {!isTableLoading && filteredFolderNames.map((folderName, index) => ( - - ))} - {!isTableLoading && (userAvailableEnvs?.length > 0 ? ( - filteredSecretNames.map((key, index) => ( - ( + - )) - ) : ( - - ))} + ))} + {!isTableLoading && + (userAvailableEnvs?.length > 0 ? ( + filteredSecretNames.map((key, index) => ( + + )) + ) : ( + + ))} diff --git a/package-lock.json b/package-lock.json index 8db673690..e27466d9e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -7,6 +7,7 @@ "name": "infisical", "license": "ISC", "devDependencies": { + "@types/uuid": "^9.0.7", "eslint": "^8.29.0", "husky": "^8.0.3" } @@ -102,6 +103,12 @@ "node": ">= 8" } }, + "node_modules/@types/uuid": { + "version": "9.0.7", + "resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-9.0.7.tgz", + "integrity": "sha512-WUtIVRUZ9i5dYXefDEAI7sh9/O7jGvHg7Df/5O/gtH3Yabe5odI3UWopVR1qbPXQtvOxWu3mM4XxlYeZtMWF4g==", + "dev": true + }, "node_modules/acorn": { "version": "8.8.1", "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.8.1.tgz", @@ -1202,6 +1209,12 @@ "fastq": "^1.6.0" } }, + "@types/uuid": { + "version": "9.0.7", + "resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-9.0.7.tgz", + "integrity": "sha512-WUtIVRUZ9i5dYXefDEAI7sh9/O7jGvHg7Df/5O/gtH3Yabe5odI3UWopVR1qbPXQtvOxWu3mM4XxlYeZtMWF4g==", + "dev": true + }, "acorn": { "version": "8.8.1", "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.8.1.tgz", diff --git a/package.json b/package.json index 89026e2c6..11d595549 100644 --- a/package.json +++ b/package.json @@ -19,6 +19,7 @@ ] }, "devDependencies": { + "@types/uuid": "^9.0.7", "eslint": "^8.29.0", "husky": "^8.0.3" }