From 6e6a1c87f2d9e1d018986cdd4f9aa8b42db9ef7c Mon Sep 17 00:00:00 2001 From: Akhil Mohan Date: Sun, 7 Jan 2024 20:41:23 +0530 Subject: [PATCH] feat(infisical-pg): completed secrets fetching from imports based on priority --- backend-pg/src/server/routes/index.ts | 3 +- .../server/routes/v1/secret-import-router.ts | 39 +++++++- .../secret-folder/secret-folder-dal.ts | 93 ++++++++++++++++++- .../secret-import/secret-import-service.ts | 64 ++++++++++++- .../secret-import/secret-import-types.ts | 5 + .../src/hooks/api/secretImports/queries.tsx | 4 +- frontend/src/hooks/api/secretImports/types.ts | 9 +- .../SecretImportListView.tsx | 8 +- 8 files changed, 208 insertions(+), 17 deletions(-) diff --git a/backend-pg/src/server/routes/index.ts b/backend-pg/src/server/routes/index.ts index e153c6664..0816d99fa 100644 --- a/backend-pg/src/server/routes/index.ts +++ b/backend-pg/src/server/routes/index.ts @@ -256,7 +256,8 @@ export const registerRoutes = async ( projectEnvDal, folderDal, permissionService, - secretImportDal + secretImportDal, + secretDal }); const projectBotService = projectBotServiceFactory({ permissionService, projectBotDal }); diff --git a/backend-pg/src/server/routes/v1/secret-import-router.ts b/backend-pg/src/server/routes/v1/secret-import-router.ts index e2fe1d323..90fe43434 100644 --- a/backend-pg/src/server/routes/v1/secret-import-router.ts +++ b/backend-pg/src/server/routes/v1/secret-import-router.ts @@ -1,6 +1,6 @@ import { z } from "zod"; -import { SecretImportsSchema } from "@app/db/schemas"; +import { SecretImportsSchema, SecretsSchema } from "@app/db/schemas"; import { verifyAuth } from "@app/server/plugins/auth/verify-auth"; import { AuthMode } from "@app/services/auth/auth-type"; @@ -149,4 +149,41 @@ export const registerSecretImportRouter = async (server: FastifyZodProvider) => return { message: "Successfully fetched secret imports", secretImports }; } }); + + server.route({ + url: "/secrets", + method: "GET", + schema: { + querystring: z.object({ + projectId: z.string().trim(), + environment: z.string().trim(), + path: z.string().trim().default("/") + }), + response: { + 200: z.object({ + secrets: z + .object({ + secretPath: z.string(), + environment: z.object({ + id: z.string(), + name: z.string(), + slug: z.string() + }), + folderId: z.string().optional(), + secrets: SecretsSchema.omit({ secretBlindIndex: true }).array() + }) + .array() + }) + } + }, + onRequest: verifyAuth([AuthMode.JWT, AuthMode.API_KEY]), + handler: async (req) => { + const importedSecrets = await server.services.secretImport.getSecretsFromImports({ + actorId: req.permission.id, + actor: req.permission.type, + ...req.query + }); + return { secrets: importedSecrets }; + } + }); }; diff --git a/backend-pg/src/services/secret-folder/secret-folder-dal.ts b/backend-pg/src/services/secret-folder/secret-folder-dal.ts index 289359f25..d48824789 100644 --- a/backend-pg/src/services/secret-folder/secret-folder-dal.ts +++ b/backend-pg/src/services/secret-folder/secret-folder-dal.ts @@ -10,6 +10,80 @@ export const validateFolderName = (folderName: string) => { return validNameRegex.test(folderName); }; +const sqlFindMultipleFolderByEnvPathQuery = ( + db: Knex, + query: Array<{ envId: string; secretPath: string }> +) => { + // this is removing an trailing slash like /folder1/folder2/ -> /folder1/folder2 + const formatedQuery = query.map(({ envId, secretPath }) => { + const formatedPath = + secretPath.at(-1) === "/" && secretPath.length > 1 ? secretPath.slice(0, -1) : secretPath; + const segments = formatedPath.split("/").filter(Boolean); + if (segments.some((segment) => !validateFolderName(segment))) { + throw new BadRequestError({ message: "Invalid folder name" }); + } + return { + envId, + secretPath: segments + }; + }); + // next goal to sanitize saw the raw sql query is safe + // for this we ensure folder name contains only string and - nothing else + + return db + .withRecursive("parent", (baseQb) => { + // first remember our folders are connected as a link list or known as adjacency list + // Thus each node has connection to parent node + // for a given path from root we recursively reach to the leaf path or till we get null + // the below query is the base case where we select root folder which has parent folder id as null + baseQb + .select({ + depth: 1, + // latestFolderVerId: db.raw("NULL::uuid"), + path: db.raw("'/'") + }) + .from(TableName.SecretFolder) + .where({ + parentId: null + }) + .whereIn( + "envId", + formatedQuery.map(({ envId }) => envId) + ) + .select(selectAllTableCols(TableName.SecretFolder)) + .union((qb) => + // for here on we keep going to next child node. + // we also keep a measure of depth then we check the depth matches the array path segment and folder name + // that is at depth 1 for a path /folder1/folder2 -> the name should be folder1 + qb + .select({ + depth: db.raw("parent.depth + 1"), + path: db.raw( + "CONCAT((CASE WHEN parent.path = '/' THEN '' ELSE parent.path END),'/', secret_folders.name)" + ) + }) + .select(selectAllTableCols(TableName.SecretFolder)) + .where((wb) => + formatedQuery.map(({ secretPath }) => + wb + .orWhereRaw( + `depth = array_position(ARRAY[${secretPath + .map(() => "?") + .join(",")}]::varchar[], ${TableName.SecretFolder}.name,depth)`, + [...secretPath] + ) + ) + ) + .from(TableName.SecretFolder) + .join("parent", (bd)=> + bd.on("parent.id", `${TableName.SecretFolder}.parentId`).andOn("parent.envId",`${TableName.SecretFolder}.envId`) + ) + ); + }) + .select("*") + .from("parent"); +}; + const sqlFindFolderByPathQuery = ( db: Knex, projectId: string, @@ -101,6 +175,23 @@ export const secretFolderDalFactory = (db: TDbClient) => { } }; + const findByManySecretPath = async ( + query: Array<{ envId: string; secretPath: string }>, + tx?: Knex + ) => { + try { + const folders = await sqlFindMultipleFolderByEnvPathQuery(tx || db, query); + return query.map(({ envId, secretPath }) => + folders.find( + ({ path: targetPath, envId: targetEnvId }) => + targetPath === secretPath && targetEnvId === envId + ) + ); + } catch (error) { + throw new DatabaseError({ error, name: "FindByManySecretPath" }); + } + }; + const update = async (filter: Partial, data: TSecretFoldersUpdate, tx?: Knex) => { try { const folder = await (tx || db)(TableName.SecretFolder) @@ -140,5 +231,5 @@ export const secretFolderDalFactory = (db: TDbClient) => { } }; - return { ...secretFolderOrm, update, findBySecretPath, findById }; + return { ...secretFolderOrm, update, findBySecretPath, findById, findByManySecretPath }; }; diff --git a/backend-pg/src/services/secret-import/secret-import-service.ts b/backend-pg/src/services/secret-import/secret-import-service.ts index 80267283a..11ada47ad 100644 --- a/backend-pg/src/services/secret-import/secret-import-service.ts +++ b/backend-pg/src/services/secret-import/secret-import-service.ts @@ -14,12 +14,16 @@ import { TCreateSecretImportDTO, TDeleteSecretImportDTO, TGetSecretImportsDTO, + TGetSecretsFromImportDTO, TUpdateSecretImportDTO } from "./secret-import-types"; +import { TSecretDalFactory } from "../secret/secret-dal"; +import { groupBy } from "@app/lib/fn"; type TSecretImportServiceFactoryDep = { secretImportDal: TSecretImportDalFactory; folderDal: TSecretFolderDalFactory; + secretDal: Pick; projectEnvDal: TProjectEnvDalFactory; permissionService: Pick; }; @@ -32,7 +36,8 @@ export const secretImportServiceFactory = ({ secretImportDal, projectEnvDal, permissionService, - folderDal + folderDal, + secretDal }: TSecretImportServiceFactoryDep) => { const createImport = async ({ environment, @@ -51,6 +56,7 @@ export const secretImportServiceFactory = ({ const folder = await folderDal.findBySecretPath(projectId, environment, path); if (!folder) throw new BadRequestError({ message: "Folder not found", name: "Create import" }); + // TODO(akhilmhdh-pg): updated permission check add here const [importEnv] = await projectEnvDal.findBySlugs(projectId, [data.environment]); if (!importEnv) throw new BadRequestError({ error: "Imported env not found", name: "Create import" }); @@ -168,10 +174,64 @@ export const secretImportServiceFactory = ({ const secImports = await secretImportDal.find({ folderId: folder.id }); return secImports; }; + + const getSecretsFromImports = async ({ + path, + environment, + projectId, + actor, + actorId + }: TGetSecretsFromImportDTO) => { + const { permission } = await permissionService.getProjectPermission(actor, actorId, projectId); + ForbiddenError.from(permission).throwUnlessCan( + ProjectPermissionActions.Read, + subject(ProjectPermissionSub.Secrets, { environment, secretPath: path }) + ); + const folder = await folderDal.findBySecretPath(projectId, environment, path); + if (!folder) throw new BadRequestError({ message: "Folder not found", name: "Get imports" }); + // this will already order by position + // so anything based on this order will also be in right position + const secretImports = await secretImportDal.find({ folderId: folder.id }); + + const allowedImports = secretImports.filter(({ importEnv, importPath }) => + permission.can( + ProjectPermissionActions.Read, + subject(ProjectPermissionSub.Secrets, { + environment: importEnv.slug, + secretPath: importPath + }) + ) + ); + const importedFolders = await folderDal.findByManySecretPath( + allowedImports.map(({ importEnv, importPath }) => ({ + envId: importEnv.id, + secretPath: importPath + })) + ); + const folderIds = importedFolders.map((el) => el?.id).filter(Boolean) as string[]; + if (!folderIds.length) { + return []; + } + const importedSecrets = await secretDal.find({ + $in: { folderId: folderIds } + }); + + const importedSecsGroupByFolderId = groupBy(importedSecrets, (i) => i.folderId); + return allowedImports.map(({ importPath, importEnv }, i) => ({ + secretPath: importPath, + environment: importEnv, + folderId: importedFolders?.[i]?.id, + secrets: importedFolders?.[i]?.id + ? importedSecsGroupByFolderId[importedFolders?.[i]?.id as string] + : [] + })); + }; + return { createImport, updateImport, deleteImport, - getImports + getImports, + getSecretsFromImports }; }; diff --git a/backend-pg/src/services/secret-import/secret-import-types.ts b/backend-pg/src/services/secret-import/secret-import-types.ts index c04f1c994..d123f28da 100644 --- a/backend-pg/src/services/secret-import/secret-import-types.ts +++ b/backend-pg/src/services/secret-import/secret-import-types.ts @@ -26,3 +26,8 @@ export type TGetSecretImportsDTO = { environment: string; path: string; } & TProjectPermission; + +export type TGetSecretsFromImportDTO = { + environment: string; + path: string; +} & TProjectPermission; diff --git a/frontend/src/hooks/api/secretImports/queries.tsx b/frontend/src/hooks/api/secretImports/queries.tsx index 108d012a7..f64fbd47d 100644 --- a/frontend/src/hooks/api/secretImports/queries.tsx +++ b/frontend/src/hooks/api/secretImports/queries.tsx @@ -66,9 +66,9 @@ const fetchImportedSecrets = async ( "/api/v1/secret-imports/secrets", { params: { - workspaceId, + projectId: workspaceId, environment, - directory + path: directory } } ); diff --git a/frontend/src/hooks/api/secretImports/types.ts b/frontend/src/hooks/api/secretImports/types.ts index b157b880f..68f06748c 100644 --- a/frontend/src/hooks/api/secretImports/types.ts +++ b/frontend/src/hooks/api/secretImports/types.ts @@ -1,22 +1,19 @@ import { UserWsKeyPair } from "../keys/types"; import { EncryptedSecret } from "../secrets/types"; +import { WorkspaceEnv } from "../workspace/types"; export type TSecretImport = { id: string; folderId: string; importPath: string; - importEnv: { - name: string; - slug: string; - id: string; - }; + importEnv: WorkspaceEnv; position: string; createdAt: string; updatedAt: string; }; export type TImportedSecrets = { - environment: string; + environment: WorkspaceEnv; secretPath: string; folderId: string; secrets: EncryptedSecret[]; diff --git a/frontend/src/views/SecretMainPage/components/SecretImportListView/SecretImportListView.tsx b/frontend/src/views/SecretMainPage/components/SecretImportListView/SecretImportListView.tsx index 136de9406..be8483fa5 100644 --- a/frontend/src/views/SecretMainPage/components/SecretImportListView/SecretImportListView.tsx +++ b/frontend/src/views/SecretMainPage/components/SecretImportListView/SecretImportListView.tsx @@ -18,14 +18,14 @@ import { useWorkspace } from "@app/context"; import { usePopUp } from "@app/hooks"; import { useDeleteSecretImport, useUpdateSecretImport } from "@app/hooks/api"; import { TSecretImport } from "@app/hooks/api/secretImports/types"; -import { DecryptedSecret } from "@app/hooks/api/types"; +import { DecryptedSecret, WorkspaceEnv } from "@app/hooks/api/types"; import { SecretImportItem } from "./SecretImportItem"; const SECRET_IN_DASHBOARD = "Present In Dashboard"; type TImportedSecrets = Array<{ - environment: string; + environment: WorkspaceEnv; secretPath: string; folderId: string; secrets: DecryptedSecret[]; @@ -40,7 +40,7 @@ export const computeImportedSecretRows = ( ) => { const importedSecIndex = importSecrets.findIndex( ({ secretPath, environment }) => - secretPath === importedSecPath && importedSecEnv === environment + secretPath === importedSecPath && importedSecEnv === environment.slug ); if (importedSecIndex === -1) return []; @@ -55,7 +55,7 @@ export const computeImportedSecretRows = ( for (let i = importedSecIndex + 1; i < importSecrets.length; i += 1) { importSecrets[i].secrets.forEach((el) => { overridenSec[el.key] = { - env: envSlug2Name?.[importSecrets[i].environment] || "unknown", + env: envSlug2Name?.[importSecrets[i].environment.slug] || "unknown", secretPath: importSecrets[i].secretPath }; });