feat(infisical-pg): completed secrets fetching from imports based on priority

backend-pg/src/server/routes/index.ts

@@ -256,7 +256,8 @@ export const registerRoutes = async (
     projectEnvDal,
     folderDal,
     permissionService,
-    secretImportDal
+    secretImportDal,
+    secretDal
   });
   const projectBotService = projectBotServiceFactory({ permissionService, projectBotDal });
 

backend-pg/src/server/routes/v1/secret-import-router.ts

@@ -1,6 +1,6 @@
 import { z } from "zod";
 
-import { SecretImportsSchema } from "@app/db/schemas";
+import { SecretImportsSchema, SecretsSchema } from "@app/db/schemas";
 import { verifyAuth } from "@app/server/plugins/auth/verify-auth";
 import { AuthMode } from "@app/services/auth/auth-type";
 
@@ -149,4 +149,41 @@ export const registerSecretImportRouter = async (server: FastifyZodProvider) =>
       return { message: "Successfully fetched secret imports", secretImports };
     }
   });
+
+  server.route({
+    url: "/secrets",
+    method: "GET",
+    schema: {
+      querystring: z.object({
+        projectId: z.string().trim(),
+        environment: z.string().trim(),
+        path: z.string().trim().default("/")
+      }),
+      response: {
+        200: z.object({
+          secrets: z
+            .object({
+              secretPath: z.string(),
+              environment: z.object({
+                id: z.string(),
+                name: z.string(),
+                slug: z.string()
+              }),
+              folderId: z.string().optional(),
+              secrets: SecretsSchema.omit({ secretBlindIndex: true }).array()
+            })
+            .array()
+        })
+      }
+    },
+    onRequest: verifyAuth([AuthMode.JWT, AuthMode.API_KEY]),
+    handler: async (req) => {
+      const importedSecrets = await server.services.secretImport.getSecretsFromImports({
+        actorId: req.permission.id,
+        actor: req.permission.type,
+        ...req.query
+      });
+      return { secrets: importedSecrets };
+    }
+  });
 };

backend-pg/src/services/secret-folder/secret-folder-dal.ts

@@ -10,6 +10,80 @@ export const validateFolderName = (folderName: string) => {
   return validNameRegex.test(folderName);
 };
 
+const sqlFindMultipleFolderByEnvPathQuery = (
+  db: Knex,
+  query: Array<{ envId: string; secretPath: string }>
+) => {
+  // this is removing an trailing slash like /folder1/folder2/ -> /folder1/folder2
+  const formatedQuery = query.map(({ envId, secretPath }) => {
+    const formatedPath =
+      secretPath.at(-1) === "/" && secretPath.length > 1 ? secretPath.slice(0, -1) : secretPath;
+    const segments = formatedPath.split("/").filter(Boolean);
+    if (segments.some((segment) => !validateFolderName(segment))) {
+      throw new BadRequestError({ message: "Invalid folder name" });
+    }
+    return {
+      envId,
+      secretPath: segments
+    };
+  });
+  // next goal to sanitize saw the raw sql query is safe
+  // for this we ensure folder name contains only string and - nothing else
+
+  return db
+    .withRecursive("parent", (baseQb) => {
+      // first remember our folders are connected as a link list or known as adjacency list
+      // Thus each node has connection to parent node
+      // for a given path from root we recursively reach to the leaf path or till we get null
+      // the below query is the base case where we select root folder which has parent folder id as null
+      baseQb
+        .select({
+          depth: 1,
+          // latestFolderVerId: db.raw("NULL::uuid"),
+          path: db.raw("'/'")
+        })
+        .from(TableName.SecretFolder)
+        .where({
+          parentId: null
+        })
+        .whereIn(
+          "envId",
+          formatedQuery.map(({ envId }) => envId)
+        )
+        .select(selectAllTableCols(TableName.SecretFolder))
+        .union((qb) =>
+          // for here on we keep going to next child node.
+          // we also keep a measure of depth then we check the depth matches the array path segment and folder name
+          // that is at depth 1 for a path /folder1/folder2 -> the name should be folder1
+          qb
+            .select({
+              depth: db.raw("parent.depth + 1"),
+              path: db.raw(
+                "CONCAT((CASE WHEN parent.path = '/' THEN '' ELSE parent.path END),'/', secret_folders.name)"
+              )
+            })
+            .select(selectAllTableCols(TableName.SecretFolder))
+            .where((wb) =>
+              formatedQuery.map(({ secretPath }) =>
+                wb
+                  .orWhereRaw(
+                    `depth = array_position(ARRAY[${secretPath
+                      .map(() => "?")
+                      .join(",")}]::varchar[], ${TableName.SecretFolder}.name,depth)`,
+                    [...secretPath]
+                  )
+              )
+            )
+            .from(TableName.SecretFolder)
+            .join("parent", (bd)=>
+            bd.on("parent.id", `${TableName.SecretFolder}.parentId`).andOn("parent.envId",`${TableName.SecretFolder}.envId`)
+          )
+        );
+    })
+    .select("*")
+    .from<TSecretFolders & { depth: number; path: string }>("parent");
+};
+
 const sqlFindFolderByPathQuery = (
   db: Knex,
   projectId: string,
@@ -101,6 +175,23 @@ export const secretFolderDalFactory = (db: TDbClient) => {
     }
   };
 
+  const findByManySecretPath = async (
+    query: Array<{ envId: string; secretPath: string }>,
+    tx?: Knex
+  ) => {
+    try {
+      const folders = await sqlFindMultipleFolderByEnvPathQuery(tx || db, query);
+      return query.map(({ envId, secretPath }) =>
+        folders.find(
+          ({ path: targetPath, envId: targetEnvId }) =>
+            targetPath === secretPath && targetEnvId === envId
+        )
+      );
+    } catch (error) {
+      throw new DatabaseError({ error, name: "FindByManySecretPath" });
+    }
+  };
+
   const update = async (filter: Partial<TSecretFolders>, data: TSecretFoldersUpdate, tx?: Knex) => {
     try {
       const folder = await (tx || db)(TableName.SecretFolder)
@@ -140,5 +231,5 @@ export const secretFolderDalFactory = (db: TDbClient) => {
     }
   };
 
-  return { ...secretFolderOrm, update, findBySecretPath, findById };
+  return { ...secretFolderOrm, update, findBySecretPath, findById, findByManySecretPath };
 };

backend-pg/src/services/secret-import/secret-import-service.ts

@@ -14,12 +14,16 @@ import {
   TCreateSecretImportDTO,
   TDeleteSecretImportDTO,
   TGetSecretImportsDTO,
+  TGetSecretsFromImportDTO,
   TUpdateSecretImportDTO
 } from "./secret-import-types";
+import { TSecretDalFactory } from "../secret/secret-dal";
+import { groupBy } from "@app/lib/fn";
 
 type TSecretImportServiceFactoryDep = {
   secretImportDal: TSecretImportDalFactory;
   folderDal: TSecretFolderDalFactory;
+  secretDal: Pick<TSecretDalFactory, "find">;
   projectEnvDal: TProjectEnvDalFactory;
   permissionService: Pick<TPermissionServiceFactory, "getProjectPermission">;
 };
@@ -32,7 +36,8 @@ export const secretImportServiceFactory = ({
   secretImportDal,
   projectEnvDal,
   permissionService,
-  folderDal
+  folderDal,
+  secretDal
 }: TSecretImportServiceFactoryDep) => {
   const createImport = async ({
     environment,
@@ -51,6 +56,7 @@ export const secretImportServiceFactory = ({
     const folder = await folderDal.findBySecretPath(projectId, environment, path);
     if (!folder) throw new BadRequestError({ message: "Folder not found", name: "Create import" });
 
+    // TODO(akhilmhdh-pg): updated permission check add here
     const [importEnv] = await projectEnvDal.findBySlugs(projectId, [data.environment]);
     if (!importEnv)
       throw new BadRequestError({ error: "Imported env not found", name: "Create import" });
@@ -168,10 +174,64 @@ export const secretImportServiceFactory = ({
     const secImports = await secretImportDal.find({ folderId: folder.id });
     return secImports;
   };
+
+  const getSecretsFromImports = async ({
+    path,
+    environment,
+    projectId,
+    actor,
+    actorId
+  }: TGetSecretsFromImportDTO) => {
+    const { permission } = await permissionService.getProjectPermission(actor, actorId, projectId);
+    ForbiddenError.from(permission).throwUnlessCan(
+      ProjectPermissionActions.Read,
+      subject(ProjectPermissionSub.Secrets, { environment, secretPath: path })
+    );
+    const folder = await folderDal.findBySecretPath(projectId, environment, path);
+    if (!folder) throw new BadRequestError({ message: "Folder not found", name: "Get imports" });
+    // this will already order by position
+    // so anything based on this order will also be in right position
+    const secretImports = await secretImportDal.find({ folderId: folder.id });
+
+    const allowedImports = secretImports.filter(({ importEnv, importPath }) =>
+      permission.can(
+        ProjectPermissionActions.Read,
+        subject(ProjectPermissionSub.Secrets, {
+          environment: importEnv.slug,
+          secretPath: importPath
+        })
+      )
+    );
+    const importedFolders = await folderDal.findByManySecretPath(
+      allowedImports.map(({ importEnv, importPath }) => ({
+        envId: importEnv.id,
+        secretPath: importPath
+      }))
+    );
+    const folderIds = importedFolders.map((el) => el?.id).filter(Boolean) as string[];
+    if (!folderIds.length) {
+      return [];
+    }
+    const importedSecrets = await secretDal.find({
+      $in: { folderId: folderIds }
+    });
+
+    const importedSecsGroupByFolderId = groupBy(importedSecrets, (i) => i.folderId);
+    return allowedImports.map(({ importPath, importEnv }, i) => ({
+      secretPath: importPath,
+      environment: importEnv,
+      folderId: importedFolders?.[i]?.id,
+      secrets: importedFolders?.[i]?.id
+        ? importedSecsGroupByFolderId[importedFolders?.[i]?.id as string]
+        : []
+    }));
+  };
+
   return {
     createImport,
     updateImport,
     deleteImport,
-    getImports
+    getImports,
+    getSecretsFromImports
   };
 };

backend-pg/src/services/secret-import/secret-import-types.ts

@@ -26,3 +26,8 @@ export type TGetSecretImportsDTO = {
   environment: string;
   path: string;
 } & TProjectPermission;
+
+export type TGetSecretsFromImportDTO = {
+  environment: string;
+  path: string;
+} & TProjectPermission;

frontend/src/hooks/api/secretImports/queries.tsx

@@ -66,9 +66,9 @@ const fetchImportedSecrets = async (
     "/api/v1/secret-imports/secrets",
     {
       params: {
-        workspaceId,
+        projectId: workspaceId,
         environment,
-        directory
+        path: directory
       }
     }
   );

frontend/src/hooks/api/secretImports/types.ts

@@ -1,22 +1,19 @@
 import { UserWsKeyPair } from "../keys/types";
 import { EncryptedSecret } from "../secrets/types";
+import { WorkspaceEnv } from "../workspace/types";
 
 export type TSecretImport = {
   id: string;
   folderId: string;
   importPath: string;
-  importEnv: {
+  importEnv: WorkspaceEnv;
-    name: string;
-    slug: string;
-    id: string;
-  };
   position: string;
   createdAt: string;
   updatedAt: string;
 };
 
 export type TImportedSecrets = {
-  environment: string;
+  environment: WorkspaceEnv;
   secretPath: string;
   folderId: string;
   secrets: EncryptedSecret[];

frontend/src/views/SecretMainPage/components/SecretImportListView/SecretImportListView.tsx

@@ -18,14 +18,14 @@ import { useWorkspace } from "@app/context";
 import { usePopUp } from "@app/hooks";
 import { useDeleteSecretImport, useUpdateSecretImport } from "@app/hooks/api";
 import { TSecretImport } from "@app/hooks/api/secretImports/types";
-import { DecryptedSecret } from "@app/hooks/api/types";
+import { DecryptedSecret, WorkspaceEnv } from "@app/hooks/api/types";
 
 import { SecretImportItem } from "./SecretImportItem";
 
 const SECRET_IN_DASHBOARD = "Present In Dashboard";
 
 type TImportedSecrets = Array<{
-  environment: string;
+  environment: WorkspaceEnv;
   secretPath: string;
   folderId: string;
   secrets: DecryptedSecret[];
@@ -40,7 +40,7 @@ export const computeImportedSecretRows = (
 ) => {
   const importedSecIndex = importSecrets.findIndex(
     ({ secretPath, environment }) =>
-      secretPath === importedSecPath && importedSecEnv === environment
+      secretPath === importedSecPath && importedSecEnv === environment.slug
   );
   if (importedSecIndex === -1) return [];
 
@@ -55,7 +55,7 @@ export const computeImportedSecretRows = (
   for (let i = importedSecIndex + 1; i < importSecrets.length; i += 1) {
     importSecrets[i].secrets.forEach((el) => {
       overridenSec[el.key] = {
-        env: envSlug2Name?.[importSecrets[i].environment] || "unknown",
+        env: envSlug2Name?.[importSecrets[i].environment.slug] || "unknown",
         secretPath: importSecrets[i].secretPath
       };
     });