Merge pull request #3264 from Infisical/feat/addSecretImportsToOverview

Show when folders have imports in secret overview page

backend/src/lib/api-docs/constants.ts

@@ -815,7 +815,8 @@ export const DASHBOARD = {
     search: "The text string to filter secret keys and folder names by.",
     includeSecrets: "Whether to include project secrets in the response.",
     includeFolders: "Whether to include project folders in the response.",
-    includeDynamicSecrets: "Whether to include dynamic project secrets in the response."
+    includeDynamicSecrets: "Whether to include dynamic project secrets in the response.",
+    includeImports: "Whether to include project secret imports in the response."
   },
   SECRET_DETAILS_LIST: {
     projectId: "The ID of the project to list secrets/folders from.",

backend/src/server/routes/v1/dashboard-router.ts

@@ -109,6 +109,7 @@ export const registerDashboardRouter = async (server: FastifyZodProvider) => {
         search: z.string().trim().describe(DASHBOARD.SECRET_OVERVIEW_LIST.search).optional(),
         includeSecrets: booleanSchema.describe(DASHBOARD.SECRET_OVERVIEW_LIST.includeSecrets),
         includeFolders: booleanSchema.describe(DASHBOARD.SECRET_OVERVIEW_LIST.includeFolders),
+        includeImports: booleanSchema.describe(DASHBOARD.SECRET_OVERVIEW_LIST.includeImports),
         includeDynamicSecrets: booleanSchema.describe(DASHBOARD.SECRET_OVERVIEW_LIST.includeDynamicSecrets)
       }),
       response: {
@@ -124,9 +125,17 @@ export const registerDashboardRouter = async (server: FastifyZodProvider) => {
             })
             .array()
             .optional(),
+          imports: SecretImportsSchema.omit({ importEnv: true })
+            .extend({
+              importEnv: z.object({ name: z.string(), slug: z.string(), id: z.string() }),
+              environment: z.string()
+            })
+            .array()
+            .optional(),
           totalFolderCount: z.number().optional(),
           totalDynamicSecretCount: z.number().optional(),
           totalSecretCount: z.number().optional(),
+          totalImportCount: z.number().optional(),
           totalCount: z.number()
         })
       }
@@ -143,6 +152,7 @@ export const registerDashboardRouter = async (server: FastifyZodProvider) => {
         orderDirection,
         includeFolders,
         includeSecrets,
+        includeImports,
         includeDynamicSecrets
       } = req.query;
 
@@ -159,6 +169,7 @@ export const registerDashboardRouter = async (server: FastifyZodProvider) => {
       let remainingLimit = limit;
       let adjustedOffset = offset;
 
+      let imports: Awaited<ReturnType<typeof server.services.secretImport.getImportsMultiEnv>> | undefined;
       let folders: Awaited<ReturnType<typeof server.services.folder.getFoldersMultiEnv>> | undefined;
       let secrets: Awaited<ReturnType<typeof server.services.secret.getSecretsRawMultiEnv>> | undefined;
       let dynamicSecrets:
@@ -168,6 +179,53 @@ export const registerDashboardRouter = async (server: FastifyZodProvider) => {
       let totalFolderCount: number | undefined;
       let totalDynamicSecretCount: number | undefined;
       let totalSecretCount: number | undefined;
+      let totalImportCount: number | undefined;
+
+      if (includeImports) {
+        totalImportCount = await server.services.secretImport.getProjectImportMultiEnvCount({
+          actorId: req.permission.id,
+          actor: req.permission.type,
+          actorAuthMethod: req.permission.authMethod,
+          actorOrgId: req.permission.orgId,
+          projectId,
+          environments,
+          path: secretPath,
+          search
+        });
+
+        if (remainingLimit > 0 && totalImportCount > adjustedOffset) {
+          imports = await server.services.secretImport.getImportsMultiEnv({
+            actorId: req.permission.id,
+            actor: req.permission.type,
+            actorAuthMethod: req.permission.authMethod,
+            actorOrgId: req.permission.orgId,
+            projectId,
+            environments,
+            path: secretPath,
+            search,
+            limit: remainingLimit,
+            offset: adjustedOffset
+          });
+
+          await server.services.auditLog.createAuditLog({
+            ...req.auditLogInfo,
+            projectId: req.query.projectId,
+            event: {
+              type: EventType.GET_SECRET_IMPORTS,
+              metadata: {
+                environment: environments.join(","),
+                folderId: imports?.[0]?.folderId,
+                numberOfImports: imports.length
+              }
+            }
+          });
+
+          remainingLimit -= imports.length;
+          adjustedOffset = 0;
+        } else {
+          adjustedOffset = Math.max(0, adjustedOffset - totalImportCount);
+        }
+      }
 
       if (includeFolders) {
         // this is the unique count, ie duplicate folders across envs only count as 1
@@ -345,10 +403,13 @@ export const registerDashboardRouter = async (server: FastifyZodProvider) => {
         folders,
         dynamicSecrets,
         secrets,
+        imports,
         totalFolderCount,
         totalDynamicSecretCount,
+        totalImportCount,
         totalSecretCount,
-        totalCount: (totalFolderCount ?? 0) + (totalDynamicSecretCount ?? 0) + (totalSecretCount ?? 0)
+        totalCount:
+          (totalFolderCount ?? 0) + (totalDynamicSecretCount ?? 0) + (totalSecretCount ?? 0) + (totalImportCount ?? 0)
       };
     }
   });

backend/src/services/secret-import/secret-import-service.ts

@@ -469,6 +469,58 @@ export const secretImportServiceFactory = ({
     return count;
   };
 
+  const getProjectImportMultiEnvCount = async ({
+    path: secretPath,
+    environments,
+    projectId,
+    actor,
+    actorId,
+    actorAuthMethod,
+    actorOrgId,
+    search
+  }: Omit<TGetSecretImportsDTO, "environment"> & { environments: string[] }) => {
+    const { permission } = await permissionService.getProjectPermission({
+      actor,
+      actorId,
+      projectId,
+      actorAuthMethod,
+      actorOrgId,
+      actionProjectType: ActionProjectType.SecretManager
+    });
+    const filteredEnvironments = [];
+    for (const environment of environments) {
+      if (
+        permission.can(
+          ProjectPermissionActions.Read,
+          subject(ProjectPermissionSub.SecretImports, { environment, secretPath })
+        )
+      ) {
+        filteredEnvironments.push(environment);
+      }
+    }
+    if (filteredEnvironments.length === 0) {
+      return 0;
+    }
+
+    for (const environment of filteredEnvironments) {
+      ForbiddenError.from(permission).throwUnlessCan(
+        ProjectPermissionActions.Read,
+        subject(ProjectPermissionSub.SecretImports, { environment, secretPath })
+      );
+    }
+
+    const folders = await folderDAL.findBySecretPathMultiEnv(projectId, environments, secretPath);
+    if (!folders?.length)
+      throw new NotFoundError({
+        message: `Folder with path '${secretPath}' not found on environments with slugs '${environments.join(", ")}'`
+      });
+    const counts = await Promise.all(
+      folders.map((folder) => secretImportDAL.getProjectImportCount({ folderId: folder.id, search }))
+    );
+
+    return counts.reduce((sum, count) => sum + count, 0);
+  };
+
   const getImports = async ({
     path: secretPath,
     environment,
@@ -688,6 +740,59 @@ export const secretImportServiceFactory = ({
     }));
   };
 
+  const getImportsMultiEnv = async ({
+    path: secretPath,
+    environments,
+    projectId,
+    actor,
+    actorId,
+    actorAuthMethod,
+    actorOrgId,
+    search,
+    limit,
+    offset
+  }: Omit<TGetSecretImportsDTO, "environment"> & { environments: string[] }) => {
+    const { permission } = await permissionService.getProjectPermission({
+      actor,
+      actorId,
+      projectId,
+      actorAuthMethod,
+      actorOrgId,
+      actionProjectType: ActionProjectType.SecretManager
+    });
+    const filteredEnvironments = [];
+    for (const environment of environments) {
+      if (
+        permission.can(
+          ProjectPermissionActions.Read,
+          subject(ProjectPermissionSub.SecretImports, { environment, secretPath })
+        )
+      ) {
+        filteredEnvironments.push(environment);
+      }
+    }
+    if (filteredEnvironments.length === 0) {
+      return [];
+    }
+
+    const folders = await folderDAL.findBySecretPathMultiEnv(projectId, filteredEnvironments, secretPath);
+    if (!folders?.length)
+      throw new NotFoundError({
+        message: `Folder with path '${secretPath}' not found on environments with slugs '${environments.join(", ")}'`
+      });
+
+    const secImportsArrays = await Promise.all(
+      folders.map(async (folder) => {
+        const imports = await secretImportDAL.find({ folderId: folder.id, search, limit, offset });
+        return imports.map((importItem) => ({
+          ...importItem,
+          environment: folder.environment.slug
+        }));
+      })
+    );
+    return secImportsArrays.flat();
+  };
+
   return {
     createImport,
     updateImport,
@@ -698,6 +803,8 @@ export const secretImportServiceFactory = ({
     getRawSecretsFromImports,
     resyncSecretImportReplication,
     getProjectImportCount,
-    fnSecretsFromImports
+    fnSecretsFromImports,
+    getProjectImportMultiEnvCount,
+    getImportsMultiEnv
   };
 };

frontend/src/hooks/api/dashboard/queries.tsx

@@ -143,6 +143,7 @@ export const useGetProjectSecretsOverview = (
     search = "",
     includeSecrets,
     includeFolders,
+    includeImports,
     includeDynamicSecrets,
     environments
   }: TGetDashboardProjectSecretsOverviewDTO,
@@ -170,6 +171,7 @@ export const useGetProjectSecretsOverview = (
       projectId,
       includeSecrets,
       includeFolders,
+      includeImports,
       includeDynamicSecrets,
       environments
     }),
@@ -184,6 +186,7 @@ export const useGetProjectSecretsOverview = (
         projectId,
         includeSecrets,
         includeFolders,
+        includeImports,
         includeDynamicSecrets,
         environments
       }),
@@ -197,12 +200,15 @@ export const useGetProjectSecretsOverview = (
         ? unique(select.dynamicSecrets, (i) => i.name)
         : [];
 
+      const uniqueSecretImports = select.imports ? unique(select.imports, (i) => i.id) : [];
+
       return {
         ...select,
         secrets: secrets ? mergePersonalSecrets(secrets) : undefined,
         totalUniqueSecretsInPage: uniqueSecrets.length,
         totalUniqueDynamicSecretsInPage: uniqueDynamicSecrets.length,
-        totalUniqueFoldersInPage: uniqueFolders.length
+        totalUniqueFoldersInPage: uniqueFolders.length,
+        totalUniqueSecretImportsInPage: uniqueSecretImports.length
       };
     }, []),
     placeholderData: (previousData) => previousData

frontend/src/hooks/api/dashboard/types.ts

@@ -9,13 +9,16 @@ export type DashboardProjectSecretsOverviewResponse = {
   folders?: (TSecretFolder & { environment: string })[];
   dynamicSecrets?: (TDynamicSecret & { environment: string })[];
   secrets?: SecretV3Raw[];
+  imports?: TSecretImport[];
   totalSecretCount?: number;
   totalFolderCount?: number;
   totalDynamicSecretCount?: number;
+  totalImportCount?: number;
   totalCount: number;
   totalUniqueSecretsInPage: number;
   totalUniqueDynamicSecretsInPage: number;
   totalUniqueFoldersInPage: number;
+  totalUniqueSecretImportsInPage: number;
 };
 
 export type DashboardProjectSecretsDetailsResponse = {
@@ -63,6 +66,7 @@ export type TGetDashboardProjectSecretsOverviewDTO = {
   includeSecrets?: boolean;
   includeFolders?: boolean;
   includeDynamicSecrets?: boolean;
+  includeImports?: boolean;
   environments: string[];
 };
 

frontend/src/hooks/api/secretImports/types.ts

@@ -14,6 +14,7 @@ export type TSecretImport = {
   isReplicationSuccess?: boolean;
   replicationStatus?: string;
   lastReplicated?: string;
+  environment?: string;
 };
 
 export type TGetImportedFoldersByEnvDTO = {

frontend/src/pages/secret-manager/OverviewPage/OverviewPage.tsx

@@ -7,6 +7,7 @@ import {
   faAngleDown,
   faArrowDown,
   faArrowUp,
+  faFileImport,
   faFingerprint,
   faFolder,
   faFolderBlank,
@@ -80,6 +81,7 @@ import { CreateSecretForm } from "./components/CreateSecretForm";
 import { FolderBreadCrumbs } from "./components/FolderBreadCrumbs";
 import { SecretOverviewDynamicSecretRow } from "./components/SecretOverviewDynamicSecretRow";
 import { SecretOverviewFolderRow } from "./components/SecretOverviewFolderRow";
+import { SecretOverviewImportListView } from "./components/SecretOverviewImportListView";
 import {
   SecretNoAccessOverviewTableRow,
   SecretOverviewTableRow
@@ -97,7 +99,8 @@ export enum EntryType {
 enum RowType {
   Folder = "folder",
   DynamicSecret = "dynamic",
-  Secret = "secret"
+  Secret = "secret",
+  Import = "import"
 }
 
 type Filter = {
@@ -107,7 +110,8 @@ type Filter = {
 const DEFAULT_FILTER_STATE = {
   [RowType.Folder]: true,
   [RowType.DynamicSecret]: true,
-  [RowType.Secret]: true
+  [RowType.Secret]: true,
+  [RowType.Import]: true
 };
 
 export const OverviewPage = () => {
@@ -216,6 +220,7 @@ export const OverviewPage = () => {
       includeFolders: filter.folder,
       includeDynamicSecrets: filter.dynamic,
       includeSecrets: filter.secret,
+      includeImports: filter.import,
       search: debouncedSearchFilter,
       limit,
       offset
@@ -227,15 +232,29 @@ export const OverviewPage = () => {
     secrets,
     folders,
     dynamicSecrets,
+    imports,
     totalFolderCount,
     totalSecretCount,
     totalDynamicSecretCount,
+    totalImportCount,
     totalCount = 0,
     totalUniqueFoldersInPage,
     totalUniqueSecretsInPage,
+    totalUniqueSecretImportsInPage,
     totalUniqueDynamicSecretsInPage
   } = overview ?? {};
 
+  const importsShaped = imports
+    ?.filter((el) => !el.isReserved)
+    ?.map(({ importPath, importEnv }) => ({ importPath, importEnv }))
+    .filter(
+      (el, index, self) =>
+        index ===
+        self.findIndex(
+          (item) => item.importPath === el.importPath && item.importEnv.slug === el.importEnv.slug
+        )
+    );
+
   useResetPageHelper({
     totalCount,
     offset,
@@ -678,7 +697,6 @@ export const OverviewPage = () => {
         <SecretV2MigrationSection />
       </div>
     );
-
   return (
     <div className="">
       <Helmet>
@@ -767,6 +785,19 @@ export const OverviewPage = () => {
                     </Button>
                   </DropdownMenuItem> */}
                   <DropdownMenuLabel>Filter project resources</DropdownMenuLabel>
+                  <DropdownMenuItem
+                    onClick={(e) => {
+                      e.preventDefault();
+                      handleToggleRowType(RowType.Import);
+                    }}
+                    icon={filter[RowType.Import] && <FontAwesomeIcon icon={faCheckCircle} />}
+                    iconPos="right"
+                  >
+                    <div className="flex items-center gap-2">
+                      <FontAwesomeIcon icon={faFileImport} className="text-green-700" />
+                      <span>Imports</span>
+                    </div>
+                  </DropdownMenuItem>
                   <DropdownMenuItem
                     onClick={(e) => {
                       e.preventDefault();
@@ -1093,6 +1124,17 @@ export const OverviewPage = () => {
                         key={`overview-${dynamicSecretName}-${index + 1}`}
                       />
                     ))}
+                    {filter.import &&
+                      importsShaped &&
+                      importsShaped?.length > 0 &&
+                      importsShaped?.map((item, index) => (
+                        <SecretOverviewImportListView
+                          secretImport={item}
+                          environments={visibleEnvs}
+                          key={`overview-secret-input-${index + 1}`}
+                          allSecretImports={imports}
+                        />
+                      ))}
                     {secKeys.map((key, index) => (
                       <SecretOverviewTableRow
                         isSelected={Boolean(selectedEntries.secret[key])}
@@ -1116,7 +1158,8 @@ export const OverviewPage = () => {
                         (page * perPage > totalCount ? totalCount % perPage : perPage) -
                           (totalUniqueFoldersInPage || 0) -
                           (totalUniqueDynamicSecretsInPage || 0) -
-                          (totalUniqueSecretsInPage || 0),
+                          (totalUniqueSecretsInPage || 0) -
+                          (totalUniqueSecretImportsInPage || 0),
                         0
                       )}
                     />
@@ -1156,6 +1199,7 @@ export const OverviewPage = () => {
                   dynamicSecretCount={totalDynamicSecretCount}
                   secretCount={totalSecretCount}
                   folderCount={totalFolderCount}
+                  importCount={totalImportCount}
                 />
               }
               className="rounded-b-md border-t border-solid border-t-mineshaft-600"

frontend/src/pages/secret-manager/OverviewPage/components/SecretOverviewImportListView/SecretOverviewImportListView.tsx

@@ -0,0 +1,85 @@
+import { faCheck, faFileImport, faXmark } from "@fortawesome/free-solid-svg-icons";
+import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
+import { twMerge } from "tailwind-merge";
+
+import { Td, Tr } from "@app/components/v2";
+import { TSecretImport, WorkspaceEnv } from "@app/hooks/api/types";
+import { EnvFolderIcon } from "@app/pages/secret-manager/SecretDashboardPage/components/SecretImportListView/SecretImportItem";
+
+type Props = {
+  secretImport: { importPath: string; importEnv: WorkspaceEnv };
+  environments: { name: string; slug: string }[];
+  allSecretImports?: TSecretImport[];
+};
+
+export const SecretOverviewImportListView = ({
+  secretImport,
+  environments = [],
+  allSecretImports = []
+}: Props) => {
+  const isSecretPresentInEnv = (envSlug: string) => {
+    return allSecretImports.some((item) => {
+      if (item.isReplication) {
+        if (
+          item.importPath === secretImport.importPath &&
+          item.importEnv.slug === secretImport.importEnv.slug
+        ) {
+          const reservedItem = allSecretImports.find((element) =>
+            element.importPath.includes(`__reserve_replication_${item.id}`)
+          );
+          // If the reserved item exists, check if the envSlug matches
+          if (reservedItem) {
+            return reservedItem.environment === envSlug;
+          }
+        }
+      } else {
+        // If the item is not replication, check if the envSlug matches directly
+        return (
+          item.environment === envSlug &&
+          item.importPath === secretImport.importPath &&
+          item.importEnv.slug === secretImport.importEnv.slug
+        );
+      }
+      return false;
+    });
+  };
+
+  return (
+    <Tr className="group">
+      <Td className="sticky left-0 z-10 border-r border-mineshaft-600 bg-mineshaft-800 bg-clip-padding px-0 py-0 group-hover:bg-mineshaft-700">
+        <div className="group flex cursor-pointer">
+          <div className="flex w-11 items-center py-2 pl-5 text-green-700">
+            <FontAwesomeIcon icon={faFileImport} />
+          </div>
+          <div className="flex flex-grow items-center py-2 pl-4 pr-2">
+            <EnvFolderIcon
+              env={secretImport.importEnv.slug || ""}
+              secretPath={secretImport.importPath || ""}
+            />
+          </div>
+        </div>
+      </Td>
+      {environments.map(({ slug }, i) => {
+        const isPresent = isSecretPresentInEnv(slug);
+        return (
+          <Td
+            key={`sec-overview-${slug}-${i + 1}-value`}
+            className={twMerge(
+              "px-0 py-0 group-hover:bg-mineshaft-700",
+              isPresent ? "text-green-600" : "text-red-600"
+            )}
+          >
+            <div className="h-full w-full border-r border-mineshaft-600 px-5 py-[0.85rem]">
+              <div className="flex justify-center">
+                <FontAwesomeIcon
+                  // eslint-disable-next-line no-nested-ternary
+                  icon={isSecretPresentInEnv(slug) ? faCheck : faXmark}
+                />
+              </div>
+            </div>
+          </Td>
+        );
+      })}
+    </Tr>
+  );
+};

frontend/src/pages/secret-manager/OverviewPage/components/SecretOverviewImportListView/index.tsx

@@ -0,0 +1 @@
+export { SecretOverviewImportListView } from "./SecretOverviewImportListView";