Revise Northflank integration

backend/src/controllers/v1/integrationAuthController.ts

@@ -8,6 +8,7 @@ import {
   ALGORITHM_AES_256_GCM,
   ENCODING_SCHEME_UTF8,
   INTEGRATION_BITBUCKET_API_URL,
+  INTEGRATION_NORTHFLANK_API_URL,
   INTEGRATION_RAILWAY_API_URL,
   INTEGRATION_SET,
   INTEGRATION_VERCEL_API_URL,
@@ -445,6 +446,79 @@ export const getIntegrationAuthBitBucketWorkspaces = async (req: Request, res: R
   });
 };
 
+/**
+ * Return list of secret groups for Northflank project with id [appId]
+ * @param req 
+ * @param res 
+ * @returns 
+ */
+export const getIntegrationAuthNorthflankSecretGroups = async (req: Request, res: Response) => {
+  const appId = req.query.appId as string;
+  
+  interface NorthflankSecretGroup {
+    id: string;
+    name: string;
+    description: string;
+    priority: number;
+    projectId: string;
+  }
+  
+  interface SecretGroup {
+    name: string;
+    groupId: string;
+  }
+  
+  const secretGroups: SecretGroup[] = [];
+
+  if (appId && appId !== "") { 
+    let page = 1;
+    const perPage = 10;
+    let hasMorePages = true;
+    
+    while(hasMorePages) {
+      const params = new URLSearchParams({
+        page: String(page),
+        per_page: String(perPage),
+        filter: "all",
+      });
+
+      const {
+        data: {
+          data: {
+            secrets
+          }
+        }
+      } = await standardRequest.get<{ data: { secrets: NorthflankSecretGroup[] }}>(
+        `${INTEGRATION_NORTHFLANK_API_URL}/v1/projects/${appId}/secrets`,
+        {
+          params,
+          headers: {
+            Authorization: `Bearer ${req.accessToken}`,
+            "Accept-Encoding": "application/json",
+          },
+        }
+      );
+      
+      secrets.forEach((a: any) => {
+        secretGroups.push({
+          name: a.name,
+          groupId: a.id
+        });
+      });
+
+      if (secrets.length < perPage) {
+        hasMorePages = false;
+      }
+
+      page++;
+    }
+  }
+  
+  return res.status(200).send({
+    secretGroups
+  });
+}
+
 /**
  * Delete integration authorization with id [integrationAuthId]
  * @param req
@@ -461,3 +535,4 @@ export const deleteIntegrationAuth = async (req: Request, res: Response) => {
     integrationAuth
   });
 };
+

backend/src/controllers/v1/integrationController.ts

@@ -27,8 +27,7 @@ export const createIntegration = async (req: Request, res: Response) => {
     owner,
     path,
     region,
-    secretPath,
+    secretPath
-    secretGroup
   } = req.body;
 
   const folders = await Folder.findOne({
@@ -62,7 +61,6 @@ export const createIntegration = async (req: Request, res: Response) => {
     path,
     region,
     secretPath,
-    secretGroup,
     integration: req.integrationAuth.integration,
     integrationAuth: new Types.ObjectId(integrationAuthId)
   }).save();

backend/src/integrations/apps.ts

@@ -877,7 +877,8 @@ const getAppsBitBucket = async ({
   });
   return apps;
 }
- /* Return list of projects for Northflank integration
+
+/** Return list of projects for Northflank integration
  * @param {Object} obj
  * @param {String} obj.accessToken - access token for Northflank API
  * @returns {Object[]} apps - names of Northflank apps
@@ -903,36 +904,10 @@ const getAppsNorthflank = async ({ accessToken }: { accessToken: string }) => {
   const apps = projects.map((a: any) => {
     return {
       name: a.name,
-      appId: a.id,
+      appId: a.id
-      secretGroups: []
     };
   });
 
-  for (let i = 0; i < apps.length; i++) {
-    const appName = apps[i].name;
-    const {
-      data: {
-        data: {
-          secrets
-        }
-      }
-    } = await standardRequest.get(
-      `${INTEGRATION_NORTHFLANK_API_URL}/v1/projects/${appName}/secrets`,
-      {
-        headers: {
-          Authorization: `Bearer ${accessToken}`,
-          "Accept-Encoding": "application/json",
-        },
-      }
-    );
-
-    const secretGroups = secrets.map((a: any) => {
-      return a.id
-    });
-
-    apps[i].secretGroups = secretGroups
-  }
-
   return apps;
 };
 

backend/src/integrations/sync.ts

@@ -2375,7 +2375,7 @@ const syncSecretsCloud66 = async ({
   }
 };
 
- /* Sync/push [secrets] to Northflank
+/** Sync/push [secrets] to Northflank
  * @param {Object} obj
  * @param {IIntegration} obj.integration - integration details
  * @param {Object} obj.secrets - secrets to push to integration (object where keys are secret keys and values are secret values)
@@ -2390,16 +2390,13 @@ const syncSecretsNorthflank = async ({
   secrets: any;
   accessToken: string;
 }) => {
-
+  await standardRequest.patch(
-  const modifiedFormatForSecretInjection = {
+    `${INTEGRATION_NORTHFLANK_API_URL}/v1/projects/${integration.appId}/secrets/${integration.targetServiceId}`,
-    secrets: {
+    {
-      variables: secrets
+      secrets: {
-    }
+        variables: secrets
-  }
+      }
-
+    },
-  await standardRequest.post(
-    `${INTEGRATION_NORTHFLANK_API_URL}/v1/projects/${integration.appId}/secrets/${integration.secretGroup}`,
-    modifiedFormatForSecretInjection,
     {
       headers: {
           Authorization: `Bearer ${accessToken}`,

backend/src/models/integration.ts

@@ -42,7 +42,6 @@ export interface IIntegration {
   path: string;
   region: string;
   secretPath: string;
-  secretGroup: string;
   integration:
     | "azure-key-vault"
     | "aws-parameter-store"
@@ -175,12 +174,7 @@ const integrationSchema = new Schema<IIntegration>(
       type: String,
       required: true,
       default: "/",
-    },
+    }
-    secretGroup: {
-      // northflank-specific service
-      type: String,
-      default: null,
-    },
   },
   {
     timestamps: true,

backend/src/routes/v1/integration.ts

@@ -37,7 +37,6 @@ router.post(
   body("owner").trim(),
   body("path").trim(),
   body("region").trim(),
-  body("secretGroup").isString().trim(),
   validateRequest,
   integrationController.createIntegration
 );

backend/src/routes/v1/integrationAuth.ts

@@ -155,6 +155,20 @@ router.get(
 	integrationAuthController.getIntegrationAuthBitBucketWorkspaces
 );
 
+router.get(
+	"/:integrationAuthId/northflank/secret-groups",
+	requireAuth({
+        acceptedAuthModes: [AUTH_MODE_JWT],
+    }),
+	requireIntegrationAuthorizationAuth({
+		acceptedRoles: [ADMIN, MEMBER],
+	}),
+	param("integrationAuthId").exists().isString(),
+	query("appId").exists().isString(),
+	validateRequest,
+	integrationAuthController.getIntegrationAuthNorthflankSecretGroups
+);
+
 router.delete(
 	"/:integrationAuthId",
 	requireAuth({

frontend/src/hooks/api/integrationAuth/index.tsx

@@ -3,8 +3,8 @@ export {
   useGetIntegrationAuthApps,
   useGetIntegrationAuthBitBucketWorkspaces,
   useGetIntegrationAuthById,
+  useGetIntegrationAuthNorthflankSecretGroups,
   useGetIntegrationAuthRailwayEnvironments,
   useGetIntegrationAuthRailwayServices,
   useGetIntegrationAuthTeams,
-  useGetIntegrationAuthVercelBranches,
+  useGetIntegrationAuthVercelBranches} from "./queries";
-} from "./queries";

frontend/src/hooks/api/integrationAuth/queries.tsx

@@ -3,7 +3,15 @@ import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query";
 import { apiRequest } from "@app/config/request";
 
 import { workspaceKeys } from "../workspace/queries";
-import { App, BitBucketWorkspace, Environment, IntegrationAuth, Service, Team } from "./types";
+import { 
+  App, 
+  BitBucketWorkspace, 
+  Environment, 
+  IntegrationAuth, 
+  NorthflankSecretGroup,
+  Service, 
+  Team 
+} from "./types";
 
 const integrationAuthKeys = {
   getIntegrationAuthById: (integrationAuthId: string) =>
@@ -19,7 +27,6 @@ const integrationAuthKeys = {
     integrationAuthId: string;
     appId: string;
   }) => [{ integrationAuthId, appId }, "integrationAuthVercelBranches"] as const,
-
   getIntegrationAuthRailwayEnvironments: ({
     integrationAuthId,
     appId
@@ -36,6 +43,13 @@ const integrationAuthKeys = {
   }) => [{ integrationAuthId, appId }, "integrationAuthRailwayServices"] as const,
   getIntegrationAuthBitBucketWorkspaces: (integrationAuthId: string) =>
     [{ integrationAuthId }, "integrationAuthBitbucketWorkspaces"] as const,
+  getIntegrationAuthNorthflankSecretGroups: ({
+    integrationAuthId,
+    appId
+  }: {
+    integrationAuthId: string;
+    appId: string;
+  }) => [{ integrationAuthId, appId }, "integrationAuthNorthflankSecretGroups"] as const, 
 };
 
 const fetchIntegrationAuthById = async (integrationAuthId: string) => {
@@ -148,6 +162,27 @@ const fetchIntegrationAuthBitBucketWorkspaces = async (integrationAuthId: string
   return workspaces;
 };
 
+const fetchIntegrationAuthNorthflankSecretGroups = async ({
+  integrationAuthId,
+  appId
+}: {
+  integrationAuthId: string;
+  appId: string;
+}) => {
+  const {
+    data: { secretGroups }
+  } = await apiRequest.get<{ secretGroups: NorthflankSecretGroup[] }>(
+    `/api/v1/integration-auth/${integrationAuthId}/northflank/secret-groups`,
+    {
+      params: {
+        appId
+      }
+    }
+  );
+
+  return secretGroups;
+};
+
 export const useGetIntegrationAuthById = (integrationAuthId: string) => {
   return useQuery({
     queryKey: integrationAuthKeys.getIntegrationAuthById(integrationAuthId),
@@ -256,6 +291,27 @@ export const useGetIntegrationAuthBitBucketWorkspaces = (integrationAuthId: stri
   });
 };
 
+export const useGetIntegrationAuthNorthflankSecretGroups = ({
+  integrationAuthId,
+  appId
+}: {
+  integrationAuthId: string;
+  appId: string;
+}) => {
+  return useQuery({
+    queryKey: integrationAuthKeys.getIntegrationAuthNorthflankSecretGroups({
+      integrationAuthId,
+      appId
+    }),
+    queryFn: () =>
+      fetchIntegrationAuthNorthflankSecretGroups({
+        integrationAuthId,
+        appId
+      }),
+    enabled: true
+  });
+};
+
 export const useDeleteIntegrationAuth = () => {
   const queryClient = useQueryClient();
 

frontend/src/hooks/api/integrationAuth/types.ts

@@ -35,4 +35,9 @@ export type BitBucketWorkspace = {
   uuid: string;
   name: string;
   slug: string;
+}
+
+export type NorthflankSecretGroup = {
+  name: string;
+  groupId: string;
 }

frontend/src/pages/api/integrations/createIntegration.ts

@@ -4,7 +4,6 @@ interface Props {
   integrationAuthId: string;
   isActive: boolean;
   secretPath: string;
-  secretGroup?: string;
   app: string | null;
   appId: string | null;
   sourceEnvironment: string;
@@ -36,7 +35,6 @@ const createIntegration = ({
   path,
   region,
   secretPath,
-  secretGroup
 }: Props) =>
   SecurityClient.fetchCall("/api/v1/integration", {
     method: "POST",
@@ -57,7 +55,6 @@ const createIntegration = ({
       path,
       region,
       secretPath,
-      secretGroup
     })
   }).then(async (res) => {
     if (res && res.status === 200) {

frontend/src/pages/integrations/northflank/create.tsx

@@ -13,7 +13,8 @@ import {
 } from "../../../components/v2";
 import {
   useGetIntegrationAuthApps,
-  useGetIntegrationAuthById
+  useGetIntegrationAuthById,
+  useGetIntegrationAuthNorthflankSecretGroups
 } from "../../../hooks/api/integrationAuth";
 import { useGetWorkspaceById } from "../../../hooks/api/workspace";
 import createIntegration from "../../api/integrations/createIntegration";
@@ -21,6 +22,13 @@ import createIntegration from "../../api/integrations/createIntegration";
 export default function NorthflankCreateIntegrationPage() {
   const router = useRouter();
 
+  const [selectedSourceEnvironment, setSelectedSourceEnvironment] = useState("");
+  const [secretPath, setSecretPath] = useState("/");
+  const [targetAppId, setTargetAppId] = useState("");
+  const [targetSecretGroupId, setTargetSecretGroupId] = useState<string | null>(null);
+
+  const [isLoading, setIsLoading] = useState(false);
+
   const { integrationAuthId } = queryString.parse(router.asPath.split("?")[1]);
 
   const { data: workspace } = useGetWorkspaceById(localStorage.getItem("projectData.id") ?? "");
@@ -28,15 +36,11 @@ export default function NorthflankCreateIntegrationPage() {
   const { data: integrationAuthApps } = useGetIntegrationAuthApps({
     integrationAuthId: (integrationAuthId as string) ?? ""
   });
-
+  const { data: integrationAuthSecretGroups } = useGetIntegrationAuthNorthflankSecretGroups({
-  const [selectedSourceEnvironment, setSelectedSourceEnvironment] = useState("");
+    integrationAuthId: (integrationAuthId as string) ?? "",
-  const [secretPath, setSecretPath] = useState("/");
+    appId: targetAppId
-  const [targetApp, setTargetApp] = useState("");
+  });
-  const [secretGroupList, setSecretGroupList] = useState<any>([]);
+  
-  const [targetSecretGroup, setTargetSecretGroup] = useState("");
-
-  const [isLoading, setIsLoading] = useState(false);
-
   useEffect(() => {
     if (workspace) {
       setSelectedSourceEnvironment(workspace.environments[0].slug);
@@ -46,28 +50,28 @@ export default function NorthflankCreateIntegrationPage() {
   useEffect(() => {
     if (integrationAuthApps) {
       if (integrationAuthApps.length > 0) {
-        setTargetApp(integrationAuthApps[0].name);
+        // setTargetApp(integrationAuthApps[0].name);
+        setTargetAppId(integrationAuthApps[0].appId as string);
       } else {
-        setTargetApp("none");
+        // setTargetApp("none");
+        setTargetAppId("none");
       }
     }
   }, [integrationAuthApps]);
-
+  
   useEffect(() => {
-    if (integrationAuthApps) {
+    if (integrationAuthSecretGroups) {
-      if (integrationAuthApps.length > 0) {
+      if (integrationAuthSecretGroups.length > 0) {
-        const selectedApp = integrationAuthApps?.filter((integrationAuthApp) => integrationAuthApp.name === targetApp);
+        // case: project has at least 1 secret group in Northflank
-        if (selectedApp.length > 0 && selectedApp[0].secretGroups) {
+        setTargetSecretGroupId(integrationAuthSecretGroups[0].groupId);
-          setSecretGroupList(selectedApp[0].secretGroups);
+      } else {
-          setTargetSecretGroup(selectedApp[0]?.secretGroups[0]);
+        // case: project has no secret groups in Northflank
-        } else {
+        setTargetSecretGroupId("none");
-          setSecretGroupList([]);
-          setTargetSecretGroup("none");
-        }
       }
     }
-  }, [targetApp])
+      
-
+  }, [integrationAuthSecretGroups]);
+  
   const handleButtonClick = async () => {
     try {
       if (!integrationAuth?._id) return;
@@ -77,20 +81,19 @@ export default function NorthflankCreateIntegrationPage() {
       await createIntegration({
         integrationAuthId: integrationAuth?._id,
         isActive: true,
-        app: targetApp,
+        app: integrationAuthApps?.find(
-        appId:
+            (integrationAuthApp) => integrationAuthApp.appId === targetAppId
-          integrationAuthApps?.find((integrationAuthApp) => integrationAuthApp.name === targetApp)
+          )?.name ?? null,
-            ?.appId ?? null,
+        appId: targetAppId,
         sourceEnvironment: selectedSourceEnvironment,
         targetEnvironment: null,
         targetEnvironmentId: null,
         targetService: null,
-        targetServiceId: null,
+        targetServiceId: targetSecretGroupId,
         owner: null,
         path: null,
         region: null,
-        secretPath,
+        secretPath
-        secretGroup: targetSecretGroup
       });
 
       setIsLoading(false);
@@ -100,12 +103,12 @@ export default function NorthflankCreateIntegrationPage() {
       console.error(err);
     }
   };
-
+  
   return integrationAuth &&
     workspace &&
     selectedSourceEnvironment &&
     integrationAuthApps &&
-    targetApp ? (
+    targetAppId ? (
     <div className="flex h-full w-full items-center justify-center">
       <Card className="max-w-md rounded-md p-8">
         <CardTitle className="text-center">Northflank Integration</CardTitle>
@@ -134,15 +137,15 @@ export default function NorthflankCreateIntegrationPage() {
         </FormControl>
         <FormControl label="Northflank Project" className="mt-4">
           <Select
-            value={targetApp}
+            value={targetAppId}
-            onValueChange={(val) => setTargetApp(val)}
+            onValueChange={(val) => setTargetAppId(val)}
             className="w-full border border-mineshaft-500"
             isDisabled={integrationAuthApps.length === 0}
           >
             {integrationAuthApps.length > 0 ? (
               integrationAuthApps.map((integrationAuthApp) => (
                 <SelectItem
-                  value={integrationAuthApp.name}
+                  value={integrationAuthApp.appId as string}
                   key={`target-environment-${integrationAuthApp.name}`}
                 >
                   {integrationAuthApp.name}
@@ -155,35 +158,37 @@ export default function NorthflankCreateIntegrationPage() {
             )}
           </Select>
         </FormControl>
-        <FormControl label="Secret Group" className="mt-4">
+        {targetSecretGroupId && integrationAuthSecretGroups && (
-          <Select
+          <FormControl label="Secret Group" className="mt-4">
-            value={targetSecretGroup}
+            <Select
-            onValueChange={(val) => setTargetSecretGroup(val)}
+              value={targetSecretGroupId}
-            className="w-full border border-mineshaft-500"
+              onValueChange={(val) => setTargetSecretGroupId(val)}
-            isDisabled={secretGroupList.length === 0}
+              className="w-full border border-mineshaft-500"
-          >
+              isDisabled={integrationAuthSecretGroups.length === 0}
-            {secretGroupList.length > 0 ? (
+            >
-              secretGroupList.map((group: any) => (
+              {integrationAuthSecretGroups.length > 0 ? (
-                <SelectItem
+                integrationAuthSecretGroups.map((secretGroup: any) => (
-                  value={group}
+                  <SelectItem
-                  key={`target-secret-group-${group}`}
+                    value={secretGroup.groupId}
-                >
+                    key={`target-secret-group-${secretGroup.groupId}`}
-                  {group}
+                  >
+                    {secretGroup.name}
+                  </SelectItem>
+                ))
+              ) : (
+                <SelectItem value="none" key="target-secret-group-none">
+                  No secret groups found
                 </SelectItem>
-              ))
+              )}
-            ) : (
+            </Select>
-              <SelectItem value="none" key="target-secret-group-none">
+          </FormControl>
-                No secret groups found
+        )}
-              </SelectItem>
-            )}
-          </Select>
-        </FormControl>
         <Button
           onClick={handleButtonClick}
           color="mineshaft"
           className="mt-4"
           isLoading={isLoading}
-          isDisabled={secretGroupList.length === 0}
+          isDisabled={integrationAuthApps.length === 0 || integrationAuthSecretGroups?.length === 0}
         >
           Create Integration
         </Button>