misc: unified to a single integration

backend/src/server/routes/v1/integration-auth-router.ts

@@ -1202,7 +1202,19 @@ export const registerIntegrationAuthRouter = async (server: FastifyZodProvider)
           organizations: z
             .object({
               name: z.string(),
-              slug: z.string()
+              slug: z.string(),
+              projects: z
+                .object({
+                  name: z.string(),
+                  id: z.string()
+                })
+                .array(),
+              contexts: z
+                .object({
+                  name: z.string(),
+                  id: z.string()
+                })
+                .array()
             })
             .array()
         })

backend/src/services/integration-auth/integration-app-list.ts

@@ -8,7 +8,6 @@ import { getConfig } from "@app/lib/config/env";
 import { request } from "@app/lib/config/request";
 import { NotFoundError } from "@app/lib/errors";
 
-import { TCircleCIContext } from "./integration-app-types";
 import { IntegrationAuthMetadataSchema, TIntegrationAuthMetadata } from "./integration-auth-schema";
 import { Integrations, IntegrationUrls } from "./integration-list";
 
@@ -490,47 +489,6 @@ const getAppsCircleCI = async ({ accessToken }: { accessToken: string }) => {
   return apps;
 };
 
-/**
- * Return list of contexts for CircleCI_Context integration
- */
-const getAppsCircleCIContexts = async ({ accessToken, orgSlug }: { accessToken: string; orgSlug: string }) => {
-  type NextPageToken = string | null | undefined;
-
-  type CircleCIContextResponse = {
-    items: TCircleCIContext[];
-    next_page_token: NextPageToken;
-  };
-
-  const contexts: TCircleCIContext[] = [];
-
-  let nextPageToken: NextPageToken;
-
-  while (nextPageToken !== null) {
-    const res = (
-      await request.get<CircleCIContextResponse>(`${IntegrationUrls.CIRCLECI_CONTEXT_API_URL}/v2/context`, {
-        headers: {
-          "Circle-Token": accessToken,
-          "Accept-Encoding": "application/json"
-        },
-        params: new URLSearchParams({
-          "owner-slug": orgSlug,
-          ...(nextPageToken ? { "page-token": nextPageToken } : {})
-        })
-      })
-    ).data;
-
-    contexts.push(...res.items);
-    nextPageToken = res.next_page_token;
-  }
-
-  const apps = contexts?.map((context) => ({
-    name: context.name,
-    appId: context.id
-  }));
-
-  return apps;
-};
-
 /**
  * Return list of projects for Databricks integration
  */
@@ -1237,12 +1195,6 @@ export const getApps = async ({
         accessToken
       });
 
-    case Integrations.CIRCLECI_CONTEXT:
-      return getAppsCircleCIContexts({
-        accessToken,
-        orgSlug: workspaceSlug as string
-      });
-
     case Integrations.DATABRICKS:
       return getAppsDatabricks({
         url,

backend/src/services/integration-auth/integration-auth-service.ts

@@ -1,3 +1,4 @@
+/* eslint-disable no-await-in-loop */
 import { ForbiddenError } from "@casl/ability";
 import { createAppAuth } from "@octokit/auth-app";
 import { Octokit } from "@octokit/rest";
@@ -11,6 +12,7 @@ import { getConfig } from "@app/lib/config/env";
 import { request } from "@app/lib/config/request";
 import { decryptSymmetric128BitHexKeyUTF8, encryptSymmetric128BitHexKeyUTF8 } from "@app/lib/crypto";
 import { BadRequestError, InternalServerError, NotFoundError } from "@app/lib/errors";
+import { logger } from "@app/lib/logger";
 import { TGenericPermission, TProjectPermission } from "@app/lib/types";
 
 import { TIntegrationDALFactory } from "../integration/integration-dal";
@@ -18,6 +20,7 @@ import { TKmsServiceFactory } from "../kms/kms-service";
 import { KmsDataKey } from "../kms/kms-types";
 import { TProjectBotServiceFactory } from "../project-bot/project-bot-service";
 import { getApps } from "./integration-app-list";
+import { TCircleCIContext } from "./integration-app-types";
 import { TIntegrationAuthDALFactory } from "./integration-auth-dal";
 import { IntegrationAuthMetadataSchema, TIntegrationAuthMetadata } from "./integration-auth-schema";
 import {
@@ -1593,8 +1596,8 @@ export const integrationAuthServiceFactory = ({
     const { shouldUseSecretV2Bridge, botKey } = await projectBotService.getBotKey(integrationAuth.projectId);
     const { accessToken } = await getIntegrationAccessToken(integrationAuth, shouldUseSecretV2Bridge, botKey);
 
-    const { data }: { data: TCircleCIOrganization[] } = await request.get(
-      `${IntegrationUrls.CIRCLECI_CONTEXT_API_URL}/v2/me/collaborations`,
+    const { data: organizations }: { data: TCircleCIOrganization[] } = await request.get(
+      `${IntegrationUrls.CIRCLECI_API_URL}/v2/me/collaborations`,
       {
         headers: {
           "Circle-Token": `${accessToken}`,
@@ -1603,7 +1606,106 @@ export const integrationAuthServiceFactory = ({
       }
     );
 
-    return data;
+    let projects: {
+      orgName: string;
+      projectName: string;
+      projectId?: string;
+    }[] = [];
+
+    try {
+      const projectRes = (
+        await request.get<{ reponame: string; username: string; vcs_url: string }[]>(
+          `${IntegrationUrls.CIRCLECI_API_URL}/v1.1/projects`,
+          {
+            headers: {
+              "Circle-Token": accessToken,
+              "Accept-Encoding": "application/json"
+            }
+          }
+        )
+      ).data;
+
+      projects = projectRes.map((a) => ({
+        orgName: a.username, // username maps to unique organization name in CircleCI
+        projectName: a.reponame, // reponame maps to project name within an organization in CircleCI
+        projectId: a.vcs_url.split("/").pop() // vcs_url maps to the project id in CircleCI
+      }));
+    } catch (error) {
+      logger.error(error);
+    }
+
+    const projectsByOrg = projects.reduce<Record<string, { name: string; id: string }[]>>((accum, project) => {
+      if (!accum[project.orgName]) {
+        return {
+          ...accum,
+          [project.orgName]: [
+            {
+              name: project.projectName,
+              id: project.projectId as string
+            }
+          ]
+        };
+      }
+      return {
+        ...accum,
+        [project.orgName]: [
+          ...accum[project.orgName],
+          {
+            name: project.projectName,
+            id: project.projectId as string
+          }
+        ]
+      };
+    }, {});
+
+    const getOrgContexts = async (orgSlug: string) => {
+      type NextPageToken = string | null | undefined;
+
+      type CircleCIContextResponse = {
+        items: TCircleCIContext[];
+        next_page_token: NextPageToken;
+      };
+
+      try {
+        const contexts: TCircleCIContext[] = [];
+        let nextPageToken: NextPageToken;
+
+        while (nextPageToken !== null) {
+          const res = (
+            await request.get<CircleCIContextResponse>(`${IntegrationUrls.CIRCLECI_API_URL}/v2/context`, {
+              headers: {
+                "Circle-Token": accessToken,
+                "Accept-Encoding": "application/json"
+              },
+              params: new URLSearchParams({
+                "owner-slug": orgSlug,
+                ...(nextPageToken ? { "page-token": nextPageToken } : {})
+              })
+            })
+          ).data;
+
+          contexts.push(...res.items);
+          nextPageToken = res.next_page_token;
+        }
+
+        return contexts?.map((context) => ({
+          name: context.name,
+          id: context.id
+        }));
+      } catch (error) {
+        logger.error(error);
+        return [];
+      }
+    };
+
+    return Promise.all(
+      organizations.map(async (org) => ({
+        name: org.name,
+        slug: org.slug,
+        projects: projectsByOrg[org.name] ?? [],
+        contexts: (await getOrgContexts(org.slug)) ?? []
+      }))
+    );
   };
 
   const deleteIntegrationAuths = async ({

backend/src/services/integration-auth/integration-list.ts

@@ -15,7 +15,6 @@ export enum Integrations {
   FLYIO = "flyio",
   LARAVELFORGE = "laravel-forge",
   CIRCLECI = "circleci",
-  CIRCLECI_CONTEXT = "circleci-context",
   DATABRICKS = "databricks",
   TRAVISCI = "travisci",
   TEAMCITY = "teamcity",
@@ -78,7 +77,6 @@ export enum IntegrationUrls {
   FLYIO_API_URL = "https://api.fly.io/graphql",
   CIRCLECI_API_URL = "https://circleci.com/api",
   // eslint-disable-next-line
-  CIRCLECI_CONTEXT_API_URL = "https://circleci.com/api",
   DATABRICKS_API_URL = "https:/xxxx.com/api",
   TRAVISCI_API_URL = "https://api.travis-ci.com",
   SUPABASE_API_URL = "https://api.supabase.com",
@@ -229,15 +227,6 @@ export const getIntegrationOptions = async () => {
       clientId: "",
       docsLink: ""
     },
-    {
-      name: "CircleCI Contexts",
-      slug: "circleci-context",
-      image: "CircleCI.png",
-      isAvailable: true,
-      type: "pat",
-      clientId: "",
-      docsLink: ""
-    },
     {
       name: "Databricks",
       slug: "databricks",

backend/src/services/integration-auth/integration-sync-secret.ts

@@ -2245,185 +2245,184 @@ const syncSecretsCircleCI = async ({
   secrets: Record<string, { value: string; comment?: string }>;
   accessToken: string;
 }) => {
-  const getProjectSlug = async () => {
-    const requestConfig = {
-      headers: {
-        "Circle-Token": accessToken,
-        "Accept-Encoding": "application/json"
-      }
-    };
+  enum CircleCiScope {
+    Project = "project",
+    Context = "context"
+  }
 
-    try {
-      const projectDetails = (
-        await request.get<{ slug: string }>(
-          `${IntegrationUrls.CIRCLECI_API_URL}/v2/project/${integration.appId}`,
-          requestConfig
-        )
-      ).data;
-
-      return projectDetails.slug;
-    } catch (err) {
-      if (err instanceof AxiosError) {
-        if (err.response?.data?.message !== "Not Found") {
-          throw new Error("Failed to get project slug from CircleCI during first attempt.");
-        }
-      }
-    }
-
-    // For backwards compatibility with old CircleCI integrations where we don't keep track of the organization name, so we can't filter by organization
-    try {
-      const circleCiOrganization = (
-        await request.get<{ slug: string; name: string }[]>(
-          `${IntegrationUrls.CIRCLECI_API_URL}/v2/me/collaborations`,
-          requestConfig
-        )
-      ).data;
-
-      // Case 1: This is a new integration where the organization name is stored under `integration.owner`
-      if (integration.owner) {
-        const org = circleCiOrganization.find((o) => o.name === integration.owner);
-        if (org) {
-          return `${org.slug}/${integration.app}`;
-        }
-      }
-
-      // Case 2: This is an old integration where the organization name is not stored, so we have to assume the first organization is the correct one
-      return `${circleCiOrganization[0].slug}/${integration.app}`;
-    } catch (err) {
-      throw new Error("Failed to get project slug from CircleCI during second attempt.");
-    }
-  };
-
-  const projectSlug = await getProjectSlug();
-
-  // sync secrets to CircleCI
-  await Promise.all(
-    Object.keys(secrets).map(async (key) =>
-      request.post(
-        `${IntegrationUrls.CIRCLECI_API_URL}/v2/project/${projectSlug}/envvar`,
-        {
-          name: key,
-          value: secrets[key].value
-        },
-        {
-          headers: {
-            "Circle-Token": accessToken,
-            "Content-Type": "application/json"
-          }
-        }
-      )
-    )
-  );
-
-  // get secrets from CircleCI
-  const getSecretsRes = (
-    await request.get<{ items: { name: string }[] }>(
-      `${IntegrationUrls.CIRCLECI_API_URL}/v2/project/${projectSlug}/envvar`,
-      {
-        headers: {
-          "Circle-Token": accessToken,
-          "Accept-Encoding": "application/json"
-        }
-      }
-    )
-  ).data?.items;
-
-  // delete secrets from CircleCI
-  await Promise.all(
-    getSecretsRes.map(async (sec) => {
-      if (!(sec.name in secrets)) {
-        return request.delete(`${IntegrationUrls.CIRCLECI_API_URL}/v2/project/${projectSlug}/envvar/${sec.name}`, {
-          headers: {
-            "Circle-Token": accessToken,
-            "Content-Type": "application/json"
-          }
-        });
-      }
-    })
-  );
-};
-
-/**
- * Sync/push [secrets] to CircleCI Context
- */
-const syncSecretsCircleCIContext = async ({
-  integration,
-  secrets,
-  accessToken
-}: {
-  integration: TIntegrations;
-  secrets: Record<string, { value: string; comment?: string }>;
-  accessToken: string;
-}) => {
-  // sync secrets to CircleCI
-  await Promise.all(
-    Object.keys(secrets).map(async (key) =>
-      request.put(
-        `${IntegrationUrls.CIRCLECI_API_URL}/v2/context/${integration.appId}/environment-variable/${key}`,
-        {
-          value: secrets[key].value
-        },
-        {
-          headers: {
-            "Circle-Token": accessToken,
-            "Content-Type": "application/json"
-          }
-        }
-      )
-    )
-  );
-
-  // get secrets from CircleCI
-  const getSecretsRes = async () => {
-    type EnvVars = {
-      variable: string;
-      created_at: string;
-      updated_at: string;
-      context_id: string;
-    };
-
-    type ResponseSchema = {
-      items: EnvVars[];
-      next_page_token: string | null;
-    };
-
-    let nextPageToken: string | null | undefined;
-    const envVars: EnvVars[] = [];
-
-    while (nextPageToken !== null) {
-      const res = await request.get<ResponseSchema>(
-        `${IntegrationUrls.CIRCLECI_API_URL}/v2/context/${integration.appId}/environment-variable`,
-        {
-          headers: {
-            "Circle-Token": accessToken,
-            "Accept-Encoding": "application/json"
-          }
-        }
-      );
-
-      envVars.push(...res.data.items);
-      nextPageToken = res.data.next_page_token;
-    }
-
-    return envVars;
-  };
-
-  // delete secrets from CircleCI
-  await Promise.all(
-    (await getSecretsRes()).map(async (sec) => {
-      if (!(sec.variable in secrets)) {
-        return request.delete(
-          `${IntegrationUrls.CIRCLECI_API_URL}/v2/context/${integration.appId}/environment-variable/${sec.variable}`,
+  if (integration.scope === CircleCiScope.Context) {
+    // sync secrets to CircleCI
+    await Promise.all(
+      Object.keys(secrets).map(async (key) =>
+        request.put(
+          `${IntegrationUrls.CIRCLECI_API_URL}/v2/context/${integration.appId}/environment-variable/${key}`,
+          {
+            value: secrets[key].value
+          },
           {
             headers: {
               "Circle-Token": accessToken,
               "Content-Type": "application/json"
             }
           }
+        )
+      )
+    );
+
+    // get secrets from CircleCI
+    const getSecretsRes = async () => {
+      type EnvVars = {
+        variable: string;
+        created_at: string;
+        updated_at: string;
+        context_id: string;
+      };
+
+      type ResponseSchema = {
+        items: EnvVars[];
+        next_page_token: string | null;
+      };
+
+      let nextPageToken: string | null | undefined;
+      const envVars: EnvVars[] = [];
+
+      while (nextPageToken !== null) {
+        const res = await request.get<ResponseSchema>(
+          `${IntegrationUrls.CIRCLECI_API_URL}/v2/context/${integration.appId}/environment-variable`,
+          {
+            headers: {
+              "Circle-Token": accessToken,
+              "Accept-Encoding": "application/json"
+            },
+            params: nextPageToken
+              ? new URLSearchParams({
+                  "page-token": nextPageToken
+                })
+              : undefined
+          }
         );
+
+        envVars.push(...res.data.items);
+        nextPageToken = res.data.next_page_token;
       }
-    })
-  );
+
+      return envVars;
+    };
+
+    // delete secrets from CircleCI
+    await Promise.all(
+      (await getSecretsRes()).map(async (sec) => {
+        if (!(sec.variable in secrets)) {
+          return request.delete(
+            `${IntegrationUrls.CIRCLECI_API_URL}/v2/context/${integration.appId}/environment-variable/${sec.variable}`,
+            {
+              headers: {
+                "Circle-Token": accessToken,
+                "Content-Type": "application/json"
+              }
+            }
+          );
+        }
+      })
+    );
+  } else {
+    const getProjectSlug = async () => {
+      const requestConfig = {
+        headers: {
+          "Circle-Token": accessToken,
+          "Accept-Encoding": "application/json"
+        }
+      };
+
+      try {
+        const projectDetails = (
+          await request.get<{ slug: string }>(
+            `${IntegrationUrls.CIRCLECI_API_URL}/v2/project/${integration.appId}`,
+            requestConfig
+          )
+        ).data;
+
+        return projectDetails.slug;
+      } catch (err) {
+        if (err instanceof AxiosError) {
+          if (err.response?.data?.message !== "Not Found") {
+            throw new Error("Failed to get project slug from CircleCI during first attempt.");
+          }
+        }
+      }
+
+      // For backwards compatibility with old CircleCI integrations where we don't keep track of the organization name, so we can't filter by organization
+      try {
+        const circleCiOrganization = (
+          await request.get<{ slug: string; name: string }[]>(
+            `${IntegrationUrls.CIRCLECI_API_URL}/v2/me/collaborations`,
+            requestConfig
+          )
+        ).data;
+
+        // Case 1: This is a new integration where the organization name is stored under `integration.owner`
+        if (integration.owner) {
+          const org = circleCiOrganization.find((o) => o.name === integration.owner);
+          if (org) {
+            return `${org.slug}/${integration.app}`;
+          }
+        }
+
+        // Case 2: This is an old integration where the organization name is not stored, so we have to assume the first organization is the correct one
+        return `${circleCiOrganization[0].slug}/${integration.app}`;
+      } catch (err) {
+        throw new Error("Failed to get project slug from CircleCI during second attempt.");
+      }
+    };
+
+    const projectSlug = await getProjectSlug();
+
+    // sync secrets to CircleCI
+    await Promise.all(
+      Object.keys(secrets).map(async (key) =>
+        request.post(
+          `${IntegrationUrls.CIRCLECI_API_URL}/v2/project/${projectSlug}/envvar`,
+          {
+            name: key,
+            value: secrets[key].value
+          },
+          {
+            headers: {
+              "Circle-Token": accessToken,
+              "Content-Type": "application/json"
+            }
+          }
+        )
+      )
+    );
+
+    // get secrets from CircleCI
+    const getSecretsRes = (
+      await request.get<{ items: { name: string }[] }>(
+        `${IntegrationUrls.CIRCLECI_API_URL}/v2/project/${projectSlug}/envvar`,
+        {
+          headers: {
+            "Circle-Token": accessToken,
+            "Accept-Encoding": "application/json"
+          }
+        }
+      )
+    ).data?.items;
+
+    // delete secrets from CircleCI
+    await Promise.all(
+      getSecretsRes.map(async (sec) => {
+        if (!(sec.name in secrets)) {
+          return request.delete(`${IntegrationUrls.CIRCLECI_API_URL}/v2/project/${projectSlug}/envvar/${sec.name}`, {
+            headers: {
+              "Circle-Token": accessToken,
+              "Content-Type": "application/json"
+            }
+          });
+        }
+      })
+    );
+  }
 };
 
 /**
@@ -4516,13 +4515,6 @@ export const syncIntegrationSecrets = async ({
         accessToken
       });
       break;
-    case Integrations.CIRCLECI_CONTEXT:
-      await syncSecretsCircleCIContext({
-        integration,
-        secrets,
-        accessToken
-      });
-      break;
     case Integrations.DATABRICKS:
       await syncSecretsDatabricks({
         integration,