Merge pull request #2883 from Infisical/feat/sync-circle-ci-context

feat: circle ci context integration

backend/src/server/routes/v1/integration-auth-router.ts

@@ -1185,4 +1185,50 @@ export const registerIntegrationAuthRouter = async (server: FastifyZodProvider)
       return { spaces };
     }
   });
+
+  server.route({
+    method: "GET",
+    url: "/:integrationAuthId/circleci/organizations",
+    config: {
+      rateLimit: readLimit
+    },
+    onRequest: verifyAuth([AuthMode.JWT]),
+    schema: {
+      params: z.object({
+        integrationAuthId: z.string().trim()
+      }),
+      response: {
+        200: z.object({
+          organizations: z
+            .object({
+              name: z.string(),
+              slug: z.string(),
+              projects: z
+                .object({
+                  name: z.string(),
+                  id: z.string()
+                })
+                .array(),
+              contexts: z
+                .object({
+                  name: z.string(),
+                  id: z.string()
+                })
+                .array()
+            })
+            .array()
+        })
+      }
+    },
+    handler: async (req) => {
+      const organizations = await server.services.integrationAuth.getCircleCIOrganizations({
+        actorId: req.permission.id,
+        actor: req.permission.type,
+        actorAuthMethod: req.permission.authMethod,
+        actorOrgId: req.permission.orgId,
+        id: req.params.integrationAuthId
+      });
+      return { organizations };
+    }
+  });
 };

backend/src/services/integration-auth/integration-app-types.ts

@@ -0,0 +1,5 @@
+export type TCircleCIContext = {
+  id: string;
+  name: string;
+  created_at: string;
+};

backend/src/services/integration-auth/integration-auth-service.ts

@@ -17,6 +17,8 @@ import { getConfig } from "@app/lib/config/env";
 import { request } from "@app/lib/config/request";
 import { decryptSymmetric128BitHexKeyUTF8, encryptSymmetric128BitHexKeyUTF8 } from "@app/lib/crypto";
 import { BadRequestError, InternalServerError, NotFoundError } from "@app/lib/errors";
+import { groupBy } from "@app/lib/fn";
+import { logger } from "@app/lib/logger";
 import { TGenericPermission, TProjectPermission } from "@app/lib/types";
 
 import { TIntegrationDALFactory } from "../integration/integration-dal";
@@ -24,6 +26,7 @@ import { TKmsServiceFactory } from "../kms/kms-service";
 import { KmsDataKey } from "../kms/kms-types";
 import { TProjectBotServiceFactory } from "../project-bot/project-bot-service";
 import { getApps } from "./integration-app-list";
+import { TCircleCIContext } from "./integration-app-types";
 import { TIntegrationAuthDALFactory } from "./integration-auth-dal";
 import { IntegrationAuthMetadataSchema, TIntegrationAuthMetadata } from "./integration-auth-schema";
 import {
@@ -31,6 +34,7 @@ import {
   TBitbucketEnvironment,
   TBitbucketWorkspace,
   TChecklyGroups,
+  TCircleCIOrganization,
   TDeleteIntegrationAuthByIdDTO,
   TDeleteIntegrationAuthsDTO,
   TDuplicateGithubIntegrationAuthDTO,
@@ -42,6 +46,7 @@ import {
   TIntegrationAuthBitbucketEnvironmentsDTO,
   TIntegrationAuthBitbucketWorkspaceDTO,
   TIntegrationAuthChecklyGroupsDTO,
+  TIntegrationAuthCircleCIOrganizationDTO,
   TIntegrationAuthGithubEnvsDTO,
   TIntegrationAuthGithubOrgsDTO,
   TIntegrationAuthHerokuPipelinesDTO,
@@ -1578,6 +1583,120 @@ export const integrationAuthServiceFactory = ({
     return [];
   };
 
+  const getCircleCIOrganizations = async ({
+    actorId,
+    actor,
+    actorOrgId,
+    actorAuthMethod,
+    id
+  }: TIntegrationAuthCircleCIOrganizationDTO) => {
+    const integrationAuth = await integrationAuthDAL.findById(id);
+    if (!integrationAuth) throw new NotFoundError({ message: `Integration auth with ID '${id}' not found` });
+
+    const { permission } = await permissionService.getProjectPermission(
+      actor,
+      actorId,
+      integrationAuth.projectId,
+      actorAuthMethod,
+      actorOrgId
+    );
+    ForbiddenError.from(permission).throwUnlessCan(ProjectPermissionActions.Read, ProjectPermissionSub.Integrations);
+    const { shouldUseSecretV2Bridge, botKey } = await projectBotService.getBotKey(integrationAuth.projectId);
+    const { accessToken } = await getIntegrationAccessToken(integrationAuth, shouldUseSecretV2Bridge, botKey);
+
+    const { data: organizations }: { data: TCircleCIOrganization[] } = await request.get(
+      `${IntegrationUrls.CIRCLECI_API_URL}/v2/me/collaborations`,
+      {
+        headers: {
+          "Circle-Token": `${accessToken}`,
+          "Accept-Encoding": "application/json"
+        }
+      }
+    );
+
+    let projects: {
+      orgName: string;
+      projectName: string;
+      projectId?: string;
+    }[] = [];
+
+    try {
+      const projectRes = (
+        await request.get<{ reponame: string; username: string; vcs_url: string }[]>(
+          `${IntegrationUrls.CIRCLECI_API_URL}/v1.1/projects`,
+          {
+            headers: {
+              "Circle-Token": accessToken,
+              "Accept-Encoding": "application/json"
+            }
+          }
+        )
+      ).data;
+
+      projects = projectRes.map((a) => ({
+        orgName: a.username, // username maps to unique organization name in CircleCI
+        projectName: a.reponame, // reponame maps to project name within an organization in CircleCI
+        projectId: a.vcs_url.split("/").pop() // vcs_url maps to the project id in CircleCI
+      }));
+    } catch (error) {
+      logger.error(error);
+    }
+
+    const projectsByOrg = groupBy(
+      projects.map((p) => ({
+        orgName: p.orgName,
+        name: p.projectName,
+        id: p.projectId as string
+      })),
+      (p) => p.orgName
+    );
+
+    const getOrgContexts = async (orgSlug: string) => {
+      type NextPageToken = string | null | undefined;
+
+      try {
+        const contexts: TCircleCIContext[] = [];
+        let nextPageToken: NextPageToken;
+
+        while (nextPageToken !== null) {
+          // eslint-disable-next-line no-await-in-loop
+          const { data } = await request.get<{
+            items: TCircleCIContext[];
+            next_page_token: NextPageToken;
+          }>(`${IntegrationUrls.CIRCLECI_API_URL}/v2/context`, {
+            headers: {
+              "Circle-Token": accessToken,
+              "Accept-Encoding": "application/json"
+            },
+            params: new URLSearchParams({
+              "owner-slug": orgSlug,
+              ...(nextPageToken ? { "page-token": nextPageToken } : {})
+            })
+          });
+
+          contexts.push(...data.items);
+          nextPageToken = data.next_page_token;
+        }
+
+        return contexts?.map((context) => ({
+          name: context.name,
+          id: context.id
+        }));
+      } catch (error) {
+        logger.error(error);
+      }
+    };
+
+    return Promise.all(
+      organizations.map(async (org) => ({
+        name: org.name,
+        slug: org.slug,
+        projects: projectsByOrg[org.name] ?? [],
+        contexts: (await getOrgContexts(org.slug)) ?? []
+      }))
+    );
+  };
+
   const deleteIntegrationAuths = async ({
     projectId,
     integration,
@@ -1790,6 +1909,7 @@ export const integrationAuthServiceFactory = ({
     getTeamcityBuildConfigs,
     getBitbucketWorkspaces,
     getBitbucketEnvironments,
+    getCircleCIOrganizations,
     getIntegrationAccessToken,
     duplicateIntegrationAuth,
     getOctopusDeploySpaces,

backend/src/services/integration-auth/integration-auth-types.ts

@@ -128,6 +128,10 @@ export type TGetIntegrationAuthTeamCityBuildConfigDTO = {
   appId: string;
 } & Omit<TProjectPermission, "projectId">;
 
+export type TIntegrationAuthCircleCIOrganizationDTO = {
+  id: string;
+} & Omit<TProjectPermission, "projectId">;
+
 export type TVercelBranches = {
   ref: string;
   lastCommit: string;
@@ -189,6 +193,14 @@ export type TTeamCityBuildConfig = {
   webUrl: string;
 };
 
+export type TCircleCIOrganization = {
+  id: string;
+  vcsType: string;
+  name: string;
+  avatarUrl: string;
+  slug: string;
+};
+
 export type TIntegrationsWithEnvironment = TIntegrations & {
   environment?:
     | {
@@ -215,6 +227,11 @@ export enum OctopusDeployScope {
   // add tenant, variable set, etc.
 }
 
+export enum CircleCiScope {
+  Project = "project",
+  Context = "context"
+}
+
 export type TOctopusDeployVariableSet = {
   Id: string;
   OwnerId: string;

backend/src/services/integration-auth/integration-list.ts

@@ -76,7 +76,6 @@ export enum IntegrationUrls {
   RAILWAY_API_URL = "https://backboard.railway.app/graphql/v2",
   FLYIO_API_URL = "https://api.fly.io/graphql",
   CIRCLECI_API_URL = "https://circleci.com/api",
-  DATABRICKS_API_URL = "https:/xxxx.com/api",
   TRAVISCI_API_URL = "https://api.travis-ci.com",
   SUPABASE_API_URL = "https://api.supabase.com",
   LARAVELFORGE_API_URL = "https://forge.laravel.com",
@@ -218,9 +217,9 @@ export const getIntegrationOptions = async () => {
       docsLink: ""
     },
     {
-      name: "Circle CI",
+      name: "CircleCI",
       slug: "circleci",
-      image: "Circle CI.png",
+      image: "CircleCI.png",
       isAvailable: true,
       type: "pat",
       clientId: "",

backend/src/services/integration-auth/integration-sync-secret.ts

@@ -39,7 +39,12 @@ import { TCreateManySecretsRawFn, TUpdateManySecretsRawFn } from "@app/services/
 import { TIntegrationDALFactory } from "../integration/integration-dal";
 import { IntegrationMetadataSchema } from "../integration/integration-schema";
 import { IntegrationAuthMetadataSchema } from "./integration-auth-schema";
-import { OctopusDeployScope, TIntegrationsWithEnvironment, TOctopusDeployVariableSet } from "./integration-auth-types";
+import {
+  CircleCiScope,
+  OctopusDeployScope,
+  TIntegrationsWithEnvironment,
+  TOctopusDeployVariableSet
+} from "./integration-auth-types";
 import {
   IntegrationInitialSyncBehavior,
   IntegrationMappingBehavior,
@@ -2245,102 +2250,174 @@ const syncSecretsCircleCI = async ({
   secrets: Record<string, { value: string; comment?: string }>;
   accessToken: string;
 }) => {
-  const getProjectSlug = async () => {
-    const requestConfig = {
-      headers: {
-        "Circle-Token": accessToken,
-        "Accept-Encoding": "application/json"
-      }
-    };
-
-    try {
-      const projectDetails = (
-        await request.get<{ slug: string }>(
-          `${IntegrationUrls.CIRCLECI_API_URL}/v2/project/${integration.appId}`,
-          requestConfig
+  if (integration.scope === CircleCiScope.Context) {
+    // sync secrets to CircleCI
+    await Promise.all(
+      Object.keys(secrets).map(async (key) =>
+        request.put(
+          `${IntegrationUrls.CIRCLECI_API_URL}/v2/context/${integration.appId}/environment-variable/${key}`,
+          {
+            value: secrets[key].value
+          },
+          {
+            headers: {
+              "Circle-Token": accessToken,
+              "Content-Type": "application/json"
+            }
+          }
         )
-      ).data;
+      )
+    );
 
-      return projectDetails.slug;
-    } catch (err) {
-      if (err instanceof AxiosError) {
-        if (err.response?.data?.message !== "Not Found") {
-          throw new Error("Failed to get project slug from CircleCI during first attempt.");
-        }
-      }
-    }
+    // get secrets from CircleCI
+    const getSecretsRes = async () => {
+      type EnvVars = {
+        variable: string;
+        created_at: string;
+        updated_at: string;
+        context_id: string;
+      };
 
-    // For backwards compatibility with old CircleCI integrations where we don't keep track of the organization name, so we can't filter by organization
-    try {
-      const circleCiOrganization = (
-        await request.get<{ slug: string; name: string }[]>(
-          `${IntegrationUrls.CIRCLECI_API_URL}/v2/me/collaborations`,
-          requestConfig
-        )
-      ).data;
+      let nextPageToken: string | null | undefined;
+      const envVars: EnvVars[] = [];
 
-      // Case 1: This is a new integration where the organization name is stored under `integration.owner`
-      if (integration.owner) {
-        const org = circleCiOrganization.find((o) => o.name === integration.owner);
-        if (org) {
-          return `${org.slug}/${integration.app}`;
-        }
-      }
-
-      // Case 2: This is an old integration where the organization name is not stored, so we have to assume the first organization is the correct one
-      return `${circleCiOrganization[0].slug}/${integration.app}`;
-    } catch (err) {
-      throw new Error("Failed to get project slug from CircleCI during second attempt.");
-    }
-  };
-
-  const projectSlug = await getProjectSlug();
-
-  // sync secrets to CircleCI
-  await Promise.all(
-    Object.keys(secrets).map(async (key) =>
-      request.post(
-        `${IntegrationUrls.CIRCLECI_API_URL}/v2/project/${projectSlug}/envvar`,
-        {
-          name: key,
-          value: secrets[key].value
-        },
-        {
+      while (nextPageToken !== null) {
+        const res = await request.get<{
+          items: EnvVars[];
+          next_page_token: string | null;
+        }>(`${IntegrationUrls.CIRCLECI_API_URL}/v2/context/${integration.appId}/environment-variable`, {
           headers: {
             "Circle-Token": accessToken,
-            "Content-Type": "application/json"
-          }
-        }
-      )
-    )
-  );
+            "Accept-Encoding": "application/json"
+          },
+          params: nextPageToken
+            ? new URLSearchParams({
+                "page-token": nextPageToken
+              })
+            : undefined
+        });
 
-  // get secrets from CircleCI
-  const getSecretsRes = (
-    await request.get<{ items: { name: string }[] }>(
-      `${IntegrationUrls.CIRCLECI_API_URL}/v2/project/${projectSlug}/envvar`,
-      {
+        envVars.push(...res.data.items);
+        nextPageToken = res.data.next_page_token;
+      }
+
+      return envVars;
+    };
+
+    // delete secrets from CircleCI
+    await Promise.all(
+      (await getSecretsRes()).map(async (sec) => {
+        if (!(sec.variable in secrets)) {
+          return request.delete(
+            `${IntegrationUrls.CIRCLECI_API_URL}/v2/context/${integration.appId}/environment-variable/${sec.variable}`,
+            {
+              headers: {
+                "Circle-Token": accessToken,
+                "Content-Type": "application/json"
+              }
+            }
+          );
+        }
+      })
+    );
+  } else {
+    const getProjectSlug = async () => {
+      const requestConfig = {
         headers: {
           "Circle-Token": accessToken,
           "Accept-Encoding": "application/json"
         }
-      }
-    )
-  ).data?.items;
+      };
 
-  // delete secrets from CircleCI
-  await Promise.all(
-    getSecretsRes.map(async (sec) => {
-      if (!(sec.name in secrets)) {
-        return request.delete(`${IntegrationUrls.CIRCLECI_API_URL}/v2/project/${projectSlug}/envvar/${sec.name}`, {
+      try {
+        const projectDetails = (
+          await request.get<{ slug: string }>(
+            `${IntegrationUrls.CIRCLECI_API_URL}/v2/project/${integration.appId}`,
+            requestConfig
+          )
+        ).data;
+
+        return projectDetails.slug;
+      } catch (err) {
+        if (err instanceof AxiosError) {
+          if (err.response?.data?.message !== "Not Found") {
+            throw new Error("Failed to get project slug from CircleCI during first attempt.");
+          }
+        }
+      }
+
+      // For backwards compatibility with old CircleCI integrations where we don't keep track of the organization name, so we can't filter by organization
+      try {
+        const circleCiOrganization = (
+          await request.get<{ slug: string; name: string }[]>(
+            `${IntegrationUrls.CIRCLECI_API_URL}/v2/me/collaborations`,
+            requestConfig
+          )
+        ).data;
+
+        // Case 1: This is a new integration where the organization name is stored under `integration.owner`
+        if (integration.owner) {
+          const org = circleCiOrganization.find((o) => o.name === integration.owner);
+          if (org) {
+            return `${org.slug}/${integration.app}`;
+          }
+        }
+
+        // Case 2: This is an old integration where the organization name is not stored, so we have to assume the first organization is the correct one
+        return `${circleCiOrganization[0].slug}/${integration.app}`;
+      } catch (err) {
+        throw new Error("Failed to get project slug from CircleCI during second attempt.");
+      }
+    };
+
+    const projectSlug = await getProjectSlug();
+
+    // sync secrets to CircleCI
+    await Promise.all(
+      Object.keys(secrets).map(async (key) =>
+        request.post(
+          `${IntegrationUrls.CIRCLECI_API_URL}/v2/project/${projectSlug}/envvar`,
+          {
+            name: key,
+            value: secrets[key].value
+          },
+          {
+            headers: {
+              "Circle-Token": accessToken,
+              "Content-Type": "application/json"
+            }
+          }
+        )
+      )
+    );
+
+    // get secrets from CircleCI
+    const getSecretsRes = (
+      await request.get<{ items: { name: string }[] }>(
+        `${IntegrationUrls.CIRCLECI_API_URL}/v2/project/${projectSlug}/envvar`,
+        {
           headers: {
             "Circle-Token": accessToken,
-            "Content-Type": "application/json"
+            "Accept-Encoding": "application/json"
           }
-        });
-      }
-    })
-  );
+        }
+      )
+    ).data?.items;
+
+    // delete secrets from CircleCI
+    await Promise.all(
+      getSecretsRes.map(async (sec) => {
+        if (!(sec.name in secrets)) {
+          return request.delete(`${IntegrationUrls.CIRCLECI_API_URL}/v2/project/${projectSlug}/envvar/${sec.name}`, {
+            headers: {
+              "Circle-Token": accessToken,
+              "Content-Type": "application/json"
+            }
+          });
+        }
+      })
+    );
+  }
 };
 
 /**