From b7d683ee1bc0ade2b19371a7a3bb4b041f767739 Mon Sep 17 00:00:00 2001 From: Daniel Hougaard Date: Thu, 5 Sep 2024 13:30:20 +0400 Subject: [PATCH 1/4] fix(integrations/circle-ci): Refactored Circle CI integration The integration seemingly never worked in the first place due to inpropper project slugs. This PR resolves it. --- .../integration-auth/integration-app-list.ts | 21 +++++--- .../integration-auth-types.ts | 6 +++ .../integration-sync-secret.ts | 52 ++++++++++++++----- .../pages/integrations/circleci/create.tsx | 48 +++++++++-------- .../ConfiguredIntegrationItem.tsx | 1 + 5 files changed, 84 insertions(+), 44 deletions(-) diff --git a/backend/src/services/integration-auth/integration-app-list.ts b/backend/src/services/integration-auth/integration-app-list.ts index bd7c19228..2c518db1b 100644 --- a/backend/src/services/integration-auth/integration-app-list.ts +++ b/backend/src/services/integration-auth/integration-app-list.ts @@ -459,17 +459,22 @@ const getAppsFlyio = async ({ accessToken }: { accessToken: string }) => { * Return list of projects for CircleCI integration */ const getAppsCircleCI = async ({ accessToken }: { accessToken: string }) => { - const res = ( - await request.get<{ reponame: string }[]>(`${IntegrationUrls.CIRCLECI_API_URL}/v1.1/projects`, { - headers: { - "Circle-Token": accessToken, - "Accept-Encoding": "application/json" + // Fetch collaborations (v2 API) + const collaborations = ( + await request.get<{ id: string; name: string; slug: string }[]>( + `${IntegrationUrls.CIRCLECI_API_URL}/v2/me/collaborations`, + { + headers: { + "Circle-Token": accessToken, + "Accept-Encoding": "application/json" + } } - }) + ) ).data; - const apps = res?.map((a) => ({ - name: a?.reponame + const apps = collaborations.map((a) => ({ + name: a.name, + appId: a.id })); return apps; diff --git a/backend/src/services/integration-auth/integration-auth-types.ts b/backend/src/services/integration-auth/integration-auth-types.ts index af390297a..7d59fda27 100644 --- a/backend/src/services/integration-auth/integration-auth-types.ts +++ b/backend/src/services/integration-auth/integration-auth-types.ts @@ -143,6 +143,12 @@ export type TBitbucketWorkspace = { updated_on: string; }; +export enum CircleCiVcsType { + GitHub = "GitHub", + CircleCI = "CircleCI", + BitBucket = "BitBucket" +} + export type TNorthflankSecretGroup = { id: string; name: string; diff --git a/backend/src/services/integration-auth/integration-sync-secret.ts b/backend/src/services/integration-auth/integration-sync-secret.ts index 94d7a95d6..1418c66e5 100644 --- a/backend/src/services/integration-auth/integration-sync-secret.ts +++ b/backend/src/services/integration-auth/integration-sync-secret.ts @@ -35,7 +35,7 @@ import { TCreateManySecretsRawFn, TUpdateManySecretsRawFn } from "@app/services/ import { TIntegrationDALFactory } from "../integration/integration-dal"; import { IntegrationMetadataSchema } from "../integration/integration-schema"; -import { TIntegrationsWithEnvironment } from "./integration-auth-types"; +import { CircleCiVcsType, TIntegrationsWithEnvironment } from "./integration-auth-types"; import { IntegrationInitialSyncBehavior, IntegrationMappingBehavior, @@ -1930,21 +1930,48 @@ const syncSecretsCircleCI = async ({ accessToken: string; }) => { const circleciOrganizationDetail = ( - await request.get(`${IntegrationUrls.CIRCLECI_API_URL}/v2/me/collaborations`, { + await request.get<{ slug: string; name: string }[]>(`${IntegrationUrls.CIRCLECI_API_URL}/v2/me/collaborations`, { headers: { "Circle-Token": accessToken, "Accept-Encoding": "application/json" } }) - ).data[0]; + ).data; - const { slug } = circleciOrganizationDetail; + let orgSlug: string | null = null; + if (!integration.owner) { + orgSlug = `${circleciOrganizationDetail[0].slug}/${integration.app}`; + } else { + const projectDetails = ( + await request.get<{ vcs_info: { provider: CircleCiVcsType } }>( + `${IntegrationUrls.CIRCLECI_API_URL}/v2/project/${integration.app}`, + { + headers: { + "Circle-Token": accessToken, + "Accept-Encoding": "application/json" + } + } + ) + ).data; + + const vcsProviderMap: Record = { + [CircleCiVcsType.GitHub]: "gh", + [CircleCiVcsType.BitBucket]: "bb", + [CircleCiVcsType.CircleCI]: "circleci" + }; + + orgSlug = `${vcsProviderMap[projectDetails.vcs_info.provider]}/${integration.owner}/${integration.app}`; + + if (!orgSlug) { + throw new Error("CircleCI: Organization not found"); + } + } // sync secrets to CircleCI await Promise.all( Object.keys(secrets).map(async (key) => request.post( - `${IntegrationUrls.CIRCLECI_API_URL}/v2/project/${slug}/${integration.app}/envvar`, + `${IntegrationUrls.CIRCLECI_API_URL}/v2/project/${orgSlug}/envvar`, { name: key, value: secrets[key].value @@ -1962,7 +1989,7 @@ const syncSecretsCircleCI = async ({ // get secrets from CircleCI const getSecretsRes = ( await request.get<{ items: { name: string }[] }>( - `${IntegrationUrls.CIRCLECI_API_URL}/v2/project/${slug}/${integration.app}/envvar`, + `${IntegrationUrls.CIRCLECI_API_URL}/v2/project/${orgSlug}/envvar`, { headers: { "Circle-Token": accessToken, @@ -1976,15 +2003,12 @@ const syncSecretsCircleCI = async ({ await Promise.all( getSecretsRes.map(async (sec) => { if (!(sec.name in secrets)) { - return request.delete( - `${IntegrationUrls.CIRCLECI_API_URL}/v2/project/${slug}/${integration.app}/envvar/${sec.name}`, - { - headers: { - "Circle-Token": accessToken, - "Content-Type": "application/json" - } + return request.delete(`${IntegrationUrls.CIRCLECI_API_URL}/v2/project/${orgSlug}/envvar/${sec.name}`, { + headers: { + "Circle-Token": accessToken, + "Content-Type": "application/json" } - ); + }); } }) ); diff --git a/frontend/src/pages/integrations/circleci/create.tsx b/frontend/src/pages/integrations/circleci/create.tsx index 117fd8444..6c734b64a 100644 --- a/frontend/src/pages/integrations/circleci/create.tsx +++ b/frontend/src/pages/integrations/circleci/create.tsx @@ -45,6 +45,7 @@ export default function CircleCICreateIntegrationPage() { }); const [selectedSourceEnvironment, setSelectedSourceEnvironment] = useState(""); + const [targetOrganization, setTargetOrganization] = useState(""); const [secretPath, setSecretPath] = useState("/"); const [targetApp, setTargetApp] = useState(""); @@ -57,26 +58,19 @@ export default function CircleCICreateIntegrationPage() { } }, [workspace]); - useEffect(() => { - if (integrationAuthApps) { - if (integrationAuthApps.length > 0) { - setTargetApp(integrationAuthApps[0]?.name); - } else { - setTargetApp("none"); - } - } - }, [integrationAuthApps]); - const handleButtonClick = async () => { try { if (!integrationAuth?.id) return; + if (!targetApp || targetOrganization === "none") return; + setIsLoading(true); await mutateAsync({ integrationAuthId: integrationAuth?.id, isActive: true, app: targetApp, + owner: targetOrganization, appId: integrationAuthApps?.find( (integrationAuthApp) => integrationAuthApp.name === targetApp )?.appId, @@ -92,11 +86,7 @@ export default function CircleCICreateIntegrationPage() { } }; - return integrationAuth && - workspace && - selectedSourceEnvironment && - integrationAuthApps && - targetApp ? ( + return integrationAuth && workspace && selectedSourceEnvironment && integrationAuthApps ? (
Set Up CircleCI Integration @@ -108,7 +98,7 @@ export default function CircleCICreateIntegrationPage() { subTitle="Choose which environment or folder in Infisical you want to sync to CircleCI environment variables." >
-
+
+ setTargetApp(val)} + value={targetOrganization} + onValueChange={(val) => { + setTargetOrganization(val); + }} className="w-full border border-mineshaft-500" isDisabled={integrationAuthApps.length === 0} > {integrationAuthApps.length > 0 ? ( integrationAuthApps.map((integrationAuthApp) => ( {integrationAuthApp.name} )) ) : ( - No projects found + No organizations found )} + + {targetOrganization && ( + + setTargetApp(evt.target.value)} + /> + + )}
)} + {integration.integration === "circleci" && integration.owner && ( +
+ +
+ {integration.owner} +
+
+ )} {integration.integration === "terraform-cloud" && integration.targetService && (
From da561e37c56a68ce02348b7a8997a5567f75935f Mon Sep 17 00:00:00 2001 From: Daniel Hougaard Date: Thu, 5 Sep 2024 21:43:10 +0400 Subject: [PATCH 4/4] Fix: Backwards compatibility and UI fixes --- .../integration-sync-secret.ts | 51 ++++++++++++++++--- .../pages/integrations/circleci/create.tsx | 28 +++++++--- 2 files changed, 65 insertions(+), 14 deletions(-) diff --git a/backend/src/services/integration-auth/integration-sync-secret.ts b/backend/src/services/integration-auth/integration-sync-secret.ts index eb8645530..d0fa2279a 100644 --- a/backend/src/services/integration-auth/integration-sync-secret.ts +++ b/backend/src/services/integration-auth/integration-sync-secret.ts @@ -1929,17 +1929,56 @@ const syncSecretsCircleCI = async ({ secrets: Record; accessToken: string; }) => { - let projectSlug: string | null = null; - const projectDetails = ( - await request.get<{ slug: string }>(`${IntegrationUrls.CIRCLECI_API_URL}/v2/project/${integration.appId}`, { + const getProjectSlug = async () => { + const requestConfig = { headers: { "Circle-Token": accessToken, "Accept-Encoding": "application/json" } - }) - ).data; + }; - projectSlug = `${projectDetails.slug}`; + try { + const projectDetails = ( + await request.get<{ slug: string }>( + `${IntegrationUrls.CIRCLECI_API_URL}/v2/project/${integration.appId}`, + requestConfig + ) + ).data; + + return projectDetails.slug; + } catch (err) { + if (err instanceof AxiosError) { + if (err.response?.data?.message !== "Not Found") { + throw new Error("Failed to get project slug from CircleCI during first attempt."); + } + } + } + + // For backwards compatibility with old CircleCI integrations where we don't keep track of the organization name, so we can't filter by organization + try { + const circleCiOrganization = ( + await request.get<{ slug: string; name: string }[]>( + `${IntegrationUrls.CIRCLECI_API_URL}/v2/me/collaborations`, + requestConfig + ) + ).data; + + // Case 1: This is a new integration where the organization name is stored under `integration.owner` + if (integration.owner) { + const org = circleCiOrganization.find((o) => o.name === integration.owner); + if (org) { + return `${org.slug}/${integration.app}`; + } + } + + // Case 2: This is an old integration where the organization name is not stored, so we have to assume the first organization is the correct one + return `${circleCiOrganization[0].slug}/${integration.app}`; + } catch (err) { + throw new Error("Failed to get project slug from CircleCI during second attempt."); + } + }; + + const projectSlug = await getProjectSlug(); // sync secrets to CircleCI await Promise.all( diff --git a/frontend/src/pages/integrations/circleci/create.tsx b/frontend/src/pages/integrations/circleci/create.tsx index bb3541875..edd8dacfe 100644 --- a/frontend/src/pages/integrations/circleci/create.tsx +++ b/frontend/src/pages/integrations/circleci/create.tsx @@ -68,6 +68,7 @@ export default function CircleCICreateIntegrationPage() { type: "error", text: "Please select a project" }); + setIsLoading(false); return; } @@ -82,6 +83,7 @@ export default function CircleCICreateIntegrationPage() { type: "error", text: "Invalid project selected" }); + setIsLoading(false); return; } @@ -111,6 +113,19 @@ export default function CircleCICreateIntegrationPage() { }); }, [integrationAuthApps, targetOrganization]); + const filteredOrganizations = useMemo(() => { + const organizations = new Set(); + + if (integrationAuthApps) { + integrationAuthApps.forEach((integrationAuthApp) => { + if (!integrationAuthApp.owner) return; + organizations.add(integrationAuthApp.owner); + }); + } + + return Array.from(organizations); + }, [integrationAuthApps]); + return integrationAuth && workspace && selectedSourceEnvironment && integrationAuthApps ? (
@@ -179,15 +194,12 @@ export default function CircleCICreateIntegrationPage() { setTargetProjectId("none"); }} className="w-full border border-mineshaft-500" - isDisabled={integrationAuthApps.length === 0} + isDisabled={filteredOrganizations.length === 0} > - {integrationAuthApps.length > 0 ? ( - integrationAuthApps.map((integrationAuthApp) => ( - - {integrationAuthApp.owner} + {filteredOrganizations.length > 0 ? ( + filteredOrganizations.map((org) => ( + + {org} )) ) : (