General improvements to Terraform Integration

backend/src/lib/api-docs/constants.ts

@@ -1694,6 +1694,9 @@ export const AppConnections = {
       sslEnabled: "Whether or not to use SSL when connecting to the database.",
       sslRejectUnauthorized: "Whether or not to reject unauthorized SSL certificates.",
       sslCertificate: "The SSL certificate to use for connection."
+    },
+    TERRAFORM_CLOUD: {
+      apiToken: "The API token to use to connect with Terraform Cloud."
     }
   }
 };

backend/src/services/app-connection/terraform-cloud/terraform-cloud-connection-fns.ts

@@ -64,62 +64,52 @@ export const listOrganizations = async (
     credentials: { apiToken }
   } = appConnection;
 
-  const orgsResponse = await request.get<{ data: { id: string; attributes: { name: string } }[] }>(
-    `${IntegrationUrls.TERRAFORM_CLOUD_API_URL}/api/v2/organizations`,
-    {
-      headers: {
-        Authorization: `Bearer ${apiToken}`,
-        "Content-Type": "application/vnd.api+json"
-      }
+  const headers = {
+    Authorization: `Bearer ${apiToken}`,
+    "Content-Type": "application/vnd.api+json"
+  };
+
+  const fetchAllPages = async <T>(url: string): Promise<T[]> => {
+    let results: T[] = [];
+    let nextUrl: string | null = url;
+
+    while (nextUrl) {
+      // eslint-disable-next-line no-await-in-loop
+      const res: AxiosResponse<{ data: T[]; links?: { next?: string } }> = await request.get(nextUrl, { headers });
+      results = results.concat(res.data.data);
+      nextUrl = res.data.links?.next || null;
     }
+
+    return results;
+  };
+
+  const orgEntities = await fetchAllPages<{ id: string; attributes: { name: string } }>(
+    `${IntegrationUrls.TERRAFORM_CLOUD_API_URL}/api/v2/organizations`
   );
 
-  if (!orgsResponse.data?.data) {
-    throw new InternalServerError({
-      message: "Failed to get organizations: Response was empty"
-    });
-  }
-
-  const orgEntities = orgsResponse.data.data;
   const orgsWithVariableSetsAndWorkspaces: TTerraformCloudOrganization[] = [];
 
   const variableSetPromises = orgEntities.map((org) =>
-    request
-      .get<{ data: { id: string; attributes: { name: string; description?: string; global?: boolean } }[] }>(
-        `${IntegrationUrls.TERRAFORM_CLOUD_API_URL}/api/v2/organizations/${org.id}/varsets`,
-        {
-          headers: {
-            Authorization: `Bearer ${apiToken}`,
-            "Content-Type": "application/vnd.api+json"
-          }
-        }
-      )
-      .catch(() => ({ data: { data: [] } }))
+    fetchAllPages<{ id: string; attributes: { name: string; description?: string; global?: boolean } }>(
+      `${IntegrationUrls.TERRAFORM_CLOUD_API_URL}/api/v2/organizations/${org.id}/varsets`
+    ).catch(() => [])
   );
 
   const workspacePromises = orgEntities.map((org) =>
-    request
-      .get<{ data: { id: string; attributes: { name: string } }[] }>(
-        `${IntegrationUrls.TERRAFORM_CLOUD_API_URL}/api/v2/organizations/${org.id}/workspaces`,
-        {
-          headers: {
-            Authorization: `Bearer ${apiToken}`,
-            "Content-Type": "application/vnd.api+json"
-          }
-        }
-      )
-      .catch(() => ({ data: { data: [] } }))
+    fetchAllPages<{ id: string; attributes: { name: string } }>(
+      `${IntegrationUrls.TERRAFORM_CLOUD_API_URL}/api/v2/organizations/${org.id}/workspaces`
+    ).catch(() => [])
   );
 
-  const [variableSetResponses, workspaceResponses] = await Promise.all([
+  const [variableSetResults, workspaceResults] = await Promise.all([
     Promise.all(variableSetPromises),
     Promise.all(workspacePromises)
   ]);
 
   for (let i = 0; i < orgEntities.length; i += 1) {
     const org = orgEntities[i];
-    const variableSetsData = variableSetResponses[i].data?.data || [];
-    const workspacesData = workspaceResponses[i].data?.data || [];
+    const variableSetsData = variableSetResults[i];
+    const workspacesData = workspaceResults[i];
 
     const variableSets: TTerraformCloudVariableSet[] = variableSetsData.map((varSet) => ({
       id: varSet.id,

backend/src/services/app-connection/terraform-cloud/terraform-cloud-connection-types.ts

@@ -25,21 +25,6 @@ export type TTerraformCloudConnectionConfig = DiscriminativePick<
   orgId: string;
 };
 
-export type TerraformCloudOrg = {
-  id: string;
-  name: string;
-};
-
-export type TerraformCloudApp = {
-  name: string;
-  id: string;
-  envs: { name: string; id: string }[];
-};
-
-export type TerraformCloudOrgWithApps = TerraformCloudOrg & {
-  apps: TerraformCloudApp[];
-};
-
 export type TTerraformCloudVariableSet = {
   id: string;
   name: string;
@@ -58,12 +43,3 @@ export type TTerraformCloudOrganization = {
   variableSets: TTerraformCloudVariableSet[];
   workspaces: TTerraformCloudWorkspace[];
 };
-
-export type TTerraformCloudConnectionOrganization = TTerraformCloudOrganization;
-export type TTerraformCloudConnectionVariableSet = TTerraformCloudVariableSet;
-export type TTerraformCloudConnectionWorkspace = TTerraformCloudWorkspace;
-
-export enum TerraformCloudSyncScope {
-  VariableSet = "variableSet",
-  Workspace = "workspace"
-}

backend/src/services/secret-sync/terraform-cloud/terraform-cloud-sync-fns.ts

@@ -1,6 +1,5 @@
 /* eslint-disable no-await-in-loop */
 import { request } from "@app/lib/config/request";
-import { logger } from "@app/lib/logger";
 import { IntegrationUrls } from "@app/services/integration-auth/integration-list";
 import { SecretSyncError } from "@app/services/secret-sync/secret-sync-errors";
 import { TSecretMap } from "@app/services/secret-sync/secret-sync-types";
@@ -13,6 +12,7 @@ import {
   TerraformCloudVariable,
   TTerraformCloudSyncWithCredentials
 } from "./terraform-cloud-sync-types";
+import { AxiosResponse } from "axios";
 
 const getTerraformCloudVariables = async (
   secretSync: TTerraformCloudSyncWithCredentials
@@ -24,7 +24,7 @@ const getTerraformCloudVariables = async (
     }
   } = secretSync;
 
-  let url;
+  let url: string;
   let source: TerraformCloudVariable["source"];
 
   if (destinationConfig.scope === TerraformCloudSyncScope.VariableSet) {
@@ -35,18 +35,35 @@ const getTerraformCloudVariables = async (
     source = "workspace";
   }
 
-  const response = await request.get<TerraformCloudApiResponse<TerraformCloudApiVariable[]>>(url, {
-    headers: {
-      Authorization: `Bearer ${apiToken}`,
-      "Content-Type": "application/vnd.api+json"
+  const headers = {
+    Authorization: `Bearer ${apiToken}`,
+    "Content-Type": "application/vnd.api+json"
+  };
+
+  const fetchAllPages = async (): Promise<TerraformCloudApiVariable[]> => {
+    let results: TerraformCloudApiVariable[] = [];
+    let nextUrl: string | null = url;
+
+    while (nextUrl) {
+      const res: AxiosResponse<TerraformCloudApiResponse<TerraformCloudApiVariable[]>> = await request.get<
+        TerraformCloudApiResponse<TerraformCloudApiVariable[]>
+      >(nextUrl, {
+        headers
+      });
+
+      if (res.data?.data) {
+        results = results.concat(res.data.data);
+      }
+
+      nextUrl = res.data?.links?.next ?? null;
     }
-  });
 
-  if (!response.data || !response.data.data) {
-    return [];
-  }
+    return results;
+  };
 
-  const variables: TerraformCloudVariable[] = response.data.data.map((variable: TerraformCloudApiVariable) => ({
+  const allVariableData = await fetchAllPages();
+
+  const variables: TerraformCloudVariable[] = allVariableData.map((variable) => ({
     id: variable.id,
     key: variable.attributes.key,
     value: variable.attributes.value || "",
@@ -85,8 +102,6 @@ const deleteVariable = async (
         "Content-Type": "application/vnd.api+json"
       }
     });
-
-    logger.info(`Deleted variable ${variable.key} from Terraform Cloud ${destinationConfig.scope}`);
   } catch (error) {
     throw new SecretSyncError({
       error,
@@ -137,8 +152,6 @@ const createVariable = async (
         }
       }
     );
-
-    logger.info(`Created variable ${key} in Terraform Cloud ${destinationConfig.scope}`);
   } catch (error) {
     throw new SecretSyncError({
       error,
@@ -188,8 +201,6 @@ const updateVariable = async (
         }
       }
     );
-
-    logger.info(`Updated variable ${variable.key} in Terraform Cloud ${destinationConfig.scope}`);
   } catch (error) {
     throw new SecretSyncError({
       error,

backend/src/services/secret-sync/terraform-cloud/terraform-cloud-sync-types.ts

@@ -18,14 +18,6 @@ export type TTerraformCloudSyncWithCredentials = TTerraformCloudSync & {
   connection: TTerraformCloudConnection;
 };
 
-export type TerraformCloudSecret = {
-  description: string;
-  is_secret: boolean;
-  key: string;
-  source: "project" | "workspace";
-  value: string;
-};
-
 export type TerraformCloudApiVariable = {
   id: string;
   type: string;