General improvements to Terraform Integration

backend/src/lib/api-docs/constants.ts

@@ -1694,6 +1694,9 @@ export const AppConnections = {
       sslEnabled: "Whether or not to use SSL when connecting to the database.",
       sslRejectUnauthorized: "Whether or not to reject unauthorized SSL certificates.",
       sslCertificate: "The SSL certificate to use for connection."
+    },
+    TERRAFORM_CLOUD: {
+      apiToken: "The API token to use to connect with Terraform Cloud."
     }
   }
 };

backend/src/services/app-connection/terraform-cloud/terraform-cloud-connection-fns.ts

@@ -64,62 +64,52 @@ export const listOrganizations = async (
     credentials: { apiToken }
   } = appConnection;
 
-  const orgsResponse = await request.get<{ data: { id: string; attributes: { name: string } }[] }>(
+  const headers = {
-    `${IntegrationUrls.TERRAFORM_CLOUD_API_URL}/api/v2/organizations`,
+    Authorization: `Bearer ${apiToken}`,
-    {
+    "Content-Type": "application/vnd.api+json"
-      headers: {
+  };
-        Authorization: `Bearer ${apiToken}`,
+
-        "Content-Type": "application/vnd.api+json"
+  const fetchAllPages = async <T>(url: string): Promise<T[]> => {
-      }
+    let results: T[] = [];
+    let nextUrl: string | null = url;
+
+    while (nextUrl) {
+      // eslint-disable-next-line no-await-in-loop
+      const res: AxiosResponse<{ data: T[]; links?: { next?: string } }> = await request.get(nextUrl, { headers });
+      results = results.concat(res.data.data);
+      nextUrl = res.data.links?.next || null;
     }
+
+    return results;
+  };
+
+  const orgEntities = await fetchAllPages<{ id: string; attributes: { name: string } }>(
+    `${IntegrationUrls.TERRAFORM_CLOUD_API_URL}/api/v2/organizations`
   );
 
-  if (!orgsResponse.data?.data) {
-    throw new InternalServerError({
-      message: "Failed to get organizations: Response was empty"
-    });
-  }
-
-  const orgEntities = orgsResponse.data.data;
   const orgsWithVariableSetsAndWorkspaces: TTerraformCloudOrganization[] = [];
 
   const variableSetPromises = orgEntities.map((org) =>
-    request
+    fetchAllPages<{ id: string; attributes: { name: string; description?: string; global?: boolean } }>(
-      .get<{ data: { id: string; attributes: { name: string; description?: string; global?: boolean } }[] }>(
+      `${IntegrationUrls.TERRAFORM_CLOUD_API_URL}/api/v2/organizations/${org.id}/varsets`
-        `${IntegrationUrls.TERRAFORM_CLOUD_API_URL}/api/v2/organizations/${org.id}/varsets`,
+    ).catch(() => [])
-        {
-          headers: {
-            Authorization: `Bearer ${apiToken}`,
-            "Content-Type": "application/vnd.api+json"
-          }
-        }
-      )
-      .catch(() => ({ data: { data: [] } }))
   );
 
   const workspacePromises = orgEntities.map((org) =>
-    request
+    fetchAllPages<{ id: string; attributes: { name: string } }>(
-      .get<{ data: { id: string; attributes: { name: string } }[] }>(
+      `${IntegrationUrls.TERRAFORM_CLOUD_API_URL}/api/v2/organizations/${org.id}/workspaces`
-        `${IntegrationUrls.TERRAFORM_CLOUD_API_URL}/api/v2/organizations/${org.id}/workspaces`,
+    ).catch(() => [])
-        {
-          headers: {
-            Authorization: `Bearer ${apiToken}`,
-            "Content-Type": "application/vnd.api+json"
-          }
-        }
-      )
-      .catch(() => ({ data: { data: [] } }))
   );
 
-  const [variableSetResponses, workspaceResponses] = await Promise.all([
+  const [variableSetResults, workspaceResults] = await Promise.all([
     Promise.all(variableSetPromises),
     Promise.all(workspacePromises)
   ]);
 
   for (let i = 0; i < orgEntities.length; i += 1) {
     const org = orgEntities[i];
-    const variableSetsData = variableSetResponses[i].data?.data || [];
+    const variableSetsData = variableSetResults[i];
-    const workspacesData = workspaceResponses[i].data?.data || [];
+    const workspacesData = workspaceResults[i];
 
     const variableSets: TTerraformCloudVariableSet[] = variableSetsData.map((varSet) => ({
       id: varSet.id,

backend/src/services/app-connection/terraform-cloud/terraform-cloud-connection-types.ts

@@ -25,21 +25,6 @@ export type TTerraformCloudConnectionConfig = DiscriminativePick<
   orgId: string;
 };
 
-export type TerraformCloudOrg = {
-  id: string;
-  name: string;
-};
-
-export type TerraformCloudApp = {
-  name: string;
-  id: string;
-  envs: { name: string; id: string }[];
-};
-
-export type TerraformCloudOrgWithApps = TerraformCloudOrg & {
-  apps: TerraformCloudApp[];
-};
-
 export type TTerraformCloudVariableSet = {
   id: string;
   name: string;
@@ -58,12 +43,3 @@ export type TTerraformCloudOrganization = {
   variableSets: TTerraformCloudVariableSet[];
   workspaces: TTerraformCloudWorkspace[];
 };
-
-export type TTerraformCloudConnectionOrganization = TTerraformCloudOrganization;
-export type TTerraformCloudConnectionVariableSet = TTerraformCloudVariableSet;
-export type TTerraformCloudConnectionWorkspace = TTerraformCloudWorkspace;
-
-export enum TerraformCloudSyncScope {
-  VariableSet = "variableSet",
-  Workspace = "workspace"
-}

backend/src/services/secret-sync/terraform-cloud/terraform-cloud-sync-fns.ts

@@ -1,6 +1,5 @@
 /* eslint-disable no-await-in-loop */
 import { request } from "@app/lib/config/request";
-import { logger } from "@app/lib/logger";
 import { IntegrationUrls } from "@app/services/integration-auth/integration-list";
 import { SecretSyncError } from "@app/services/secret-sync/secret-sync-errors";
 import { TSecretMap } from "@app/services/secret-sync/secret-sync-types";
@@ -13,6 +12,7 @@ import {
   TerraformCloudVariable,
   TTerraformCloudSyncWithCredentials
 } from "./terraform-cloud-sync-types";
+import { AxiosResponse } from "axios";
 
 const getTerraformCloudVariables = async (
   secretSync: TTerraformCloudSyncWithCredentials
@@ -24,7 +24,7 @@ const getTerraformCloudVariables = async (
     }
   } = secretSync;
 
-  let url;
+  let url: string;
   let source: TerraformCloudVariable["source"];
 
   if (destinationConfig.scope === TerraformCloudSyncScope.VariableSet) {
@@ -35,18 +35,35 @@ const getTerraformCloudVariables = async (
     source = "workspace";
   }
 
-  const response = await request.get<TerraformCloudApiResponse<TerraformCloudApiVariable[]>>(url, {
+  const headers = {
-    headers: {
+    Authorization: `Bearer ${apiToken}`,
-      Authorization: `Bearer ${apiToken}`,
+    "Content-Type": "application/vnd.api+json"
-      "Content-Type": "application/vnd.api+json"
+  };
+
+  const fetchAllPages = async (): Promise<TerraformCloudApiVariable[]> => {
+    let results: TerraformCloudApiVariable[] = [];
+    let nextUrl: string | null = url;
+
+    while (nextUrl) {
+      const res: AxiosResponse<TerraformCloudApiResponse<TerraformCloudApiVariable[]>> = await request.get<
+        TerraformCloudApiResponse<TerraformCloudApiVariable[]>
+      >(nextUrl, {
+        headers
+      });
+
+      if (res.data?.data) {
+        results = results.concat(res.data.data);
+      }
+
+      nextUrl = res.data?.links?.next ?? null;
     }
-  });
 
-  if (!response.data || !response.data.data) {
+    return results;
-    return [];
+  };
-  }
 
-  const variables: TerraformCloudVariable[] = response.data.data.map((variable: TerraformCloudApiVariable) => ({
+  const allVariableData = await fetchAllPages();
+
+  const variables: TerraformCloudVariable[] = allVariableData.map((variable) => ({
     id: variable.id,
     key: variable.attributes.key,
     value: variable.attributes.value || "",
@@ -85,8 +102,6 @@ const deleteVariable = async (
         "Content-Type": "application/vnd.api+json"
       }
     });
-
-    logger.info(`Deleted variable ${variable.key} from Terraform Cloud ${destinationConfig.scope}`);
   } catch (error) {
     throw new SecretSyncError({
       error,
@@ -137,8 +152,6 @@ const createVariable = async (
         }
       }
     );
-
-    logger.info(`Created variable ${key} in Terraform Cloud ${destinationConfig.scope}`);
   } catch (error) {
     throw new SecretSyncError({
       error,
@@ -188,8 +201,6 @@ const updateVariable = async (
         }
       }
     );
-
-    logger.info(`Updated variable ${variable.key} in Terraform Cloud ${destinationConfig.scope}`);
   } catch (error) {
     throw new SecretSyncError({
       error,

backend/src/services/secret-sync/terraform-cloud/terraform-cloud-sync-types.ts

@@ -18,14 +18,6 @@ export type TTerraformCloudSyncWithCredentials = TTerraformCloudSync & {
   connection: TTerraformCloudConnection;
 };
 
-export type TerraformCloudSecret = {
-  description: string;
-  is_secret: boolean;
-  key: string;
-  source: "project" | "workspace";
-  value: string;
-};
-
 export type TerraformCloudApiVariable = {
   id: string;
   type: string;

docs/api-reference/endpoints/app-connections/terraform-cloud/create.mdx

@@ -1,4 +1,9 @@
 ---
 title: "Create"
 openapi: "POST /api/v1/app-connections/terraform-cloud"
----
+---
+
+<Note>
+    Check out the configuration docs for [Terraform Cloud Connections](/integrations/app-connections/terraform-cloud) to learn how to obtain
+    the required credentials.
+</Note>

docs/api-reference/endpoints/app-connections/terraform-cloud/update.mdx

@@ -1,4 +1,9 @@
 ---
 title: "Update"
 openapi: "PATCH /api/v1/app-connections/terraform-cloud/{connectionId}"
----
+---
+
+<Note>
+    Check out the configuration docs for [Terraform Cloud Connections](/integrations/app-connections/terraform-cloud) to learn how to obtain
+    the required credentials.
+</Note>

docs/api-reference/endpoints/secret-syncs/terraform-cloud/create.mdx

@@ -1,4 +1,4 @@
 ---
 title: "Create"
-openapi: "POST /api/v1/secret-syncs/terraform-cloud"
+openapi: "POST /api/v1/secret-syncs/humanitec"
 ---

docs/integrations/app-connections/terraform-cloud.mdx

@@ -34,7 +34,7 @@ Infisical supports connecting to Terraform Cloud using a service user.
                 ![App Connections Tab](/images/app-connections/general/add-connection.png)
                 2. Select the **Terraform Cloud Connection** option from the connection options modal.
                     ![Select Terraform Cloud Connection](/images/app-connections/terraform-cloud/terraform-cloud-app-connection-option.png)
-                3. Fill the Terraform Cloud Connection modal, here you will need to provide the API Token generated in the previous step.
+                3. Fill out the Terraform Cloud Connection modal, here you will need to provide the API Token generated in the previous step.
                     ![Terraform Cloud Connection Modal](/images/app-connections/terraform-cloud/terraform-cloud-app-connection-modal.png)
                 4. Your **Terraform Cloud Connection** is now available for use.
                     ![Terraform Cloud Connection Created](/images/app-connections/terraform-cloud/terraform-cloud-app-connection-created.png)

docs/integrations/secret-syncs/terraform-cloud.mdx

@@ -94,7 +94,7 @@ description: "Learn how to configure a Terraform Cloud Sync for Infisical."
             },
             "destinationConfig": {
                 "scope": "variable-set",
-                "destinationId": "3c90c3cc-0d44-4b50-8888-8dd25736052a"
+                "destinationId": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
                 "destinationName": "my-variable-set",
                 "org": "my-organization-id",
                 "category": "env"

frontend/src/components/secret-syncs/forms/SecretSyncDestinationFields/SecretSyncDestinationFields.tsx

@@ -17,7 +17,6 @@ export const SecretSyncDestinationFields = () => {
   const { watch } = useFormContext<TSecretSyncForm>();
 
   const destination = watch("destination");
-  console.log(destination);
 
   switch (destination) {
     case SecretSync.AWSParameterStore:

frontend/src/components/secret-syncs/forms/SecretSyncDestinationFields/TerraformCloudSyncFields.tsx

@@ -1,10 +1,8 @@
 import { Controller, useFormContext, useWatch } from "react-hook-form";
 import { SingleValue } from "react-select";
-import { faCircleInfo } from "@fortawesome/free-solid-svg-icons";
-import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
 
 import { SecretSyncConnectionField } from "@app/components/secret-syncs/forms/SecretSyncConnectionField";
-import { FilterableSelect, FormControl, Select, SelectItem, Tooltip } from "@app/components/v2";
+import { FilterableSelect, FormControl, Select, SelectItem } from "@app/components/v2";
 import {
   TERRAFORM_CLOUD_SYNC_SCOPES,
   TerraformCloudSyncCategory,
@@ -82,6 +80,28 @@ export const TerraformCloudSyncFields = () => {
             errorText={error?.message}
             isError={Boolean(error?.message)}
             label="Category"
+            tooltipClassName="max-w-lg py-3"
+            tooltipText={
+              <div className="flex flex-col gap-3">
+                <ul className="flex list-disc flex-col gap-3 pl-4">
+                  <li>
+                    <p className="text-mineshaft-300">
+                      <span className="font-medium text-bunker-200">
+                        Environment variables configure Terraform&apos;s behavior (e.g.,
+                        credentials).
+                      </span>
+                    </p>
+                  </li>
+                  <li>
+                    <p className="text-mineshaft-300">
+                      <span className="font-medium text-bunker-200">
+                        Terraform variables are used as input values in your configuration.
+                      </span>
+                    </p>
+                  </li>
+                </ul>
+              </div>
+            }
           >
             <Select
               value={value}
@@ -92,9 +112,9 @@ export const TerraformCloudSyncFields = () => {
               placeholder="Select category..."
               dropdownContainerClassName="max-w-none"
             >
-              {Object.values(TerraformCloudSyncCategory).map((category) => (
+              {Object.entries(TerraformCloudSyncCategory).map(([envKey, envValue]) => (
-                <SelectItem className="capitalize" value={category} key={category}>
+                <SelectItem className="capitalize" value={envValue} key={envValue}>
-                  {category.replace("-", " ")}
+                  {envKey.replace("-", " ")}
                 </SelectItem>
               ))}
             </Select>
@@ -157,22 +177,7 @@ export const TerraformCloudSyncFields = () => {
           name="destinationConfig.destinationId"
           control={control}
           render={({ field: { value, onChange }, fieldState: { error } }) => (
-            <FormControl
+            <FormControl isError={Boolean(error)} errorText={error?.message} label="Variable Set">
-              isError={Boolean(error)}
-              errorText={error?.message}
-              label="Variable Set"
-              helperText={
-                <Tooltip
-                  className="max-w-md"
-                  content="Ensure that the variable set exists in the selected organization and the service account used on this connection has write permissions for the specified variable set."
-                >
-                  <div>
-                    <span>Don&#39;t see the variable set you&#39;re looking for?</span>{" "}
-                    <FontAwesomeIcon icon={faCircleInfo} className="text-mineshaft-400" />
-                  </div>
-                </Tooltip>
-              }
-            >
               <FilterableSelect
                 menuPlacement="top"
                 isLoading={isOrganizationsPending && Boolean(connectionId) && Boolean(currentOrg)}
@@ -203,22 +208,7 @@ export const TerraformCloudSyncFields = () => {
           name="destinationConfig.destinationId"
           control={control}
           render={({ field: { value, onChange }, fieldState: { error } }) => (
-            <FormControl
+            <FormControl isError={Boolean(error)} errorText={error?.message} label="Workspace">
-              isError={Boolean(error)}
-              errorText={error?.message}
-              label="Workspace"
-              helperText={
-                <Tooltip
-                  className="max-w-md"
-                  content="Ensure that the workspace exists in the selected organization and the service account used on this connection has write permissions for the specified workspace."
-                >
-                  <div>
-                    <span>Don&#39;t see the workspace you&#39;re looking for?</span>{" "}
-                    <FontAwesomeIcon icon={faCircleInfo} className="text-mineshaft-400" />
-                  </div>
-                </Tooltip>
-              }
-            >
               <FilterableSelect
                 menuPlacement="top"
                 isLoading={isOrganizationsPending && Boolean(connectionId) && Boolean(currentOrg)}

frontend/src/components/secret-syncs/forms/schemas/terraform-cloud-destination-schema.ts

@@ -14,14 +14,14 @@ export const TerraformCloudSyncDestinationSchema = BaseSecretSyncSchema().merge(
       z.object({
         scope: z.literal(TerraformCloudSyncScope.VariableSet),
         org: z.string().trim().min(1, "Organization required"),
-        destinationId: z.string().trim().min(1, "Variable set id required"),
+        destinationId: z.string().trim().min(1, "Variable Set required"),
         destinationName: z.string().trim().min(1, "Variable set name required"),
         category: z.nativeEnum(TerraformCloudSyncCategory)
       }),
       z.object({
         scope: z.literal(TerraformCloudSyncScope.Workspace),
         org: z.string().trim().min(1, "Organization required"),
-        destinationId: z.string().trim().min(1, "Workspace id required"),
+        destinationId: z.string().trim().min(1, "Workspace required"),
         destinationName: z.string().trim().min(1, "Workspace name required"),
         category: z.nativeEnum(TerraformCloudSyncCategory)
       })

frontend/src/helpers/appConnections.ts

@@ -52,7 +52,6 @@ export const getAppConnectionMethodDetails = (method: TAppConnection["method"])
     case DatabricksConnectionMethod.ServicePrincipal:
       return { name: "Service Principal", icon: faUser };
     case HumanitecConnectionMethod.ApiToken:
-      return { name: "API Token", icon: faKey };
     case TerraformCloudConnectionMethod.ApiToken:
       return { name: "API Token", icon: faKey };
     case PostgresConnectionMethod.UsernameAndPassword:

frontend/src/pages/organization/AppConnections/AppConnectionsPage/components/AppConnectionForm/TerraformCloudConnectionForm.tsx

@@ -35,7 +35,7 @@ const formSchema = z.discriminatedUnion("method", [
   rootSchema.extend({
     method: z.literal(TerraformCloudConnectionMethod.ApiToken),
     credentials: z.object({
-      apiToken: z.string().trim().min(1, "Service API Token required")
+      apiToken: z.string().trim().min(1, "API Token required")
     })
   })
 ]);
@@ -102,7 +102,7 @@ export const TerraformCloudConnectionForm = ({ appConnection, onSubmit }: Props)
             <FormControl
               errorText={error?.message}
               isError={Boolean(error?.message)}
-              label="Service API Token"
+              label="API Token"
             >
               <SecretInput
                 containerClassName="text-gray-400 group-focus-within:!border-primary-400/50 border border-mineshaft-500 bg-mineshaft-900 px-2.5 py-1.5"

frontend/src/pages/secret-manager/SecretSyncDetailsByIDPage/components/SecretSyncDestinationSection/SecretSyncDestinatonSection.tsx

@@ -9,7 +9,6 @@ import { ProjectPermissionSub } from "@app/context";
 import { ProjectPermissionSecretSyncActions } from "@app/context/ProjectPermissionContext/types";
 import { APP_CONNECTION_MAP } from "@app/helpers/appConnections";
 import { SecretSync, TSecretSync } from "@app/hooks/api/secretSyncs";
-import { TerraformCloudSyncDestinationCol } from "@app/pages/secret-manager/IntegrationsListPage/components/SecretSyncsTab/SecretSyncTable/SecretSyncDestinationCol/TerraformCloudSyncDestinationCol";
 import { AwsParameterStoreSyncDestinationSection } from "@app/pages/secret-manager/SecretSyncDetailsByIDPage/components/SecretSyncDestinationSection/AwsParameterStoreSyncDestinationSection";
 import { AwsSecretsManagerSyncDestinationSection } from "@app/pages/secret-manager/SecretSyncDetailsByIDPage/components/SecretSyncDestinationSection/AwsSecretsManagerSyncDestinationSection";
 import { DatabricksSyncDestinationSection } from "@app/pages/secret-manager/SecretSyncDetailsByIDPage/components/SecretSyncDestinationSection/DatabricksSyncDestinationSection";
@@ -19,6 +18,7 @@ import { AzureAppConfigurationSyncDestinationSection } from "./AzureAppConfigura
 import { AzureKeyVaultSyncDestinationSection } from "./AzureKeyVaultSyncDestinationSection";
 import { GcpSyncDestinationSection } from "./GcpSyncDestinationSection";
 import { HumanitecSyncDestinationSection } from "./HumanitecSyncDestinationSection";
+import { TerraformCloudSyncDestinationSection } from "./TerraformCloudSyncDestinationCol";
 
 type Props = {
   secretSync: TSecretSync;
@@ -59,7 +59,7 @@ export const SecretSyncDestinationSection = ({ secretSync, onEditDestination }:
       DestinationComponents = <HumanitecSyncDestinationSection secretSync={secretSync} />;
       break;
     case SecretSync.TerraformCloud:
-      DestinationComponents = <TerraformCloudSyncDestinationCol secretSync={secretSync} />;
+      DestinationComponents = <TerraformCloudSyncDestinationSection secretSync={secretSync} />;
       break;
     default:
       throw new Error(`Unhandled Destination Section components: ${destination}`);