feat: Add Netlify app connection and secrets sync (#4205)

* fix: save wip

* feat: final impl

* feat: docs

* Update backend/src/services/app-connection/digital-ocean/digital-ocean-connection-service.ts

Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com>

* chore: remove empty conflict files

* Update backend/src/server/routes/v1/app-connection-routers/app-connection-router.ts

Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com>

* Update frontend/src/components/secret-syncs/forms/schemas/digital-ocean-app-platform-sync-destination-schema.ts

Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com>

* Update frontend/src/components/secret-syncs/forms/schemas/digital-ocean-app-platform-sync-destination-schema.ts

Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com>

* Update frontend/src/components/secret-syncs/forms/SecretSyncDestinationFields/DigitalOceanAppPlatformSyncFields.tsx

Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com>

* Update backend/src/services/secret-sync/digital-ocean-app-platform/digital-ocean-app-platform-sync-schemas.ts

Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com>

* fix: lint

* feat: Netlify app connection and secrets sync

* feat: docs

* fix: type check

* fix: api client

* fix: lint and types

* fix: typecheck lint

* fix: docs

* fix: lint

* fix: lint

* fix: PR changes

* fix: typecheck

* fix: PR changes

* fix PR changes

* fix: PR Change

* fix: type error

* Small tweaks

* fix: support is_secret

* fix: revert is_secret

* fix: force update existing netlify secret

---------

Co-authored-by: sidwebworks <[email protected]>
Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com>
Co-authored-by: x032205 <[email protected]>

backend/src/services/secret-sync/netlify/index.ts

@@ -0,0 +1,4 @@
+export * from "./netlify-sync-constants";
+export * from "./netlify-sync-fns";
+export * from "./netlify-sync-schemas";
+export * from "./netlify-sync-types";

backend/src/services/secret-sync/netlify/netlify-sync-constants.ts

@@ -0,0 +1,19 @@
+import { AppConnection } from "@app/services/app-connection/app-connection-enums";
+import { SecretSync } from "@app/services/secret-sync/secret-sync-enums";
+import { TSecretSyncListItem } from "@app/services/secret-sync/secret-sync-types";
+
+export const NETLIFY_SYNC_LIST_OPTION: TSecretSyncListItem = {
+  name: "Netlify",
+  destination: SecretSync.Netlify,
+  connection: AppConnection.Netlify,
+  canImportSecrets: true
+};
+
+export enum NetlifySyncContext {
+  All = "all",
+  DeployPreview = "deploy-preview",
+  Production = "production",
+  BranchDeploy = "branch-deploy",
+  Dev = "dev",
+  Branch = "branch"
+}

backend/src/services/secret-sync/netlify/netlify-sync-fns.ts

@@ -0,0 +1,125 @@
+/* eslint-disable no-continue */
+import { NetlifyPublicAPI } from "@app/services/app-connection/netlify/netlify-connection-public-client";
+import { matchesSchema } from "@app/services/secret-sync/secret-sync-fns";
+import { TSecretMap } from "@app/services/secret-sync/secret-sync-types";
+
+import { SecretSyncError } from "../secret-sync-errors";
+import type { TNetlifySyncWithCredentials } from "./netlify-sync-types";
+
+export const NetlifySyncFns = {
+  async getSecrets(secretSync: TNetlifySyncWithCredentials): Promise<TSecretMap> {
+    const config = secretSync.destinationConfig;
+
+    const params = {
+      account_id: config.accountId,
+      context_name: config.context, // Only used in the case of getVariables
+      site_id: config.siteId
+    };
+
+    const variables = await NetlifyPublicAPI.getVariables(secretSync.connection, params);
+
+    const map = {} as TSecretMap;
+
+    for (const variable of variables) {
+      if (variable.is_secret) continue;
+
+      const latest = variable.values.sort((a, b) => Number(b.created_at) - Number(a.created_at))[0];
+
+      map[variable.key] = {
+        value: latest?.value ?? ""
+      };
+    }
+
+    return map;
+  },
+
+  async syncSecrets(secretSync: TNetlifySyncWithCredentials, secretMap: TSecretMap) {
+    const {
+      environment,
+      syncOptions: { disableSecretDeletion, keySchema }
+    } = secretSync;
+
+    const config = secretSync.destinationConfig;
+
+    const params = {
+      account_id: config.accountId,
+      context_name: config.context,
+      site_id: config.siteId
+    };
+
+    const variables = await NetlifyPublicAPI.getVariables(secretSync.connection, params);
+
+    const existing = Object.fromEntries(variables.map((variable) => [variable.key, variable]));
+
+    for await (const key of Object.keys(secretMap)) {
+      try {
+        const entry = secretMap[key];
+
+        await NetlifyPublicAPI.upsertVariable(secretSync.connection, params, {
+          key,
+          is_secret: false,
+          values: [
+            {
+              value: entry.value,
+              context: config.context
+            }
+          ]
+        });
+      } catch (error) {
+        throw new SecretSyncError({
+          error,
+          secretKey: key
+        });
+      }
+    }
+
+    if (disableSecretDeletion) return;
+
+    for await (const key of Object.keys(existing)) {
+      try {
+        // eslint-disable-next-line no-continue, @typescript-eslint/no-unsafe-argument
+        if (!matchesSchema(key, environment?.slug || "", keySchema)) continue;
+
+        if (!secretMap[key]) {
+          await NetlifyPublicAPI.deleteVariable(secretSync.connection, params, {
+            key
+          });
+        }
+      } catch (error) {
+        throw new SecretSyncError({
+          error,
+          secretKey: key
+        });
+      }
+    }
+  },
+
+  async removeSecrets(secretSync: TNetlifySyncWithCredentials, secretMap: TSecretMap) {
+    const config = secretSync.destinationConfig;
+
+    const params = {
+      account_id: config.accountId,
+      context_name: config.context,
+      site_id: config.siteId
+    };
+
+    const variables = await NetlifyPublicAPI.getVariables(secretSync.connection, params);
+
+    const existingSecrets = Object.fromEntries(variables.map((variable) => [variable.key, variable]));
+
+    for await (const secret of Object.keys(existingSecrets)) {
+      try {
+        if (secret in secretMap) {
+          await NetlifyPublicAPI.deleteVariable(secretSync.connection, params, {
+            key: secret
+          });
+        }
+      } catch (error) {
+        throw new SecretSyncError({
+          error,
+          secretKey: secret
+        });
+      }
+    }
+  }
+};

backend/src/services/secret-sync/netlify/netlify-sync-schemas.ts

@@ -0,0 +1,68 @@
+import { z } from "zod";
+
+import { SecretSyncs } from "@app/lib/api-docs";
+import { AppConnection } from "@app/services/app-connection/app-connection-enums";
+import { SecretSync } from "@app/services/secret-sync/secret-sync-enums";
+import {
+  BaseSecretSyncSchema,
+  GenericCreateSecretSyncFieldsSchema,
+  GenericUpdateSecretSyncFieldsSchema
+} from "@app/services/secret-sync/secret-sync-schemas";
+import { TSyncOptionsConfig } from "@app/services/secret-sync/secret-sync-types";
+
+import { NetlifySyncContext } from "./netlify-sync-constants";
+
+const NetlifySyncDestinationConfigSchema = z.object({
+  accountId: z
+    .string()
+    .min(1, "Account ID is required")
+    .max(255, "Account ID must be less than 255 characters")
+    .describe(SecretSyncs.DESTINATION_CONFIG.NETLIFY.accountId),
+  accountName: z
+    .string()
+    .min(1, "Account Name is required")
+    .max(255, "Account Name must be less than 255 characters")
+    .optional()
+    .describe(SecretSyncs.DESTINATION_CONFIG.NETLIFY.accountName),
+  siteId: z
+    .string()
+    .min(1, "Site ID is required")
+    .max(255, "Site ID must be less than 255 characters")
+    .optional()
+    .describe(SecretSyncs.DESTINATION_CONFIG.NETLIFY.siteId),
+  siteName: z
+    .string()
+    .min(1, "Site Name is required")
+    .max(255, "Site Name must be less than 255 characters")
+    .optional()
+    .describe(SecretSyncs.DESTINATION_CONFIG.NETLIFY.siteName),
+  context: z.nativeEnum(NetlifySyncContext).optional().describe(SecretSyncs.DESTINATION_CONFIG.NETLIFY.context)
+});
+
+const NetlifySyncOptionsConfig: TSyncOptionsConfig = { canImportSecrets: true };
+
+export const NetlifySyncSchema = BaseSecretSyncSchema(SecretSync.Netlify, NetlifySyncOptionsConfig).extend({
+  destination: z.literal(SecretSync.Netlify),
+  destinationConfig: NetlifySyncDestinationConfigSchema
+});
+
+export const CreateNetlifySyncSchema = GenericCreateSecretSyncFieldsSchema(
+  SecretSync.Netlify,
+  NetlifySyncOptionsConfig
+).extend({
+  destinationConfig: NetlifySyncDestinationConfigSchema
+});
+
+export const UpdateNetlifySyncSchema = GenericUpdateSecretSyncFieldsSchema(
+  SecretSync.Netlify,
+  NetlifySyncOptionsConfig
+).extend({
+  destinationConfig: NetlifySyncDestinationConfigSchema.optional()
+});
+
+export const NetlifySyncListItemSchema = z.object({
+  name: z.literal("Netlify"),
+  connection: z.literal(AppConnection.Netlify),
+  destination: z.literal(SecretSync.Netlify),
+  canImportSecrets: z.literal(true)
+});

backend/src/services/secret-sync/netlify/netlify-sync-types.ts

@@ -0,0 +1,15 @@
+import z from "zod";
+
+import { TNetlifyConnection } from "@app/services/app-connection/netlify";
+
+import { CreateNetlifySyncSchema, NetlifySyncListItemSchema, NetlifySyncSchema } from "./netlify-sync-schemas";
+
+export type TNetlifySyncListItem = z.infer<typeof NetlifySyncListItemSchema>;
+
+export type TNetlifySync = z.infer<typeof NetlifySyncSchema>;
+
+export type TNetlifySyncInput = z.infer<typeof CreateNetlifySyncSchema>;
+
+export type TNetlifySyncWithCredentials = TNetlifySync & {
+  connection: TNetlifyConnection;
+};

backend/src/services/secret-sync/secret-sync-enums.ts

@@ -27,6 +27,7 @@ export enum SecretSync {
   Railway = "railway",
   Checkly = "checkly",
   DigitalOceanAppPlatform = "digital-ocean-app-platform",
+  Netlify = "netlify",
   Bitbucket = "bitbucket"
 }
 

backend/src/services/secret-sync/secret-sync-fns.ts

@@ -48,6 +48,7 @@ import { HC_VAULT_SYNC_LIST_OPTION, HCVaultSyncFns } from "./hc-vault";
 import { HEROKU_SYNC_LIST_OPTION, HerokuSyncFns } from "./heroku";
 import { HUMANITEC_SYNC_LIST_OPTION } from "./humanitec";
 import { HumanitecSyncFns } from "./humanitec/humanitec-sync-fns";
+import { NETLIFY_SYNC_LIST_OPTION, NetlifySyncFns } from "./netlify";
 import { RAILWAY_SYNC_LIST_OPTION } from "./railway/railway-sync-constants";
 import { RailwaySyncFns } from "./railway/railway-sync-fns";
 import { RENDER_SYNC_LIST_OPTION, RenderSyncFns } from "./render";
@@ -88,6 +89,7 @@ const SECRET_SYNC_LIST_OPTIONS: Record<SecretSync, TSecretSyncListItem> = {
   [SecretSync.Railway]: RAILWAY_SYNC_LIST_OPTION,
   [SecretSync.Checkly]: CHECKLY_SYNC_LIST_OPTION,
   [SecretSync.DigitalOceanAppPlatform]: DIGITAL_OCEAN_APP_PLATFORM_SYNC_LIST_OPTION,
+  [SecretSync.Netlify]: NETLIFY_SYNC_LIST_OPTION,
   [SecretSync.Bitbucket]: BITBUCKET_SYNC_LIST_OPTION
 };
 
@@ -269,6 +271,8 @@ export const SecretSyncFns = {
         return SupabaseSyncFns.syncSecrets(secretSync, schemaSecretMap);
       case SecretSync.DigitalOceanAppPlatform:
         return DigitalOceanAppPlatformSyncFns.syncSecrets(secretSync, schemaSecretMap);
+      case SecretSync.Netlify:
+        return NetlifySyncFns.syncSecrets(secretSync, schemaSecretMap);
       case SecretSync.Bitbucket:
         return BitbucketSyncFns.syncSecrets(secretSync, schemaSecretMap);
       default:
@@ -381,6 +385,9 @@ export const SecretSyncFns = {
       case SecretSync.DigitalOceanAppPlatform:
         secretMap = await DigitalOceanAppPlatformSyncFns.getSecrets(secretSync);
         break;
+      case SecretSync.Netlify:
+        secretMap = await NetlifySyncFns.getSecrets(secretSync);
+        break;
       case SecretSync.Bitbucket:
         secretMap = await BitbucketSyncFns.getSecrets(secretSync);
         break;
@@ -473,6 +480,8 @@ export const SecretSyncFns = {
         return SupabaseSyncFns.removeSecrets(secretSync, schemaSecretMap);
       case SecretSync.DigitalOceanAppPlatform:
         return DigitalOceanAppPlatformSyncFns.removeSecrets(secretSync, schemaSecretMap);
+      case SecretSync.Netlify:
+        return NetlifySyncFns.removeSecrets(secretSync, schemaSecretMap);
       case SecretSync.Bitbucket:
         return BitbucketSyncFns.removeSecrets(secretSync, schemaSecretMap);
       default:

backend/src/services/secret-sync/secret-sync-maps.ts

@@ -30,6 +30,7 @@ export const SECRET_SYNC_NAME_MAP: Record<SecretSync, string> = {
   [SecretSync.Railway]: "Railway",
   [SecretSync.Checkly]: "Checkly",
   [SecretSync.DigitalOceanAppPlatform]: "Digital Ocean App Platform",
+  [SecretSync.Netlify]: "Netlify",
   [SecretSync.Bitbucket]: "Bitbucket"
 };
 
@@ -62,6 +63,7 @@ export const SECRET_SYNC_CONNECTION_MAP: Record<SecretSync, AppConnection> = {
   [SecretSync.Railway]: AppConnection.Railway,
   [SecretSync.Checkly]: AppConnection.Checkly,
   [SecretSync.DigitalOceanAppPlatform]: AppConnection.DigitalOcean,
+  [SecretSync.Netlify]: AppConnection.Netlify,
   [SecretSync.Bitbucket]: AppConnection.Bitbucket
 };
 
@@ -94,5 +96,6 @@ export const SECRET_SYNC_PLAN_MAP: Record<SecretSync, SecretSyncPlanType> = {
   [SecretSync.Railway]: SecretSyncPlanType.Regular,
   [SecretSync.Checkly]: SecretSyncPlanType.Regular,
   [SecretSync.DigitalOceanAppPlatform]: SecretSyncPlanType.Regular,
+  [SecretSync.Netlify]: SecretSyncPlanType.Regular,
   [SecretSync.Bitbucket]: SecretSyncPlanType.Regular
 };

backend/src/services/secret-sync/secret-sync-types.ts

@@ -117,6 +117,7 @@ import {
   THumanitecSyncListItem,
   THumanitecSyncWithCredentials
 } from "./humanitec";
+import { TNetlifySync, TNetlifySyncInput, TNetlifySyncListItem, TNetlifySyncWithCredentials } from "./netlify";
 import {
   TRailwaySync,
   TRailwaySyncInput,
@@ -178,6 +179,7 @@ export type TSecretSync =
   | TRailwaySync
   | TChecklySync
   | TSupabaseSync
+  | TNetlifySync
   | TBitbucketSync;
 
 export type TSecretSyncWithCredentials =
@@ -209,6 +211,7 @@ export type TSecretSyncWithCredentials =
   | TChecklySyncWithCredentials
   | TSupabaseSyncWithCredentials
   | TDigitalOceanAppPlatformSyncWithCredentials
+  | TNetlifySyncWithCredentials
   | TBitbucketSyncWithCredentials;
 
 export type TSecretSyncInput =
@@ -240,6 +243,7 @@ export type TSecretSyncInput =
   | TChecklySyncInput
   | TSupabaseSyncInput
   | TDigitalOceanAppPlatformSyncInput
+  | TNetlifySyncInput
   | TBitbucketSyncInput;
 
 export type TSecretSyncListItem =
@@ -271,6 +275,7 @@ export type TSecretSyncListItem =
   | TChecklySyncListItem
   | TSupabaseSyncListItem
   | TDigitalOceanAppPlatformSyncListItem
+  | TNetlifySyncListItem
   | TBitbucketSyncListItem;
 
 export type TSyncOptionsConfig = {