Improve render retries and rate limits

backend/src/services/secret-sync/render/render-sync-fns.ts

@@ -8,7 +8,22 @@ import { TSecretMap } from "@app/services/secret-sync/secret-sync-types";
 
 import { TRenderSecret, TRenderSyncWithCredentials } from "./render-sync-types";
 
-const getRenderEnvironmentSecrets = async (secretSync: TRenderSyncWithCredentials) => {
+const MAX_RETRIES = 5;
+
+const sleep = async () =>
+  new Promise((resolve) => {
+    setTimeout(resolve, 2000);
+  });
+
+const retrySleep = async () =>
+  new Promise((resolve) => {
+    setTimeout(resolve, 60000);
+  });
+
+const getRenderEnvironmentSecrets = async (
+  secretSync: TRenderSyncWithCredentials,
+  attempt = 0
+): Promise<TRenderSecret[]> => {
   const {
     destinationConfig,
     connection: {
@@ -22,35 +37,48 @@ const getRenderEnvironmentSecrets = async (secretSync: TRenderSyncWithCredential
 
   do {
     const url = cursor ? `${baseUrl}?cursor=${cursor}` : baseUrl;
-    const { data } = await request.get<
-      {
-        envVar: {
-          key: string;
-          value: string;
-        };
-        cursor: string;
-      }[]
-    >(url, {
-      headers: {
-        Authorization: `Bearer ${apiKey}`,
-        Accept: "application/json"
+
+    try {
+      const { data } = await request.get<
+        {
+          envVar: {
+            key: string;
+            value: string;
+          };
+          cursor: string;
+        }[]
+      >(url, {
+        headers: {
+          Authorization: `Bearer ${apiKey}`,
+          Accept: "application/json"
+        }
+      });
+
+      const secrets = data.map((item) => ({
+        key: item.envVar.key,
+        value: item.envVar.value
+      }));
+
+      allSecrets.push(...secrets);
+      cursor = data[data.length - 1]?.cursor;
+    } catch (error) {
+      if (isAxiosError(error) && error.response?.status === 429 && attempt < MAX_RETRIES) {
+        await retrySleep();
+        return await getRenderEnvironmentSecrets(secretSync, attempt + 1);
       }
-    });
-
-    const secrets = data.map((item) => ({
-      key: item.envVar.key,
-      value: item.envVar.value
-    }));
-
-    allSecrets.push(...secrets);
-
-    cursor = data[data.length - 1]?.cursor;
+      throw error;
+    }
   } while (cursor);
 
   return allSecrets;
 };
 
-const putEnvironmentSecret = async (secretSync: TRenderSyncWithCredentials, secretMap: TSecretMap, key: string) => {
+const putEnvironmentSecret = async (
+  secretSync: TRenderSyncWithCredentials,
+  secretMap: TSecretMap,
+  key: string,
+  attempt = 0
+): Promise<void> => {
   const {
     destinationConfig,
     connection: {
@@ -58,22 +86,34 @@ const putEnvironmentSecret = async (secretSync: TRenderSyncWithCredentials, secr
     }
   } = secretSync;
 
-  await request.put(
-    `${IntegrationUrls.RENDER_API_URL}/v1/services/${destinationConfig.serviceId}/env-vars/${key}`,
-    {
-      key,
-      value: secretMap[key].value
-    },
-    {
-      headers: {
-        Authorization: `Bearer ${apiKey}`,
-        Accept: "application/json"
+  try {
+    await request.put(
+      `${IntegrationUrls.RENDER_API_URL}/v1/services/${destinationConfig.serviceId}/env-vars/${key}`,
+      {
+        key,
+        value: secretMap[key].value
+      },
+      {
+        headers: {
+          Authorization: `Bearer ${apiKey}`,
+          Accept: "application/json"
+        }
       }
+    );
+  } catch (error) {
+    if (isAxiosError(error) && error.response?.status === 429 && attempt < MAX_RETRIES) {
+      await retrySleep();
+      return await putEnvironmentSecret(secretSync, secretMap, key, attempt + 1);
     }
-  );
+    throw error;
+  }
 };
 
-const deleteEnvironmentSecret = async (secretSync: TRenderSyncWithCredentials, secret: Pick<TRenderSecret, "key">) => {
+const deleteEnvironmentSecret = async (
+  secretSync: TRenderSyncWithCredentials,
+  secret: Pick<TRenderSecret, "key">,
+  attempt = 0
+): Promise<void> => {
   const {
     destinationConfig,
     connection: {
@@ -97,18 +137,19 @@ const deleteEnvironmentSecret = async (secretSync: TRenderSyncWithCredentials, s
       return;
     }
 
+    if (isAxiosError(error) && error.response?.status === 429 && attempt < MAX_RETRIES) {
+      await retrySleep();
+      return await deleteEnvironmentSecret(secretSync, secret, attempt + 1);
+    }
+
     throw error;
   }
 };
 
-const sleep = async () =>
-  new Promise((resolve) => {
-    setTimeout(resolve, 500);
-  });
-
 export const RenderSyncFns = {
   syncSecrets: async (secretSync: TRenderSyncWithCredentials, secretMap: TSecretMap) => {
     const renderSecrets = await getRenderEnvironmentSecrets(secretSync);
+
     for await (const key of Object.keys(secretMap)) {
       // If value is empty skip it as render does not allow empty variables
       if (secretMap[key].value === "") {
@@ -132,6 +173,7 @@ export const RenderSyncFns = {
       }
     }
   },
+
   getSecrets: async (secretSync: TRenderSyncWithCredentials): Promise<TSecretMap> => {
     const renderSecrets = await getRenderEnvironmentSecrets(secretSync);
     return Object.fromEntries(renderSecrets.map((secret) => [secret.key, { value: secret.value ?? "" }]));