Improve render retries and rate limits

backend/src/services/secret-sync/render/render-sync-fns.ts

@@ -8,7 +8,22 @@ import { TSecretMap } from "@app/services/secret-sync/secret-sync-types";
 
 import { TRenderSecret, TRenderSyncWithCredentials } from "./render-sync-types";
 
-const getRenderEnvironmentSecrets = async (secretSync: TRenderSyncWithCredentials) => {
+const MAX_RETRIES = 5;
+
+const sleep = async () =>
+  new Promise((resolve) => {
+    setTimeout(resolve, 2000);
+  });
+
+const retrySleep = async () =>
+  new Promise((resolve) => {
+    setTimeout(resolve, 60000);
+  });
+
+const getRenderEnvironmentSecrets = async (
+  secretSync: TRenderSyncWithCredentials,
+  attempt = 0
+): Promise<TRenderSecret[]> => {
   const {
     destinationConfig,
     connection: {
@@ -22,6 +37,8 @@ const getRenderEnvironmentSecrets = async (secretSync: TRenderSyncWithCredential
 
   do {
     const url = cursor ? `${baseUrl}?cursor=${cursor}` : baseUrl;
+
+    try {
       const { data } = await request.get<
         {
           envVar: {
@@ -43,14 +60,25 @@ const getRenderEnvironmentSecrets = async (secretSync: TRenderSyncWithCredential
       }));
 
       allSecrets.push(...secrets);
-
       cursor = data[data.length - 1]?.cursor;
+    } catch (error) {
+      if (isAxiosError(error) && error.response?.status === 429 && attempt < MAX_RETRIES) {
+        await retrySleep();
+        return await getRenderEnvironmentSecrets(secretSync, attempt + 1);
+      }
+      throw error;
+    }
   } while (cursor);
 
   return allSecrets;
 };
 
-const putEnvironmentSecret = async (secretSync: TRenderSyncWithCredentials, secretMap: TSecretMap, key: string) => {
+const putEnvironmentSecret = async (
+  secretSync: TRenderSyncWithCredentials,
+  secretMap: TSecretMap,
+  key: string,
+  attempt = 0
+): Promise<void> => {
   const {
     destinationConfig,
     connection: {
@@ -58,6 +86,7 @@ const putEnvironmentSecret = async (secretSync: TRenderSyncWithCredentials, secr
     }
   } = secretSync;
 
+  try {
     await request.put(
       `${IntegrationUrls.RENDER_API_URL}/v1/services/${destinationConfig.serviceId}/env-vars/${key}`,
       {
@@ -71,9 +100,20 @@ const putEnvironmentSecret = async (secretSync: TRenderSyncWithCredentials, secr
         }
       }
     );
+  } catch (error) {
+    if (isAxiosError(error) && error.response?.status === 429 && attempt < MAX_RETRIES) {
+      await retrySleep();
+      return await putEnvironmentSecret(secretSync, secretMap, key, attempt + 1);
+    }
+    throw error;
+  }
 };
 
-const deleteEnvironmentSecret = async (secretSync: TRenderSyncWithCredentials, secret: Pick<TRenderSecret, "key">) => {
+const deleteEnvironmentSecret = async (
+  secretSync: TRenderSyncWithCredentials,
+  secret: Pick<TRenderSecret, "key">,
+  attempt = 0
+): Promise<void> => {
   const {
     destinationConfig,
     connection: {
@@ -97,18 +137,19 @@ const deleteEnvironmentSecret = async (secretSync: TRenderSyncWithCredentials, s
       return;
     }
 
+    if (isAxiosError(error) && error.response?.status === 429 && attempt < MAX_RETRIES) {
+      await retrySleep();
+      return await deleteEnvironmentSecret(secretSync, secret, attempt + 1);
+    }
+
     throw error;
   }
 };
 
-const sleep = async () =>
-  new Promise((resolve) => {
-    setTimeout(resolve, 500);
-  });
-
 export const RenderSyncFns = {
   syncSecrets: async (secretSync: TRenderSyncWithCredentials, secretMap: TSecretMap) => {
     const renderSecrets = await getRenderEnvironmentSecrets(secretSync);
+
     for await (const key of Object.keys(secretMap)) {
       // If value is empty skip it as render does not allow empty variables
       if (secretMap[key].value === "") {
@@ -132,6 +173,7 @@ export const RenderSyncFns = {
       }
     }
   },
+
   getSecrets: async (secretSync: TRenderSyncWithCredentials): Promise<TSecretMap> => {
     const renderSecrets = await getRenderEnvironmentSecrets(secretSync);
     return Object.fromEntries(renderSecrets.map((secret) => [secret.key, { value: secret.value ?? "" }]));