Add secret recovery for setting deleted secrets for Azure integration

backend/src/integrations/sync.ts

@@ -273,9 +273,23 @@ const syncSecretsAzureKeyVault = async ({
       }
     });
 
-    // Sync/push set secrets
+    const setSecretAzureKeyVault = async ({
-    if (setSecrets.length > 0) {
+      key,
-      setSecrets.forEach(async ({ key, value }) => {
+      value,
+      integration,
+      accessToken
+    }: {
+      key: string;
+      value: string;
+      integration: IIntegration;
+      accessToken: string;
+    }) => {
+      let isSecretSet = false;
+      let maxTries = 6;
+      
+      while (!isSecretSet && maxTries > 0) {
+        // try to set secret
+        try {
           await request.put(
             `${integration.app}/secrets/${key}?api-version=7.3`,
             {
@@ -287,17 +301,47 @@ const syncSecretsAzureKeyVault = async ({
               }
             }
           );
+
+          isSecretSet = true;
+        
+        } catch (err) {
+          const error: any = err;
+          if (error?.response?.data?.error?.innererror?.code === 'ObjectIsDeletedButRecoverable') {
+            await request.post(
+              `${integration.app}/deletedsecrets/${key}/recover?api-version=7.3`, {},
+              {
+                headers: {
+                  Authorization: `Bearer ${accessToken}`
+                }
+              }
+            );
+            await new Promise(resolve => setTimeout(resolve, 10000));
+          } else {
+            await new Promise(resolve => setTimeout(resolve, 10000));
+            maxTries--;
+          }
+        }
+      }
+    }
+    
+    // Sync/push set secrets
+    for await (const setSecret of setSecrets) {
+      const { key, value } = setSecret;
+      setSecretAzureKeyVault({
+        key,
+        value,
+        integration,
+        accessToken
       });
     }
     
-    if (deleteSecrets.length > 0) {
+    for await (const deleteSecret of deleteSecrets) {
-      deleteSecrets.forEach(async (secret) => {
+      const { key } = deleteSecret;
-        await request.delete(`${integration.app}/secrets/${secret.key}?api-version=7.3`, {
+      await request.delete(`${integration.app}/secrets/${key}?api-version=7.3`, {
         headers: {
           'Authorization': `Bearer ${accessToken}`
         }
       });
-      });
     }
   } catch (err) {
     Sentry.setUser(null);