[AJ-1160] Add Retries to WSM and Leo (#396)

This PR hooks up all Leo calls in LeoDao, all WSM calls in WorkspaceManagerDao, and a single call to createBackup in WorkspaceDataServiceDao using RestClientRetry.withRetryAndErrorHandling()

service/src/main/java/org/databiosphere/workspacedataservice/InstanceInitializerBean.java

@@ -4,12 +4,12 @@
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.locks.Lock;
 import org.apache.commons.lang3.StringUtils;
-import org.databiosphere.workspacedata.client.ApiException;
 import org.databiosphere.workspacedataservice.dao.CloneDao;
 import org.databiosphere.workspacedataservice.dao.InstanceDao;
 import org.databiosphere.workspacedataservice.leonardo.LeonardoDao;
 import org.databiosphere.workspacedataservice.service.BackupRestoreService;
 import org.databiosphere.workspacedataservice.service.model.exception.CloningException;
+import org.databiosphere.workspacedataservice.service.model.exception.RestException;
 import org.databiosphere.workspacedataservice.shared.model.CloneResponse;
 import org.databiosphere.workspacedataservice.shared.model.CloneStatus;
 import org.databiosphere.workspacedataservice.shared.model.CloneTable;
@@ -23,6 +23,7 @@
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.context.event.ContextRefreshedEvent;
 import org.springframework.dao.DataAccessException;
+import org.springframework.http.HttpStatus;
 import org.springframework.integration.support.locks.LockRegistry;
 
 public class InstanceInitializerBean {
@@ -245,8 +246,8 @@ private String requestRemoteBackup(UUID trackingId) {
       }
     } catch (WorkspaceDataServiceException wdsE) {
       if (wdsE.getCause() != null
-          && wdsE.getCause() instanceof ApiException apiException
-          && apiException.getCode() == 404) {
+          && wdsE.getCause() instanceof RestException restException
+          && restException.getStatus() == HttpStatus.NOT_FOUND) {
         LOGGER.error(
             "Remote source WDS in workspace {} does not support cloning", sourceWorkspaceId);
         cloneDao.terminateCloneToError(

service/src/main/java/org/databiosphere/workspacedataservice/leonardo/LeonardoConfig.java

@@ -1,5 +1,6 @@
 package org.databiosphere.workspacedataservice.leonardo;
 
+import org.databiosphere.workspacedataservice.retry.RestClientRetry;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Value;
@@ -24,7 +25,8 @@ public LeonardoClientFactory getLeonardoClientFactory() {
   }
 
   @Bean
-  public LeonardoDao leonardoDao(LeonardoClientFactory leonardoClientFactory) {
-    return new LeonardoDao(leonardoClientFactory, workspaceId);
+  public LeonardoDao leonardoDao(
+      LeonardoClientFactory leonardoClientFactory, RestClientRetry restClientRetry) {
+    return new LeonardoDao(leonardoClientFactory, workspaceId, restClientRetry);
   }
 }

service/src/main/java/org/databiosphere/workspacedataservice/leonardo/LeonardoDao.java

@@ -5,25 +5,36 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
-import org.broadinstitute.dsde.workbench.client.leonardo.ApiException;
 import org.broadinstitute.dsde.workbench.client.leonardo.model.AppStatus;
 import org.broadinstitute.dsde.workbench.client.leonardo.model.AppType;
 import org.broadinstitute.dsde.workbench.client.leonardo.model.ListAppResponse;
+import org.databiosphere.workspacedataservice.retry.RestClientRetry;
+import org.databiosphere.workspacedataservice.retry.RestClientRetry.RestCall;
+import org.databiosphere.workspacedataservice.service.model.exception.RestException;
+import org.springframework.http.HttpStatus;
 
 public class LeonardoDao {
   private final LeonardoClientFactory leonardoClientFactory;
   private final String workspaceId;
+  private final RestClientRetry restClientRetry;
 
-  public LeonardoDao(LeonardoClientFactory leonardoClientFactory, String workspaceId) {
+  public LeonardoDao(
+      LeonardoClientFactory leonardoClientFactory,
+      String workspaceId,
+      RestClientRetry restClientRetry) {
     this.leonardoClientFactory = leonardoClientFactory;
     this.workspaceId = workspaceId;
+    this.restClientRetry = restClientRetry;
   }
 
   /** Asks leo to return apps running in a given workspace setting. */
   public String getWdsEndpointUrl(String token) {
     var workspaceApps = this.leonardoClientFactory.getAppsV2Api(token);
     try {
-      var response = workspaceApps.listAppsV2(workspaceId, null, false, null);
+      RestCall<List<ListAppResponse>> listAppsFunction =
+          () -> workspaceApps.listAppsV2(workspaceId, null, false, null);
+      List<ListAppResponse> response =
+          restClientRetry.withRetryAndErrorHandling(listAppsFunction, "WorkspaceApps.listApps");
       // unsure what the key would be if there is more than 1 wds present in the listed apps, but in
       // this case our assumption is
       // it is acceptable to fail if we cant find a single RUNNING wds in the proxy urls
@@ -32,8 +43,9 @@ public String getWdsEndpointUrl(String token) {
         return url;
       }
 
-      throw new ApiException("Did not locate an app running WDS.");
-    } catch (ApiException e) {
+      throw new RestException(
+          HttpStatus.INTERNAL_SERVER_ERROR, "Did not locate an app running WDS.");
+    } catch (RestException e) {
       throw new LeonardoServiceException(e);
     }
   }

service/src/main/java/org/databiosphere/workspacedataservice/leonardo/LeonardoServiceException.java

@@ -1,16 +1,10 @@
 package org.databiosphere.workspacedataservice.leonardo;
 
-import java.util.Optional;
-import org.broadinstitute.dsde.workbench.client.leonardo.ApiException;
-import org.springframework.http.HttpStatus;
+import org.databiosphere.workspacedataservice.service.model.exception.RestException;
 import org.springframework.web.server.ResponseStatusException;
 
 public class LeonardoServiceException extends ResponseStatusException {
-  public LeonardoServiceException(ApiException cause) {
-    super(
-        Optional.ofNullable(HttpStatus.resolve(cause.getCode()))
-            .orElse(HttpStatus.INTERNAL_SERVER_ERROR),
-        null,
-        cause);
+  public LeonardoServiceException(RestException cause) {
+    super(cause.getStatus(), cause.getMessage(), cause);
   }
 }

service/src/main/java/org/databiosphere/workspacedataservice/service/model/exception/RestServerException.java

@@ -2,7 +2,7 @@
 
 import org.springframework.http.HttpStatus;
 
-/** Indicates an error response from Sam of a 500, 502, 503, or 504 */
+/** Indicates an error response from our service client of a 500, 502, 503, or 504 */
 public class RestServerException extends RestRetryableException {
   public RestServerException(HttpStatus status, String message) {
     super(status, message);

service/src/main/java/org/databiosphere/workspacedataservice/sourcewds/WorkspaceDataServiceConfig.java

@@ -1,5 +1,6 @@
 package org.databiosphere.workspacedataservice.sourcewds;
 
+import org.databiosphere.workspacedataservice.retry.RestClientRetry;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
 
@@ -13,7 +14,8 @@ public WorkspaceDataServiceClientFactory getWorkspaceDataServiceClientFactory()
 
   @Bean
   public WorkspaceDataServiceDao workspaceDataServiceDao(
-      WorkspaceDataServiceClientFactory workspaceDataServiceClientFactory) {
-    return new WorkspaceDataServiceDao(workspaceDataServiceClientFactory);
+      WorkspaceDataServiceClientFactory workspaceDataServiceClientFactory,
+      RestClientRetry restClientRetry) {
+    return new WorkspaceDataServiceDao(workspaceDataServiceClientFactory, restClientRetry);
   }
 }

service/src/main/java/org/databiosphere/workspacedataservice/sourcewds/WorkspaceDataServiceDao.java

@@ -4,19 +4,25 @@
 import org.databiosphere.workspacedata.client.ApiException;
 import org.databiosphere.workspacedata.model.BackupJob;
 import org.databiosphere.workspacedata.model.BackupRestoreRequest;
+import org.databiosphere.workspacedataservice.retry.RestClientRetry;
+import org.databiosphere.workspacedataservice.retry.RestClientRetry.RestCall;
+import org.databiosphere.workspacedataservice.service.model.exception.RestException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 public class WorkspaceDataServiceDao {
   private final WorkspaceDataServiceClientFactory workspaceDataServiceClientFactory;
+  private final RestClientRetry restClientRetry;
 
   private String workspaceDataServiceUrl;
 
   private static final Logger LOGGER = LoggerFactory.getLogger(WorkspaceDataServiceDao.class);
 
   public WorkspaceDataServiceDao(
-      WorkspaceDataServiceClientFactory workspaceDataServiceClientFactory) {
+      WorkspaceDataServiceClientFactory workspaceDataServiceClientFactory,
+      RestClientRetry restClientRetry) {
     this.workspaceDataServiceClientFactory = workspaceDataServiceClientFactory;
+    this.restClientRetry = restClientRetry;
   }
 
   public void setWorkspaceDataServiceUrl(String endpointUrl) {
@@ -30,8 +36,9 @@ public BackupJob triggerBackup(String token, UUID requesterWorkspaceId) {
           this.workspaceDataServiceClientFactory.getBackupClient(token, workspaceDataServiceUrl);
       BackupRestoreRequest body = new BackupRestoreRequest();
       body.setRequestingWorkspaceId(requesterWorkspaceId);
-      return backupClient.createBackup(body, "v0.2");
-    } catch (ApiException e) {
+      RestCall<BackupJob> createBackupFunction = () -> backupClient.createBackup(body, "v0.2");
+      return restClientRetry.withRetryAndErrorHandling(createBackupFunction, "WDS.createBackup");
+    } catch (RestException e) {
       LOGGER.warn("Error from remote WDS: {}", e.getMessage());
       throw new WorkspaceDataServiceException(e);
     }

service/src/main/java/org/databiosphere/workspacedataservice/sourcewds/WorkspaceDataServiceException.java

@@ -2,6 +2,7 @@
 
 import java.util.Optional;
 import org.databiosphere.workspacedata.client.ApiException;
+import org.databiosphere.workspacedataservice.service.model.exception.RestException;
 import org.springframework.http.HttpStatus;
 import org.springframework.web.server.ResponseStatusException;
 
@@ -13,4 +14,8 @@ public WorkspaceDataServiceException(ApiException cause) {
         null,
         cause);
   }
+
+  public WorkspaceDataServiceException(RestException cause) {
+    super(cause.getStatus(), cause.getMessage(), cause);
+  }
 }

service/src/main/java/org/databiosphere/workspacedataservice/workspacemanager/WorkspaceManagerConfig.java

@@ -1,5 +1,6 @@
 package org.databiosphere.workspacedataservice.workspacemanager;
 
+import org.databiosphere.workspacedataservice.retry.RestClientRetry;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Value;
@@ -25,7 +26,8 @@ public WorkspaceManagerClientFactory getWorkspaceManagerClientFactory() {
 
   @Bean
   public WorkspaceManagerDao workspaceManagerDao(
-      WorkspaceManagerClientFactory workspaceManagerClientFactory) {
-    return new WorkspaceManagerDao(workspaceManagerClientFactory, workspaceId);
+      WorkspaceManagerClientFactory workspaceManagerClientFactory,
+      RestClientRetry restClientRetry) {
+    return new WorkspaceManagerDao(workspaceManagerClientFactory, workspaceId, restClientRetry);
   }
 }