Retry dsci modification

src/main/java/io/odh/test/TestUtils.java

@@ -20,6 +20,7 @@
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.time.Duration;
+import java.util.concurrent.Callable;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.Executor;
 import java.util.concurrent.ExecutorService;
@@ -213,4 +214,28 @@ public static Path getLogPath(String folderName, String testClassName, String te
         }
         return path;
     }
+
+    /**
+     * Repeat command n-times
+     *
+     * @param retry count of remaining retries
+     * @param fn request function
+     * @return The value from the first successful call to the callable
+     */
+    public static <T> T runUntilPass(int retry, Callable<T> fn) {
+        for (int i = 0; i < retry; i++) {
+            try {
+                LOGGER.debug("Running command, attempt: {}", i);
+                return fn.call();
+            } catch (Exception | AssertionError ex) {
+                LOGGER.warn("Command failed", ex);
+            }
+            try {
+                Thread.sleep(1000);
+            } catch (InterruptedException e) {
+                throw new RuntimeException(e);
+            }
+        }
+        throw new IllegalStateException(String.format("Command wasn't pass in %s attempts", retry));
+    }
 }

src/main/java/io/odh/test/framework/manager/resources/DataScienceInitializationResource.java

@@ -28,9 +28,9 @@ public DSCInitialization get(String namespace, String name) {
     @Override
     public void create(DSCInitialization resource) {
         if (get("", resource.getMetadata().getName()) == null) {
-            dsciClient().resource(resource).create();
+            TestUtils.runUntilPass(5, () -> dsciClient().resource(resource).create());
         } else {
-            dsciClient().resource(resource).update();
+            TestUtils.runUntilPass(5, () -> dsciClient().resource(resource).update());
         }
     }
 
@@ -41,7 +41,7 @@ public void delete(DSCInitialization resource) {
 
     @Override
     public void update(DSCInitialization resource) {
-        dsciClient().resource(resource).update();
+        TestUtils.runUntilPass(5, () -> dsciClient().resource(resource).update());
     }
 
     @Override