[BUG-1402] Return HTTP 409 on version confict when creating & updating tenants - added relevant unit test

src/integrationTest/java/org/opensearch/test/framework/AsyncActions.java

@@ -49,7 +49,7 @@ public static <T> List<CompletableFuture<T>> generate(final Callable<T> generato
      * @return Completed results from the futures
      */
     public static <T> List<T> getAll(final List<CompletableFuture<T>> futures, final int n, final TimeUnit unit) {
-        LOG.info("Starting to wait for " + futures.size() + " futures to complete in " + unit.toSeconds(n) + " seconds.");
+        LOG.info("Starting to wait for " + futures.size() + " futures to complete in for " + unit.toSeconds(n) + " seconds.");
         final long startTimeMs = System.currentTimeMillis();
         final CompletableFuture<Void> futuresCompleted = CompletableFuture.allOf(futures.toArray(new CompletableFuture[0]));
         try {
@@ -58,7 +58,7 @@ public static <T> List<T> getAll(final List<CompletableFuture<T>> futures, final
             final long completedFuturesCount = futures.stream().filter(CompletableFuture::isDone).count();
             final String perfReport = calculatePerfReport(startTimeMs, completedFuturesCount);
             throw new RuntimeException(
-                "Unable to wait for all futures to complete, of "
+                "Unable to wait for all futures to compete, of "
                     + futures.size()
                     + " futures "
                     + completedFuturesCount

src/test/java/org/opensearch/security/dlic/rest/api/TenantInfoActionTest.java

@@ -12,16 +12,27 @@
 package org.opensearch.security.dlic.rest.api;
 
 import org.apache.hc.core5.http.Header;
-import org.apache.hc.core5.http.HttpStatus;
+import org.apache.http.HttpStatus;
 import org.junit.Assert;
 import org.junit.Test;
 
 import org.opensearch.common.settings.Settings;
 import org.opensearch.security.support.ConfigConstants;
 import org.opensearch.security.test.helper.rest.RestHelper;
+import org.opensearch.test.framework.AsyncActions;
 
 import static org.opensearch.security.OpenSearchSecurityPlugin.PLUGINS_PREFIX;
 
+import java.util.List;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.CoreMatchers.anyOf;
+import static org.hamcrest.MatcherAssert.assertThat;
+
 public class TenantInfoActionTest extends AbstractRestApiUnitTest {
     private String payload = "{\"hosts\":[],\"users\":[\"sarek\"],"
         + "\"backend_roles\":[\"starfleet*\",\"ambassador\"],\"and_backend_roles\":[],\"description\":\"Migrated "
@@ -85,4 +96,36 @@ public void testTenantInfoAPIUpdate() throws Exception {
         response = rh.executeGetRequest(ENDPOINT);
         Assert.assertEquals(HttpStatus.SC_OK, response.getStatusCode());
     }
+
+    @Test
+    public void testParallelPutRequests() throws Exception {
+        setup();
+
+        rh.keystore = "restapi/kirk-keystore.jks";
+        rh.sendAdminCertificate = true;
+        final String TENANTS_ENDPOINT = BASE_ENDPOINT + "/api/tenants/tenant1";
+        final String TENANTS_BODY = "{\"description\":\"create new tenant\"}";
+
+        final CountDownLatch countDownLatch = new CountDownLatch(1);
+        final List<CompletableFuture<RestHelper.HttpResponse>> conflictingRequests = AsyncActions.generate(() -> {
+            return rh.executePutRequest(TENANTS_ENDPOINT, TENANTS_BODY);
+        }, 2, 4);
+
+        // Make sure all requests start at the same time
+        countDownLatch.countDown();
+
+        AtomicInteger numCreatedResponses = new AtomicInteger();
+        AsyncActions.getAll(conflictingRequests, 1, TimeUnit.SECONDS).forEach((response) -> {
+            assertThat(response.getStatusCode(), anyOf(equalTo(HttpStatus.SC_CREATED), equalTo(HttpStatus.SC_CONFLICT))); // make sure every
+                                                                                                                          // response is
+                                                                                                                          // either 201 or
+                                                                                                                          // 409
+            if (response.getStatusCode() == HttpStatus.SC_CREATED) numCreatedResponses.getAndIncrement(); // check how many synchronous
+                                                                                                          // requests are going through
+        });
+        assertThat(numCreatedResponses.get(), equalTo(1)); // should only be one 201
+
+        RestHelper.HttpResponse getResponse = rh.executeGetRequest(TENANTS_ENDPOINT); // make sure the one 201 works
+        assertThat(getResponse.findValueInJson("tenant1" + ".description"), equalTo("create new tenant"));
+    }
 }