[Backport 1.x] Support for Google Application Default Credentials (#8394)

CHANGELOG.md

@@ -4,6 +4,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 ## [Unreleased 1.x]
 ### Added
 - Maintainer approval check ([#11378](https://github.com/opensearch-project/OpenSearch/pull/11378))
+- Add support for Google Application Default Credentials in repository-gcs ([#8394](https://github.com/opensearch-project/OpenSearch/pull/8394))
 
 ### Dependencies
 - Bump `netty` from 4.1.96.Final to 4.1.97.Final ([#9553](https://github.com/opensearch-project/OpenSearch/pull/9553))

plugins/repository-gcs/src/main/java/org/opensearch/repositories/gcs/GoogleApplicationDefaultCredentials.java

@@ -0,0 +1,33 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.repositories.gcs;
+
+import com.google.auth.oauth2.GoogleCredentials;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import java.io.IOException;
+
+/**
+ * This class facilitates to fetch Application Default Credentials
+ * see <a href="https://cloud.google.com/docs/authentication/application-default-credentials">How Application Default Credentials works</a>
+ */
+public class GoogleApplicationDefaultCredentials {
+    private static final Logger logger = LogManager.getLogger(GoogleApplicationDefaultCredentials.class);
+
+    public GoogleCredentials get() {
+        GoogleCredentials credentials = null;
+        try {
+            credentials = SocketAccess.doPrivilegedIOException(GoogleCredentials::getApplicationDefault);
+        } catch (IOException e) {
+            logger.error("Failed to retrieve \"Application Default Credentials\"", e);
+        }
+        return credentials;
+    }
+}

plugins/repository-gcs/src/main/java/org/opensearch/repositories/gcs/GoogleCloudStorageService.java

@@ -36,6 +36,7 @@
 import com.google.api.client.http.HttpRequestInitializer;
 import com.google.api.client.http.HttpTransport;
 import com.google.api.client.http.javanet.NetHttpTransport;
+import com.google.auth.oauth2.GoogleCredentials;
 import com.google.auth.oauth2.ServiceAccountCredentials;
 import com.google.cloud.ServiceOptions;
 import com.google.cloud.http.HttpTransportOptions;
@@ -68,6 +69,16 @@ public class GoogleCloudStorageService {
      */
     private volatile Map<String, Storage> clientCache = emptyMap();
 
+    final private GoogleApplicationDefaultCredentials googleApplicationDefaultCredentials;
+
+    public GoogleCloudStorageService() {
+        this.googleApplicationDefaultCredentials = new GoogleApplicationDefaultCredentials();
+    }
+
+    public GoogleCloudStorageService(GoogleApplicationDefaultCredentials googleApplicationDefaultCredentials) {
+        this.googleApplicationDefaultCredentials = googleApplicationDefaultCredentials;
+    }
+
     /**
      * Refreshes the client settings and clears the client cache. Subsequent calls to
      * {@code GoogleCloudStorageService#client} will return new clients constructed
@@ -195,10 +206,11 @@ StorageOptions createStorageOptions(
             storageOptionsBuilder.setProjectId(clientSettings.getProjectId());
         }
         if (clientSettings.getCredential() == null) {
-            logger.warn(
-                "\"Application Default Credentials\" are not supported out of the box."
-                    + " Additional file system permissions have to be granted to the plugin."
-            );
+            logger.info("\"Application Default Credentials\" will be in use");
+            final GoogleCredentials credentials = googleApplicationDefaultCredentials.get();
+            if (credentials != null) {
+                storageOptionsBuilder.setCredentials(credentials);
+            }
         } else {
             ServiceAccountCredentials serviceAccountCredentials = clientSettings.getCredential();
             // override token server URI

plugins/repository-gcs/src/test/java/org/opensearch/repositories/gcs/GoogleCloudStorageServiceTests.java

@@ -33,40 +33,49 @@
 package org.opensearch.repositories.gcs;
 
 import com.google.auth.Credentials;
+import com.google.auth.oauth2.GoogleCredentials;
 import com.google.cloud.http.HttpTransportOptions;
 import com.google.cloud.storage.Storage;
 
 import org.opensearch.common.bytes.BytesReference;
+import com.google.cloud.storage.StorageOptions;
 import org.opensearch.common.settings.MockSecureSettings;
 import org.opensearch.common.settings.Setting;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.common.xcontent.XContentBuilder;
 import org.opensearch.test.OpenSearchTestCase;
+import org.hamcrest.MatcherAssert;
 import org.hamcrest.Matchers;
 
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
 import java.security.KeyPair;
 import java.security.KeyPairGenerator;
 import java.util.Base64;
 import java.util.Locale;
 import java.util.UUID;
 
+import org.mockito.Mockito;
+
 import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.containsString;
 
 public class GoogleCloudStorageServiceTests extends OpenSearchTestCase {
 
+    final TimeValue connectTimeValue = TimeValue.timeValueNanos(randomIntBetween(0, 2000000));
+    final TimeValue readTimeValue = TimeValue.timeValueNanos(randomIntBetween(0, 2000000));
+    final String applicationName = randomAlphaOfLength(randomIntBetween(1, 10)).toLowerCase(Locale.ROOT);
+    final String endpoint = randomFrom("http://", "https://")
+        + randomFrom("www.opensearch.org", "www.googleapis.com", "localhost/api", "google.com/oauth")
+        + ":"
+        + randomIntBetween(1, 65535);
+    final String projectIdName = randomAlphaOfLength(randomIntBetween(1, 10)).toLowerCase(Locale.ROOT);
+
     public void testClientInitializer() throws Exception {
         final String clientName = randomAlphaOfLength(randomIntBetween(1, 10)).toLowerCase(Locale.ROOT);
-        final TimeValue connectTimeValue = TimeValue.timeValueNanos(randomIntBetween(0, 2000000));
-        final TimeValue readTimeValue = TimeValue.timeValueNanos(randomIntBetween(0, 2000000));
-        final String applicationName = randomAlphaOfLength(randomIntBetween(1, 10)).toLowerCase(Locale.ROOT);
-        final String endpoint = randomFrom("http://", "https://")
-            + randomFrom("www.opensearch.org", "www.googleapis.com", "localhost/api", "google.com/oauth")
-            + ":"
-            + randomIntBetween(1, 65535);
-        final String projectIdName = randomAlphaOfLength(randomIntBetween(1, 10)).toLowerCase(Locale.ROOT);
         final Settings settings = Settings.builder()
             .put(
                 GoogleCloudStorageClientSettings.CONNECT_TIMEOUT_SETTING.getConcreteSettingForNamespace(clientName).getKey(),
@@ -83,31 +92,35 @@ public void testClientInitializer() throws Exception {
             .put(GoogleCloudStorageClientSettings.ENDPOINT_SETTING.getConcreteSettingForNamespace(clientName).getKey(), endpoint)
             .put(GoogleCloudStorageClientSettings.PROJECT_ID_SETTING.getConcreteSettingForNamespace(clientName).getKey(), projectIdName)
             .build();
-        final GoogleCloudStorageService service = new GoogleCloudStorageService();
+        GoogleCredentials mockGoogleCredentials = Mockito.mock(GoogleCredentials.class);
+        GoogleApplicationDefaultCredentials mockDefaultCredentials = Mockito.mock(GoogleApplicationDefaultCredentials.class);
+        Mockito.when(mockDefaultCredentials.get()).thenReturn(mockGoogleCredentials);
+
+        final GoogleCloudStorageService service = new GoogleCloudStorageService(mockDefaultCredentials);
         service.refreshAndClearCache(GoogleCloudStorageClientSettings.load(settings));
         GoogleCloudStorageOperationsStats statsCollector = new GoogleCloudStorageOperationsStats("bucket");
         final IllegalArgumentException e = expectThrows(
             IllegalArgumentException.class,
             () -> service.client("another_client", "repo", statsCollector)
         );
-        assertThat(e.getMessage(), Matchers.startsWith("Unknown client name"));
+        MatcherAssert.assertThat(e.getMessage(), Matchers.startsWith("Unknown client name"));
         assertSettingDeprecationsAndWarnings(
             new Setting<?>[] { GoogleCloudStorageClientSettings.APPLICATION_NAME_SETTING.getConcreteSettingForNamespace(clientName) }
         );
         final Storage storage = service.client(clientName, "repo", statsCollector);
-        assertThat(storage.getOptions().getApplicationName(), Matchers.containsString(applicationName));
-        assertThat(storage.getOptions().getHost(), Matchers.is(endpoint));
-        assertThat(storage.getOptions().getProjectId(), Matchers.is(projectIdName));
-        assertThat(storage.getOptions().getTransportOptions(), Matchers.instanceOf(HttpTransportOptions.class));
-        assertThat(
+        MatcherAssert.assertThat(storage.getOptions().getApplicationName(), Matchers.containsString(applicationName));
+        MatcherAssert.assertThat(storage.getOptions().getHost(), Matchers.is(endpoint));
+        MatcherAssert.assertThat(storage.getOptions().getProjectId(), Matchers.is(projectIdName));
+        MatcherAssert.assertThat(storage.getOptions().getTransportOptions(), Matchers.instanceOf(HttpTransportOptions.class));
+        MatcherAssert.assertThat(
             ((HttpTransportOptions) storage.getOptions().getTransportOptions()).getConnectTimeout(),
             Matchers.is((int) connectTimeValue.millis())
         );
-        assertThat(
+        MatcherAssert.assertThat(
             ((HttpTransportOptions) storage.getOptions().getTransportOptions()).getReadTimeout(),
             Matchers.is((int) readTimeValue.millis())
         );
-        assertThat(storage.getOptions().getCredentials(), Matchers.nullValue(Credentials.class));
+        MatcherAssert.assertThat(storage.getOptions().getCredentials(), Matchers.instanceOf(Credentials.class));
     }
 
     public void testReinitClientSettings() throws Exception {
@@ -123,33 +136,33 @@ public void testReinitClientSettings() throws Exception {
             final GoogleCloudStorageService storageService = plugin.storageService;
             GoogleCloudStorageOperationsStats statsCollector = new GoogleCloudStorageOperationsStats("bucket");
             final Storage client11 = storageService.client("gcs1", "repo1", statsCollector);
-            assertThat(client11.getOptions().getProjectId(), equalTo("project_gcs11"));
+            MatcherAssert.assertThat(client11.getOptions().getProjectId(), equalTo("project_gcs11"));
             final Storage client12 = storageService.client("gcs2", "repo2", statsCollector);
-            assertThat(client12.getOptions().getProjectId(), equalTo("project_gcs12"));
+            MatcherAssert.assertThat(client12.getOptions().getProjectId(), equalTo("project_gcs12"));
             // client 3 is missing
             final IllegalArgumentException e1 = expectThrows(
                 IllegalArgumentException.class,
                 () -> storageService.client("gcs3", "repo3", statsCollector)
             );
-            assertThat(e1.getMessage(), containsString("Unknown client name [gcs3]."));
+            MatcherAssert.assertThat(e1.getMessage(), containsString("Unknown client name [gcs3]."));
             // update client settings
             plugin.reload(settings2);
             // old client 1 not changed
-            assertThat(client11.getOptions().getProjectId(), equalTo("project_gcs11"));
+            MatcherAssert.assertThat(client11.getOptions().getProjectId(), equalTo("project_gcs11"));
             // new client 1 is changed
             final Storage client21 = storageService.client("gcs1", "repo1", statsCollector);
-            assertThat(client21.getOptions().getProjectId(), equalTo("project_gcs21"));
+            MatcherAssert.assertThat(client21.getOptions().getProjectId(), equalTo("project_gcs21"));
             // old client 2 not changed
-            assertThat(client12.getOptions().getProjectId(), equalTo("project_gcs12"));
+            MatcherAssert.assertThat(client12.getOptions().getProjectId(), equalTo("project_gcs12"));
             // new client2 is gone
             final IllegalArgumentException e2 = expectThrows(
                 IllegalArgumentException.class,
                 () -> storageService.client("gcs2", "repo2", statsCollector)
             );
-            assertThat(e2.getMessage(), containsString("Unknown client name [gcs2]."));
+            MatcherAssert.assertThat(e2.getMessage(), containsString("Unknown client name [gcs2]."));
             // client 3 emerged
             final Storage client23 = storageService.client("gcs3", "repo3", statsCollector);
-            assertThat(client23.getOptions().getProjectId(), equalTo("project_gcs23"));
+            MatcherAssert.assertThat(client23.getOptions().getProjectId(), equalTo("project_gcs23"));
         }
     }
 
@@ -194,4 +207,71 @@ public void testToTimeout() {
         assertEquals(-1, GoogleCloudStorageService.toTimeout(TimeValue.ZERO).intValue());
         assertEquals(0, GoogleCloudStorageService.toTimeout(TimeValue.MINUS_ONE).intValue());
     }
+
+    /**
+     * The following method test the Google Application Default Credential instead of
+     * using service account file.
+     * Considered use of JUnit Mocking due to static method GoogleCredentials.getApplicationDefault
+     * and avoiding environment variables to set which later use GCE.
+     * @throws URISyntaxException when URI used for creating GoogleCloudStorageClientSetting cannot be phased
+     */
+    public void testApplicationDefaultCredential() throws URISyntaxException {
+        GoogleCloudStorageClientSettings settings = getGCSClientSettingsWithoutCredentials();
+        GoogleCredentials mockGoogleCredentials = Mockito.mock(GoogleCredentials.class);
+        HttpTransportOptions mockHttpTransportOptions = Mockito.mock(HttpTransportOptions.class);
+        GoogleApplicationDefaultCredentials mockDefaultCredentials = Mockito.mock(GoogleApplicationDefaultCredentials.class);
+        Mockito.when(mockDefaultCredentials.get()).thenReturn(mockGoogleCredentials);
+
+        GoogleCloudStorageService service = new GoogleCloudStorageService(mockDefaultCredentials);
+        StorageOptions storageOptions = service.createStorageOptions(settings, mockHttpTransportOptions);
+        assertNotNull(storageOptions);
+        assertEquals(storageOptions.getCredentials().toString(), mockGoogleCredentials.toString());
+    }
+
+    /**
+     * The application default credential throws exception when there are
+     * no Environment Variables provided or Google Compute Engine is not running
+     * @throws URISyntaxException when URI used for creating GoogleCloudStorageClientSetting cannot be phased
+     */
+    public void testApplicationDefaultCredentialsWhenNoSettingProvided() throws URISyntaxException {
+        GoogleCloudStorageClientSettings settings = getGCSClientSettingsWithoutCredentials();
+        HttpTransportOptions mockHttpTransportOptions = Mockito.mock(HttpTransportOptions.class);
+        GoogleCloudStorageService service = new GoogleCloudStorageService();
+        StorageOptions storageOptions = service.createStorageOptions(settings, mockHttpTransportOptions);
+
+        Exception exception = assertThrows(IOException.class, GoogleCredentials::getApplicationDefault);
+        assertNotNull(storageOptions);
+        assertNull(storageOptions.getCredentials());
+        MatcherAssert.assertThat(exception.getMessage(), containsString("The Application Default Credentials are not available"));
+    }
+
+    /**
+     * The application default credential throws IOException when it is
+     * used without GoogleCloudStorageService
+     */
+    public void testDefaultCredentialsThrowsExceptionWithoutGCStorageService() {
+        GoogleApplicationDefaultCredentials googleApplicationDefaultCredentials = new GoogleApplicationDefaultCredentials();
+        GoogleCredentials credentials = googleApplicationDefaultCredentials.get();
+        assertNull(credentials);
+        Exception exception = assertThrows(IOException.class, GoogleCredentials::getApplicationDefault);
+        MatcherAssert.assertThat(exception.getMessage(), containsString("The Application Default Credentials are not available"));
+    }
+
+    /**
+     * This is a helper method to provide GCS Client settings without credentials
+     * @return GoogleCloudStorageClientSettings
+     * @throws URISyntaxException when URI used for creating GoogleCloudStorageClientSetting cannot be phased
+     */
+    private GoogleCloudStorageClientSettings getGCSClientSettingsWithoutCredentials() throws URISyntaxException {
+        return new GoogleCloudStorageClientSettings(
+            null,
+            endpoint,
+            projectIdName,
+            connectTimeValue,
+            readTimeValue,
+            applicationName,
+            new URI("")
+        );
+    }
+
 }