- * When it fails to authenticate, it raises a - * {@link NoAwsCredentialsException} which can be recognized by retry handlers + * The provider is initialized with async credential refresh enabled to be less + * brittle against transient network issues. + *
+ * If the ContainerCredentialsProvider fails to authenticate, then an instance of + * {@link InstanceProfileCredentialsProvider} is created and attemped to + * be used instead, again with async credential refresh enabled. + *
+ * If both credential providers fail, a {@link NoAwsCredentialsException} + * is thrown, which can be recognized by retry handlers * as a non-recoverable failure. *
* It is implicitly public; marked evolving as we can change its semantics. - * */ @InterfaceAudience.Public @InterfaceStability.Evolving public class IAMInstanceCredentialsProvider implements AwsCredentialsProvider, Closeable { - private final AwsCredentialsProvider containerCredentialsProvider = - ContainerCredentialsProvider.builder().build(); + private static final Logger LOG = + LoggerFactory.getLogger(IAMInstanceCredentialsProvider.class); + + /** + * The credentials provider. + * Initially a container credentials provider, but if that fails + * fall back to the instance profile provider. + */ + private HttpCredentialsProvider iamCredentialsProvider; + + /** + * Is the container credentials provider in use? + */ + private boolean isContainerCredentialsProvider; + /** + * Constructor. + * Build credentials provider with async refresh, + * mark {@link #isContainerCredentialsProvider} as true. + */ public IAMInstanceCredentialsProvider() { + isContainerCredentialsProvider = true; + iamCredentialsProvider = ContainerCredentialsProvider.builder() + .asyncCredentialUpdateEnabled(true) + .build(); } /** @@ -65,9 +97,16 @@ public AwsCredentials resolveCredentials() { try { return getCredentials(); } catch (SdkClientException e) { + + // if the exception contains an IOE, extract it + // so its type is the immediate cause of this new exception. + Throwable t = e; + final IOException ioe = maybeExtractIOException("IAM endpoint", e); + if (ioe != null) { + t = ioe; + } throw new NoAwsCredentialsException("IAMInstanceCredentialsProvider", - e.getMessage(), - e); + e.getMessage(), t); } } @@ -78,23 +117,52 @@ public AwsCredentials resolveCredentials() { * * @return credentials */ - private AwsCredentials getCredentials() { + private synchronized AwsCredentials getCredentials() { try { - return containerCredentialsProvider.resolveCredentials(); + return iamCredentialsProvider.resolveCredentials(); } catch (SdkClientException e) { - return InstanceProfileCredentialsProvider.create().resolveCredentials(); + LOG.debug("Failed to get credentials from container provider,", e); + if (isContainerCredentialsProvider) { + // create instance profile provider + LOG.debug("Switching to instance provider", e); + + // close it to shut down any thread + iamCredentialsProvider.close(); + isContainerCredentialsProvider = false; + iamCredentialsProvider = InstanceProfileCredentialsProvider.builder() + .asyncCredentialUpdateEnabled(true) + .build(); + return iamCredentialsProvider.resolveCredentials(); + } else { + // already using instance profile provider, so fail + throw e; + } + } } + /** + * Is this a container credentials provider? + * @return true if the container credentials provider is in use; + * false for InstanceProfileCredentialsProvider + */ + public boolean isContainerCredentialsProvider() { + return isContainerCredentialsProvider; + } + @Override - public void close() throws IOException { - // no-op. + public synchronized void close() throws IOException { + // this be true but just for safety... + if (iamCredentialsProvider != null) { + iamCredentialsProvider.close(); + } } @Override public String toString() { return "IAMInstanceCredentialsProvider{" + - "containerCredentialsProvider=" + containerCredentialsProvider + + "credentialsProvider=" + iamCredentialsProvider + + ", isContainerCredentialsProvider=" + isContainerCredentialsProvider + '}'; } } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ErrorTranslation.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ErrorTranslation.java index 7b5190becc487..f8a1f907bb3b1 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ErrorTranslation.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ErrorTranslation.java @@ -79,7 +79,7 @@ public static boolean isObjectNotFound(AwsServiceException e) { * @param thrown exception * @return a translated exception or null. */ - public static IOException maybeExtractNetworkException(String path, Throwable thrown) { + public static IOException maybeExtractIOException(String path, Throwable thrown) { if (thrown == null) { return null; @@ -100,7 +100,9 @@ public static IOException maybeExtractNetworkException(String path, Throwable th // as a new instance is created through reflection, the // class of the returned instance will be that of the innermost, // unless no suitable constructor is available. - return wrapWithInnerIOE(path, thrown, (IOException) cause); + final IOException ioe = (IOException) cause; + + return wrapWithInnerIOE(path, thrown, ioe); } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/TestIAMInstanceCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/TestIAMInstanceCredentialsProvider.java new file mode 100644 index 0000000000000..c8986eab9b850 --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/TestIAMInstanceCredentialsProvider.java @@ -0,0 +1,107 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.s3a.auth; + +import java.io.IOException; + +import org.assertj.core.api.Assertions; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import software.amazon.awssdk.auth.credentials.AwsCredentials; + +import org.apache.hadoop.test.AbstractHadoopTestBase; + +/** + * Unit tests for IAMInstanceCredentials provider. + * This is a bit tricky as we don't want to require running in EC2, + * but nor do we want a test which doesn't work in EC2. + */ +public class TestIAMInstanceCredentialsProvider extends AbstractHadoopTestBase { + + private static final Logger LOG = + LoggerFactory.getLogger(TestIAMInstanceCredentialsProvider.class); + + /** + * Error string from + * software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider, + * if IAM resolution has been disabled: {@value}. + */ + public static final String DISABLED = + "IMDS credentials have been disabled by environment variable or system property"; + + /** + * Test an immediate create/close. + */ + @Test + public void testIAMInstanceCredentialsProviderClose() throws Throwable { + new IAMInstanceCredentialsProvider().close(); + } + + /** + * Test instantiation. + * Multiple outcomes depending on host setup. + *