diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java index 1892b13d372f6..eb12827ca7bd7 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java @@ -450,8 +450,16 @@ public AbfsConfiguration(final Configuration rawConfig, String accountName) this(rawConfig, accountName, AbfsServiceType.DFS); } + /** + * Returns the account type as per the user configuration. Gets the account + * specific value if it exists, then looks for an account agnostic value. + * If not configured driver makes additional getAcl call to determine + * the account type during file system initialization. + * @return TRUE/FALSE value if configured, UNKNOWN if not configured. + */ public Trilean getIsNamespaceEnabledAccount() { - return Trilean.getTrilean(isNamespaceEnabledAccount); + return Trilean.getTrilean( + getString(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, isNamespaceEnabledAccount)); } /** diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java index e903421f469a7..510ed733e2da1 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java @@ -405,7 +405,7 @@ private synchronized boolean getNamespaceEnabledInformationFromServer( } try { LOG.debug("Get root ACL status"); - getClient().getAclStatus(AbfsHttpConstants.ROOT_PATH, tracingContext); + getClient(AbfsServiceType.DFS).getAclStatus(AbfsHttpConstants.ROOT_PATH, tracingContext); // If getAcl succeeds, namespace is enabled. isNamespaceEnabled = Trilean.getTrilean(true); } catch (AbfsRestOperationException ex) { diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemE2E.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemE2E.java index c7a1fa91a98f9..00e9fc5b6143a 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemE2E.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemE2E.java @@ -40,6 +40,7 @@ import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_HTTP_READ_TIMEOUT; import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_MAX_IO_RETRIES; import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_TOLERATE_CONCURRENT_APPEND; +import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ACCOUNT_IS_HNS_ENABLED; import static org.apache.hadoop.fs.contract.ContractTestUtils.assertPathDoesNotExist; import static org.apache.hadoop.fs.contract.ContractTestUtils.assertPathExists; import static org.apache.hadoop.test.LambdaTestUtils.intercept; @@ -259,6 +260,9 @@ public void testHttpReadTimeout() throws Exception { public void testHttpTimeouts(int connectionTimeoutMs, int readTimeoutMs) throws Exception { + // This is to make sure File System creation goes through before network calls start failing. + assumeValidTestConfigPresent(this.getRawConfiguration(), FS_AZURE_ACCOUNT_IS_HNS_ENABLED); + Configuration conf = this.getRawConfiguration(); // set to small values that will cause timeouts conf.setInt(AZURE_HTTP_CONNECTION_TIMEOUT, connectionTimeoutMs); diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemInitAndCreate.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemInitAndCreate.java index 44b1685a3f3f2..69d4f79f8b099 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemInitAndCreate.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemInitAndCreate.java @@ -28,6 +28,7 @@ import org.junit.Test; import org.mockito.Mockito; +import org.apache.hadoop.fs.azurebfs.constants.AbfsServiceType; import org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys; import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AbfsRestOperationException; import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AzureBlobFileSystemException; @@ -76,7 +77,7 @@ public void testGetAclCallOnHnsConfigAbsence() throws Exception { getRawConfiguration())); AzureBlobFileSystemStore store = Mockito.spy(fs.getAbfsStore()); AbfsClient client = Mockito.spy(fs.getAbfsClient()); - Mockito.doReturn(client).when(store).getClient(); + Mockito.doReturn(client).when(store).getClient(AbfsServiceType.DFS); Mockito.doThrow(TrileanConversionException.class) .when(store) diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestGetNameSpaceEnabled.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestGetNameSpaceEnabled.java index d4c58c9705a68..5732ce8cfea70 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestGetNameSpaceEnabled.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestGetNameSpaceEnabled.java @@ -28,6 +28,7 @@ import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.azurebfs.constants.AbfsServiceType; import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AbfsRestOperationException; import org.apache.hadoop.fs.azurebfs.services.AbfsClient; import org.apache.hadoop.fs.azurebfs.services.AbfsDfsClient; @@ -41,6 +42,8 @@ import static java.net.HttpURLConnection.HTTP_INTERNAL_ERROR; import static java.net.HttpURLConnection.HTTP_NOT_FOUND; import static java.net.HttpURLConnection.HTTP_UNAVAILABLE; +import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_MAX_IO_RETRIES; +import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.accountProperty; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.doReturn; @@ -126,6 +129,8 @@ private AzureBlobFileSystem getNewFSWithHnsConf( Configuration rawConfig = new Configuration(); rawConfig.addResource(TEST_CONFIGURATION_FILE_NAME); rawConfig.set(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, isNamespaceEnabledAccount); + rawConfig.set(accountProperty(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, + this.getAccountName()), isNamespaceEnabledAccount); rawConfig .setBoolean(AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION, true); rawConfig.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, @@ -247,7 +252,7 @@ private void ensureGetAclDetermineHnsStatusAccuratelyInternal(int statusCode, AzureBlobFileSystemStore store = Mockito.spy(getFileSystem().getAbfsStore()); AbfsClient mockClient = mock(AbfsClient.class); store.setNamespaceEnabled(Trilean.UNKNOWN); - doReturn(mockClient).when(store).getClient(); + doReturn(mockClient).when(store).getClient(AbfsServiceType.DFS); AbfsRestOperationException ex = new AbfsRestOperationException( statusCode, null, Integer.toString(statusCode), null); doThrow(ex).when(mockClient).getAclStatus(anyString(), any(TracingContext.class)); @@ -271,4 +276,65 @@ private void ensureGetAclDetermineHnsStatusAccuratelyInternal(int statusCode, Mockito.verify(mockClient, times(1)) .getAclStatus(anyString(), any(TracingContext.class)); } + + @Test + public void testAccountSpecificConfig() throws Exception { + Configuration rawConfig = new Configuration(); + rawConfig.addResource(TEST_CONFIGURATION_FILE_NAME); + rawConfig.unset(FS_AZURE_ACCOUNT_IS_HNS_ENABLED); + rawConfig.unset(accountProperty(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, + this.getAccountName())); + String testAccountName = "testAccount.dfs.core.windows.net"; + String otherAccountName = "otherAccount.dfs.core.windows.net"; + String defaultUri = this.getTestUrl().replace(this.getAccountName(), testAccountName); + String otherUri = this.getTestUrl().replace(this.getAccountName(), otherAccountName); + + // Set both account specific and account agnostic config for test account + rawConfig.set(accountProperty(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, testAccountName), FALSE_STR); + rawConfig.set(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, TRUE_STR); + // Assert that account specific config takes precedence + rawConfig.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, defaultUri); + assertFileSystemInitWithExpectedHNSSettings(rawConfig, false); + // Assert that other account still uses account agnostic config + rawConfig.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, otherUri); + assertFileSystemInitWithExpectedHNSSettings(rawConfig, true); + + // Set only the account specific config for test account + rawConfig.set(accountProperty(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, testAccountName), FALSE_STR); + rawConfig.unset(FS_AZURE_ACCOUNT_IS_HNS_ENABLED); + // Assert that only account specific config is enough for test account + rawConfig.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, defaultUri); + assertFileSystemInitWithExpectedHNSSettings(rawConfig, false); + + // Set only account agnostic config + rawConfig.set(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, FALSE_STR); + rawConfig.unset(accountProperty(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, testAccountName)); + rawConfig.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, defaultUri); + assertFileSystemInitWithExpectedHNSSettings(rawConfig, false); + + // Unset both account specific and account agnostic config + rawConfig.unset(FS_AZURE_ACCOUNT_IS_HNS_ENABLED); + rawConfig.unset(accountProperty(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, testAccountName)); + rawConfig.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, defaultUri); + rawConfig.set(AZURE_MAX_IO_RETRIES, "0"); + // Assert that file system init fails with UnknownHost exception as getAcl() is needed. + try { + assertFileSystemInitWithExpectedHNSSettings(rawConfig, false); + } catch (Exception e) { + Assertions.assertThat(e.getCause().getMessage()) + .describedAs("getAcl() to determine HNS Nature of account should" + + "fail with Unknown Host Exception").contains("UnknownHostException"); + } + } + + private void assertFileSystemInitWithExpectedHNSSettings( + Configuration configuration, boolean expectedIsHnsEnabledValue) throws IOException { + try (AzureBlobFileSystem fs = (AzureBlobFileSystem) FileSystem.newInstance(configuration)) { + Assertions.assertThat(getIsNamespaceEnabled(fs)).describedAs( + "getIsNamespaceEnabled should return true when the " + + "account specific config is not set").isEqualTo(expectedIsHnsEnabledValue); + } catch (Exception e) { + throw e; + } + } }