diff --git a/catalog/files/impl/src/main/java/org/projectnessie/catalog/files/adls/AdlsObjectIO.java b/catalog/files/impl/src/main/java/org/projectnessie/catalog/files/adls/AdlsObjectIO.java index f718697b1ee..6a3350be247 100644 --- a/catalog/files/impl/src/main/java/org/projectnessie/catalog/files/adls/AdlsObjectIO.java +++ b/catalog/files/impl/src/main/java/org/projectnessie/catalog/files/adls/AdlsObjectIO.java @@ -189,6 +189,14 @@ void icebergConfigOverrides( storageLocations.warehouseLocation()); String storageAccount = storageAccounts.iterator().next(); + // Iceberg PR https://github.com/apache/iceberg/pull/11504 (since Iceberg 1.7.1) introduced a + // behavioral change. Before that PR, the whole storage account including the domain name (e.g. + // myaccount.dfs.core.windows.net) was used in Iceberg's `ADLSLocation` as the lookup key in + // `AzureProperties.applyClientConfiguration()`. After that PR, only the first part (e.g. + // myaccount) is used. This is a breaking change, as it requires changes to user configurations. + int dot = storageAccount.indexOf('.'); + Optional storageAccountShort = + dot != -1 ? Optional.of(storageAccount.substring(0, dot)) : Optional.empty(); // We just need one of the locations to resolve the GCS bucket options. Any should work, because // it's all for the same table. @@ -201,7 +209,12 @@ void icebergConfigOverrides( AdlsNamedFileSystemOptions fileSystemOptions = adlsOptions.resolveOptionsForUri(loc); fileSystemOptions .endpoint() - .ifPresent(e -> config.accept(ADLS_CONNECTION_STRING_PREFIX + storageAccount, e)); + .ifPresent( + e -> { + config.accept(ADLS_CONNECTION_STRING_PREFIX + storageAccount, e); + storageAccountShort.ifPresent( + account -> config.accept(ADLS_CONNECTION_STRING_PREFIX + account, e)); + }); adlsOptions .readBlockSize() .ifPresent(r -> config.accept(ADLS_READ_BLOCK_SIZE_BYTES, Integer.toString(r))); @@ -211,7 +224,12 @@ void icebergConfigOverrides( clientSupplier .generateUserDelegationSas(storageLocations, fileSystemOptions) - .ifPresent(sasToken -> config.accept(ADLS_SAS_TOKEN_PREFIX + storageAccount, sasToken)); + .ifPresent( + sasToken -> { + config.accept(ADLS_SAS_TOKEN_PREFIX + storageAccount, sasToken); + storageAccountShort.ifPresent( + account -> config.accept(ADLS_SAS_TOKEN_PREFIX + account, sasToken)); + }); } void icebergConfigDefaults(BiConsumer config) { diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 98eeea612bb..42af1e365f8 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -9,7 +9,7 @@ gatling = "3.13.1" googleCloudBigdataoss = "3.0.4" googleJavaFormat = "1.25.0" hadoop = "3.4.1" -iceberg = "1.7.0" # While bumping this version, 'nessieClientVersion' must also be updated. +iceberg = "1.7.1" # While bumping this version, 'nessieClientVersion' must also be updated. immutables = "2.10.1" jandex = "3.1.8" jmh = "1.37" diff --git a/servers/quarkus-server/src/test/java/org/projectnessie/server/catalog/TestAdlsIcebergCatalog.java b/servers/quarkus-server/src/test/java/org/projectnessie/server/catalog/TestAdlsIcebergCatalog.java index 13713cd3c6f..64599ccf9e6 100644 --- a/servers/quarkus-server/src/test/java/org/projectnessie/server/catalog/TestAdlsIcebergCatalog.java +++ b/servers/quarkus-server/src/test/java/org/projectnessie/server/catalog/TestAdlsIcebergCatalog.java @@ -19,7 +19,9 @@ import io.quarkus.test.junit.QuarkusTest; import io.quarkus.test.junit.TestProfile; +import java.util.Map; import java.util.UUID; +import org.testcontainers.shaded.com.google.common.collect.ImmutableMap; @QuarkusTest @TestProfile(AdlsUnitTestProfile.class) @@ -34,4 +36,13 @@ protected String temporaryLocation() { protected String scheme() { return "adls"; } + + @Override + protected Map catalogOptions() { + return ImmutableMap.builder() + .putAll(super.catalogOptions()) + .put("adls.auth.shared-key.account.name", "account@account.dfs.core.windows.net") + .put("adls.auth.shared-key.account.key", "key") + .build(); + } }