Merge branch 'trunk' into HDFS-17496

.github/workflows/website.yml

@@ -56,4 +56,5 @@ jobs:
           publish_dir: ./staging/hadoop-project
           user_name: 'github-actions[bot]'
           user_email: 'github-actions[bot]@users.noreply.github.com'
+          force_orphan: true
 

LICENSE-binary

@@ -216,7 +216,6 @@ com.aliyun:aliyun-java-sdk-kms:2.11.0
 com.aliyun:aliyun-java-sdk-ram:3.1.0
 com.aliyun:aliyun-java-sdk-sts:3.0.0
 com.aliyun.oss:aliyun-sdk-oss:3.13.2
-com.amazonaws:aws-java-sdk-bundle:1.12.599
 com.cedarsoftware:java-util:1.9.0
 com.cedarsoftware:json-io:2.5.1
 com.fasterxml.jackson.core:jackson-annotations:2.12.7
@@ -227,28 +226,29 @@ com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider:2.12.7
 com.fasterxml.jackson.module:jackson-module-jaxb-annotations:2.12.7
 com.fasterxml.uuid:java-uuid-generator:3.1.4
 com.fasterxml.woodstox:woodstox-core:5.4.0
+com.github.ben-manes.caffeine:caffeine:2.9.3
 com.github.davidmoten:rxjava-extras:0.8.0.17
 com.github.stephenc.jcip:jcip-annotations:1.0-1
-com.google:guice:4.0
-com.google:guice-servlet:4.0
+com.google:guice:5.1.0
+com.google:guice-servlet:5.1.0
 com.google.api.grpc:proto-google-common-protos:1.0.0
 com.google.code.gson:2.9.0
-com.google.errorprone:error_prone_annotations:2.2.0
-com.google.j2objc:j2objc-annotations:1.1
+com.google.errorprone:error_prone_annotations:2.5.1
+com.google.j2objc:j2objc-annotations:1.3
 com.google.json-simple:json-simple:1.1.1
 com.google.guava:failureaccess:1.0
 com.google.guava:guava:20.0
-com.google.guava:guava:27.0-jre
+com.google.guava:guava:32.0.1-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.microsoft.azure:azure-storage:7.0.0
 com.nimbusds:nimbus-jose-jwt:9.37.2
 com.zaxxer:HikariCP:4.0.3
 commons-beanutils:commons-beanutils:1.9.4
-commons-cli:commons-cli:1.5.0
-commons-codec:commons-codec:1.11
-commons-collections:commons-collections:3.2.2
+commons-cli:commons-cli:1.9.0
+commons-codec:commons-codec:1.15
+org.apache.commons:commons-collections4:4.4
 commons-daemon:commons-daemon:1.0.13
-commons-io:commons-io:2.14.0
+commons-io:commons-io:2.16.1
 commons-net:commons-net:3.9.0
 de.ruedigermoeller:fst:2.50
 io.grpc:grpc-api:1.53.0
@@ -297,8 +297,8 @@ io.swagger:swagger-annotations:1.5.4
 javax.inject:javax.inject:1
 net.java.dev.jna:jna:5.2.0
 net.minidev:accessors-smart:1.2
-org.apache.avro:avro:1.9.2
-org.apache.commons:commons-collections4:4.2
+org.apache.avro:avro:1.11.4
+org.apache.avro:avro:1.11.3
 org.apache.commons:commons-compress:1.26.1
 org.apache.commons:commons-configuration2:2.10.1
 org.apache.commons:commons-csv:1.9.0
@@ -318,7 +318,7 @@ org.apache.htrace:htrace-core:3.1.0-incubating
 org.apache.htrace:htrace-core4:4.1.0-incubating
 org.apache.httpcomponents:httpclient:4.5.13
 org.apache.httpcomponents:httpcore:4.4.13
-org.apache.kafka:kafka-clients:2.8.2
+org.apache.kafka:kafka-clients:3.4.0
 org.apache.kerby:kerb-admin:2.0.3
 org.apache.kerby:kerb-client:2.0.3
 org.apache.kerby:kerb-common:2.0.3
@@ -361,8 +361,8 @@ org.lz4:lz4-java:1.7.1
 org.objenesis:objenesis:2.6
 org.xerial.snappy:snappy-java:1.1.10.4
 org.yaml:snakeyaml:2.0
-org.wildfly.openssl:wildfly-openssl:1.1.3.Final
-software.amazon.awssdk:bundle:jar:2.24.6
+org.wildfly.openssl:wildfly-openssl:2.1.4.Final
+software.amazon.awssdk:bundle:2.25.53
 
 
 --------------------------------------------------------------------------------
@@ -378,8 +378,8 @@ hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/com
 hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/util/tree.h
 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/compat/{fstatat|openat|unlinkat}.h
 
-com.github.luben:zstd-jni:1.4.9-1
-dnsjava:dnsjava:2.1.7
+com.github.luben:zstd-jni:1.5.2-1
+dnsjava:dnsjava:3.6.1
 org.codehaus.woodstox:stax2-api:4.2.1
 
 
@@ -395,7 +395,7 @@ hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/d3-3.5.17.min.js
 leveldb v1.13
 
 com.google.protobuf:protobuf-java:2.5.0
-com.google.protobuf:protobuf-java:3.21.12
+com.google.protobuf:protobuf-java:3.25.3
 com.google.re2j:re2j:1.1
 com.jcraft:jsch:0.1.55
 com.thoughtworks.paranamer:paranamer:2.3
@@ -482,10 +482,10 @@ com.microsoft.azure:azure-cosmosdb-gateway:2.4.5
 com.microsoft.azure:azure-data-lake-store-sdk:2.3.3
 com.microsoft.azure:azure-keyvault-core:1.0.0
 com.microsoft.sqlserver:mssql-jdbc:6.2.1.jre7
-org.bouncycastle:bcpkix-jdk18on:1.77
-org.bouncycastle:bcprov-jdk18on:1.77
-org.bouncycastle:bcutil-jdk18on:1.77
-org.checkerframework:checker-qual:2.5.2
+org.bouncycastle:bcpkix-jdk18on:1.78.1
+org.bouncycastle:bcprov-jdk18on:1.78.1
+org.bouncycastle:bcutil-jdk18on:1.78.1
+org.checkerframework:checker-qual:3.8.0
 org.codehaus.mojo:animal-sniffer-annotations:1.21
 org.jruby.jcodings:jcodings:1.0.13
 org.jruby.joni:joni:2.1.2

dev-support/bin/create-release

@@ -205,7 +205,7 @@ function set_defaults
   DOCKERRAN=false
 
   CPU_ARCH=$(echo "$MACHTYPE" | cut -d- -f1)
-  if [ "$CPU_ARCH" = "aarch64" ]; then
+  if [[ "$CPU_ARCH" = "aarch64" || "$CPU_ARCH" = "arm64" ]]; then
     DOCKERFILE="${BASEDIR}/dev-support/docker/Dockerfile_aarch64"
   fi
 
@@ -514,7 +514,7 @@ function dockermode
 
     # we always force build with the OpenJDK JDK
     # but with the correct version
-    if [ "$CPU_ARCH" = "aarch64" ]; then
+    if [[ "$CPU_ARCH" = "aarch64" || "$CPU_ARCH" = "arm64" ]]; then
       echo "ENV JAVA_HOME /usr/lib/jvm/java-${JVM_VERSION}-openjdk-arm64"
     else
       echo "ENV JAVA_HOME /usr/lib/jvm/java-${JVM_VERSION}-openjdk-amd64"

dev-support/bin/yetus-wrapper

@@ -77,7 +77,7 @@ WANTED="$1"
 shift
 ARGV=("$@")
 
-HADOOP_YETUS_VERSION=${HADOOP_YETUS_VERSION:-0.14.0}
+HADOOP_YETUS_VERSION=${HADOOP_YETUS_VERSION:-0.14.1}
 BIN=$(yetus_abs "${BASH_SOURCE-$0}")
 BINDIR=$(dirname "${BIN}")
 
@@ -123,7 +123,7 @@ fi
 ## need to DL, etc
 ##
 
-BASEURL="https://archive.apache.org/dist/yetus/${HADOOP_YETUS_VERSION}/"
+BASEURL="https://downloads.apache.org/yetus/${HADOOP_YETUS_VERSION}/"
 TARBALL="${YETUS_PREFIX}-${HADOOP_YETUS_VERSION}-bin.tar"
 
 GPGBIN=$(command -v gpg)

dev-support/docker/Dockerfile_centos_7

@@ -30,6 +30,13 @@ COPY pkg-resolver pkg-resolver
 RUN chmod a+x pkg-resolver/*.sh pkg-resolver/*.py \
     && chmod a+r pkg-resolver/*.json
 
+######
+# Centos 7 has reached its EOL and the packages
+# are no longer available on mirror.centos.org site.
+# Please see https://www.centos.org/centos-linux-eol/
+######
+RUN pkg-resolver/set-vault-as-baseurl-centos.sh centos:7
+
 ######
 # Install packages from yum
 ######
@@ -38,8 +45,13 @@ RUN yum update -y \
     && yum groupinstall -y "Development Tools" \
     && yum install -y \
         centos-release-scl \
-        python3 \
-    && yum install -y $(pkg-resolver/resolve.py centos:7)
+        python3
+
+# Apply the script again because centos-release-scl creates new YUM repo files
+RUN pkg-resolver/set-vault-as-baseurl-centos.sh centos:7
+
+# hadolint ignore=DL3008,SC2046
+RUN yum install -y $(pkg-resolver/resolve.py centos:7)
 
 # Set GCC 9 as the default C/C++ compiler
 RUN echo "source /opt/rh/devtoolset-9/enable" >> /etc/bashrc

dev-support/docker/Dockerfile_windows_10

@@ -61,8 +61,8 @@ RUN powershell Invoke-WebRequest -URI https://cdn.azul.com/zulu/bin/zulu8.62.0.1
 RUN powershell Expand-Archive -Path $Env:TEMP\zulu8.62.0.19-ca-jdk8.0.332-win_x64.zip -DestinationPath "C:\Java"
 
 # Install Apache Maven.
-RUN powershell Invoke-WebRequest -URI https://archive.apache.org/dist/maven/maven-3/3.8.6/binaries/apache-maven-3.8.6-bin.zip -OutFile $Env:TEMP\apache-maven-3.8.6-bin.zip
-RUN powershell Expand-Archive -Path $Env:TEMP\apache-maven-3.8.6-bin.zip -DestinationPath "C:\Maven"
+RUN powershell Invoke-WebRequest -URI https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.zip -OutFile $Env:TEMP\apache-maven-3.8.8-bin.zip
+RUN powershell Expand-Archive -Path $Env:TEMP\apache-maven-3.8.8-bin.zip -DestinationPath "C:\Maven"
 
 # Install CMake 3.19.0.
 RUN powershell Invoke-WebRequest -URI https://cmake.org/files/v3.19/cmake-3.19.0-win64-x64.zip -OutFile $Env:TEMP\cmake-3.19.0-win64-x64.zip
@@ -135,7 +135,7 @@ ENV MAVEN_OPTS '-Xmx2048M -Xss128M'
 ENV IS_WINDOWS 1
 RUN setx PATH "%PATH%;%ALLUSERSPROFILE%\chocolatey\bin"
 RUN setx PATH "%PATH%;%JAVA_HOME%\bin"
-RUN setx PATH "%PATH%;C:\Maven\apache-maven-3.8.6\bin"
+RUN setx PATH "%PATH%;C:\Maven\apache-maven-3.8.8\bin"
 RUN setx PATH "%PATH%;C:\CMake\cmake-3.19.0-win64-x64\bin"
 RUN setx PATH "%PATH%;C:\ZStd"
 RUN setx PATH "%PATH%;C:\Program Files\Git\usr\bin"

dev-support/docker/pkg-resolver/install-yasm.sh

@@ -40,7 +40,7 @@ fi
 
 if [ "$version_to_install" == "1.2.0-4" ]; then
   mkdir -p /tmp/yasm &&
-    curl -L -s -S https://download-ib01.fedoraproject.org/pub/epel/7/x86_64/Packages/y/yasm-1.2.0-4.el7.x86_64.rpm \
+    curl -L -s -S https://archives.fedoraproject.org/pub/archive/epel/7/x86_64/Packages/y/yasm-1.2.0-4.el7.x86_64.rpm \
       -o /tmp/yasm-1.2.0-4.el7.x86_64.rpm &&
     rpm -Uvh /tmp/yasm-1.2.0-4.el7.x86_64.rpm
 else

dev-support/docker/pkg-resolver/set-vault-as-baseurl-centos.sh

@@ -24,7 +24,7 @@ fi
 if [ "$1" == "centos:7" ] || [ "$1" == "centos:8" ]; then
   cd /etc/yum.repos.d/ || exit &&
     sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* &&
-    sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* &&
+    sed -i 's|# *baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* &&
     yum update -y &&
     cd /root || exit
 else

hadoop-client-modules/hadoop-client-check-invariants/src/test/resources/ensure-jars-have-correct-contents.sh

@@ -51,6 +51,8 @@ allowed_expr+="|^[^-]*-default.xml$"
 allowed_expr+="|^[^-]*-version-info.properties$"
 #   * Hadoop's application classloader properties file.
 allowed_expr+="|^org.apache.hadoop.application-classloader.properties$"
+# Comes from dnsjava, not sure if relocatable.
+allowed_expr+="|^messages.properties$"
 # public suffix list used by httpcomponents
 allowed_expr+="|^mozilla/$"
 allowed_expr+="|^mozilla/public-suffix-list.txt$"

hadoop-client-modules/hadoop-client-minicluster/pom.xml

@@ -168,6 +168,10 @@
           <groupId>commons-collections</groupId>
           <artifactId>commons-collections</artifactId>
         </exclusion>
+        <exclusion>
+          <groupId>org.apache.commons</groupId>
+          <artifactId>commons-collections4</artifactId>
+        </exclusion>
         <exclusion>
           <groupId>commons-io</groupId>
           <artifactId>commons-io</artifactId>
@@ -403,7 +407,7 @@
     <!-- Add back in Mockito since the hadoop-hdfs test jar needs it. -->
     <dependency>
       <groupId>org.mockito</groupId>
-      <artifactId>mockito-core</artifactId>
+      <artifactId>mockito-inline</artifactId>
       <optional>true</optional>
     </dependency>
     <!-- Add back in the transitive dependencies excluded from hadoop-common in client TODO remove once we have a filter for "is in these artifacts" -->
@@ -761,7 +765,7 @@
 
                     <!-- Mockito tries to include its own unrelocated copy of hamcrest. :( -->
                     <filter>
-                      <artifact>org.mockito:mockito-core</artifact>
+                      <artifact>org.mockito:mockito-inline</artifact>
                       <excludes>
                         <exclude>asm-license.txt</exclude>
                         <exclude>cglib-license.txt</exclude>
@@ -773,6 +777,15 @@
                         <exclude>org/objenesis/*.class</exclude>
                       </excludes>
                     </filter>
+                    <!-- Additional filters to exclude unexpected contents -->
+                    <filter>
+                      <artifact>*:*</artifact>
+                      <excludes>
+                        <exclude>mockito-extensions/**</exclude>
+                        <exclude>win32-x86/**</exclude>
+                        <exclude>win32-x86-64/**</exclude>
+                      </excludes>
+                    </filter>
                     <!-- skip grizzly internals we don't need to run. -->
                     <filter>
                       <artifact>org.glassfish.grizzly:grizzly-http-servlet</artifact>

hadoop-client-modules/hadoop-client-runtime/pom.xml

@@ -229,6 +229,9 @@
                         <exclude>jnamed*</exclude>
                         <exclude>lookup*</exclude>
                         <exclude>update*</exclude>
+                        <exclude>META-INF/versions/18/*</exclude>
+                        <exclude>META-INF/versions/18/**/*</exclude>
+                        <exclude>META-INF/services/java.net.spi.InetAddressResolverProvider</exclude>
                       </excludes>
                     </filter>
                     <filter>
@@ -243,6 +246,7 @@
                       <excludes>
                         <exclude>META-INF/versions/9/module-info.class</exclude>
                         <exclude>META-INF/versions/11/module-info.class</exclude>
+                        <exclude>META-INF/versions/18/module-info.class</exclude>
                       </excludes>
                     </filter>
 

hadoop-cloud-storage-project/hadoop-cos/src/site/markdown/cloud-storage/index.md

@@ -86,7 +86,7 @@ Linux kernel 2.6+
 - joda-time (version 2.9.9 recommended)
 - httpClient (version 4.5.1 or later recommended)
 - Jackson: jackson-core, jackson-databind, jackson-annotations (version 2.9.8 or later)
-- bcprov-jdk18on (version 1.77 recommended)
+- bcprov-jdk18on (version 1.78.1 recommended)
 
 
 #### Configure Properties

hadoop-cloud-storage-project/hadoop-huaweicloud/pom.xml

@@ -173,15 +173,27 @@
     </dependency>
     <dependency>
       <groupId>org.powermock</groupId>
-      <artifactId>powermock-api-mockito</artifactId>
-      <version>1.7.4</version>
+      <artifactId>powermock-api-mockito2</artifactId>
+      <version>2.0.9</version>
       <scope>test</scope>
+      <exclusions>
+        <exclusion>
+          <groupId>org.mockito</groupId>
+          <artifactId>mockito-core</artifactId>
+        </exclusion>
+      </exclusions>
     </dependency>
     <dependency>
       <groupId>org.powermock</groupId>
       <artifactId>powermock-module-junit4</artifactId>
-      <version>1.7.4</version>
+      <version>2.0.9</version>
       <scope>test</scope>
+      <exclusions>
+        <exclusion>
+          <groupId>org.mockito</groupId>
+          <artifactId>mockito-core</artifactId>
+        </exclusion>
+      </exclusions>
     </dependency>
   </dependencies>
 </project>

hadoop-common-project/hadoop-auth/pom.xml

@@ -49,6 +49,7 @@
     <dependency>
       <groupId>org.mockito</groupId>
       <artifactId>mockito-core</artifactId>
+      <version>4.11.0</version>
       <scope>test</scope>
     </dependency>
     <dependency>
@@ -136,7 +137,11 @@
     </dependency>
     <dependency>
       <groupId>org.apache.kerby</groupId>
-      <artifactId>kerb-simplekdc</artifactId>
+      <artifactId>kerb-core</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.kerby</groupId>
+      <artifactId>kerb-util</artifactId>
     </dependency>
     <dependency>
       <groupId>org.apache.directory.server</groupId>
@@ -178,6 +183,10 @@
           <groupId>log4j</groupId>
           <artifactId>log4j</artifactId>
         </exclusion>
+        <exclusion>
+          <groupId>commons-collections</groupId>
+          <artifactId>commons-collections</artifactId>
+        </exclusion>
       </exclusions>
     </dependency>
     <dependency>

hadoop-common-project/hadoop-common/dev-support/findbugsExcludeFile.xml

@@ -454,4 +454,10 @@
     <Class name="org.apache.hadoop.ipc.internal.ShadedProtobufHelper" />
     <Bug pattern="AT_OPERATION_SEQUENCE_ON_CONCURRENT_ABSTRACTION" />
   </Match>
+
+  <!-- class cast after an assignableFrom check. -->
+  <Match>
+    <Class name="org.apache.hadoop.util.dynamic.DynMethods" />
+    <Bug pattern="BC_UNCONFIRMED_CAST" />
+  </Match>
 </FindBugsFilter>