HADOOP-19107. Drop support for HBase v1 & upgrade HBase v2 (#6629). C…

…ontributed by Ayush Saxena
apache · Apr 22, 2024 · eec9cd2 · eec9cd2
1 parent a386ac1
commit eec9cd2
Show file tree

Hide file tree

Showing 24 changed files with 82 additions and 1,947 deletions.
diff --git a/BUILDING.txt b/BUILDING.txt
@@ -163,14 +163,7 @@ Maven build goals:
  YARN Application Timeline Service V2 build options:
 
    YARN Timeline Service v.2 chooses Apache HBase as the primary backing storage. The supported
-   versions of Apache HBase are 1.7.1 (default) and 2.2.4.
-
-  * HBase 1.7.1 is used by default to build Hadoop. The official releases are ready to use if you
-    plan on running Timeline Service v2 with HBase 1.7.1.
-
-  * Use -Dhbase.profile=2.0 to build Hadoop with HBase 2.2.4. Provide this option if you plan
-    on running Timeline Service v2 with HBase 2.x.
-
+   version of Apache HBase is 2.5.8.
 
  Snappy build options:
 

diff --git a/LICENSE-binary b/LICENSE-binary
@@ -310,10 +310,10 @@ org.apache.commons:commons-validator:1.6
 org.apache.curator:curator-client:5.2.0
 org.apache.curator:curator-framework:5.2.0
 org.apache.curator:curator-recipes:5.2.0
-org.apache.hbase:hbase-annotations:1.7.1
-org.apache.hbase:hbase-client:1.7.1
-org.apache.hbase:hbase-common:1.7.1
-org.apache.hbase:hbase-protocol:1.7.1
+org.apache.hbase:hbase-annotations:2.5.8
+org.apache.hbase:hbase-client:2.5.8
+org.apache.hbase:hbase-common:2.5.8
+org.apache.hbase:hbase-protocol:2.5.8
 org.apache.htrace:htrace-core:3.1.0-incubating
 org.apache.htrace:htrace-core4:4.1.0-incubating
 org.apache.httpcomponents:httpclient:4.5.13

diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml
@@ -245,7 +245,7 @@
     </moduleSet>
     <moduleSet>
       <includes>
-        <include>org.apache.hadoop:${hbase-server-artifactid}</include>
+        <include>org.apache.hadoop:hadoop-yarn-server-timelineservice-hbase-server-2</include>
       </includes>
       <binaries>
         <outputDirectory>share/hadoop/${hadoop.component}/timelineservice</outputDirectory>

diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
@@ -203,8 +203,7 @@
     <swagger-annotations-version>1.5.4</swagger-annotations-version>
     <snakeyaml.version>2.0</snakeyaml.version>
     <sshd.version>2.11.0</sshd.version>
-    <hbase.one.version>1.7.1</hbase.one.version>
-    <hbase.two.version>2.2.4</hbase.two.version>
+    <hbase.version>2.5.8-hadoop3</hbase.version>
     <junit.version>4.13.2</junit.version>
     <junit.jupiter.version>5.8.2</junit.jupiter.version>
     <junit.vintage.version>5.8.2</junit.vintage.version>
@@ -501,6 +500,11 @@
         <version>${hadoop.version}</version>
         <type>test-jar</type>
       </dependency>
+      <dependency>
+        <groupId>org.apache.hadoop</groupId>
+        <artifactId>hadoop-yarn-server-timelineservice-hbase-server-2</artifactId>
+        <version>${hadoop.version}</version>
+      </dependency>
 
       <dependency>
         <groupId>org.apache.hadoop</groupId>
@@ -1811,6 +1815,10 @@
             <groupId>org.apache.yetus</groupId>
             <artifactId>audience-annotations</artifactId>
           </exclusion>
+          <exclusion>
+            <groupId>org.osgi</groupId>
+            <artifactId>org.osgi.core</artifactId>
+          </exclusion>
         </exclusions>
       </dependency>
       <dependency>
@@ -1875,6 +1883,14 @@
             <groupId>org.apache.yetus</groupId>
             <artifactId>audience-annotations</artifactId>
           </exclusion>
+          <exclusion>
+            <groupId>com.google.errorprone</groupId>
+            <artifactId>error_prone_annotations</artifactId>
+          </exclusion>
+          <exclusion>
+            <groupId>org.checkerframework</groupId>
+            <artifactId>checker-qual</artifactId>
+          </exclusion>
         </exclusions>
       </dependency>
       <dependency>
@@ -2677,33 +2693,6 @@
         </plugins>
       </build>
     </profile>
-    <!-- The profile for building against HBase 1.2.x
-     This is the default.
-     -->
-    <profile>
-      <id>hbase1</id>
-      <activation>
-        <property>
-          <name>!hbase.profile</name>
-        </property>
-      </activation>
-      <properties>
-        <hbase.version>${hbase.one.version}</hbase.version>
-        <hbase-compatible-hadoop.version>2.8.5</hbase-compatible-hadoop.version>
-        <hbase-compatible-guava.version>12.0.1</hbase-compatible-guava.version>
-        <hbase-compatible-guice.version>4.0</hbase-compatible-guice.version>
-        <hbase-server-artifactid>hadoop-yarn-server-timelineservice-hbase-server-1</hbase-server-artifactid>
-      </properties>
-      <dependencyManagement>
-        <dependencies>
-          <dependency>
-            <groupId>org.apache.hadoop</groupId>
-            <artifactId>${hbase-server-artifactid}</artifactId>
-            <version>${hadoop.version}</version>
-          </dependency>
-        </dependencies>
-      </dependencyManagement>
-    </profile>
     <!-- The profile for building against HBase 2.0.0.
      Activate using: mvn -Dhbase.profile=2.0
     -->
@@ -2716,20 +2705,10 @@
         </property>
       </activation>
       <properties>
-        <hbase.version>${hbase.two.version}</hbase.version>
-        <hbase-compatible-hadoop.version>2.8.5</hbase-compatible-hadoop.version>
-        <hbase-compatible-guava.version>11.0.2</hbase-compatible-guava.version>
-        <hbase-server-artifactid>hadoop-yarn-server-timelineservice-hbase-server-2</hbase-server-artifactid>
-        <hbase-compatible-guice.version>4.0</hbase-compatible-guice.version>
-        <hbase-compatible-jetty.version>9.3.27.v20190418</hbase-compatible-jetty.version>
       </properties>
       <dependencyManagement>
         <dependencies>
-          <dependency>
-            <groupId>org.apache.hadoop</groupId>
-            <artifactId>${hbase-server-artifactid}</artifactId>
-            <version>${hadoop.version}</version>
-          </dependency>
+
         </dependencies>
       </dependencyManagement>
     </profile>

diff --git a/...ect/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/pom.xml b/...ect/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/pom.xml
@@ -104,10 +104,6 @@
           <groupId>com.sun.jersey</groupId>
           <artifactId>jersey-json</artifactId>
         </exclusion>
-        <exclusion>
-          <groupId>commons-logging</groupId>
-          <artifactId>commons-logging</artifactId>
-        </exclusion>
       </exclusions>
     </dependency>
 
@@ -182,19 +178,6 @@
       <scope>test</scope>
     </dependency>
 
-    <dependency>
-      <groupId>com.google.guava</groupId>
-      <artifactId>guava</artifactId>
-      <version>${hbase-compatible-guava.version}</version>
-      <scope>test</scope>
-    </dependency>
-
-    <dependency>
-      <groupId>com.google.inject</groupId>
-      <artifactId>guice</artifactId>
-      <version>${hbase-compatible-guice.version}</version>
-      <scope>test</scope>
-    </dependency>
 
     <dependency>
       <groupId>org.xerial.snappy</groupId>
@@ -393,13 +376,6 @@
       <scope>test</scope>
     </dependency>
 
-    <dependency>
-      <groupId>org.jmockit</groupId>
-      <artifactId>jmockit</artifactId>
-      <version>1.24</version>
-      <scope>test</scope>
-    </dependency>
-
     <dependency>
       <groupId>org.assertj</groupId>
       <artifactId>assertj-core</artifactId>
@@ -454,6 +430,25 @@
       <artifactId>commons-lang3</artifactId>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-yarn-server-timelineservice-hbase-server-2</artifactId>
+      <scope>test</scope>
+      <exclusions>
+        <exclusion>
+          <groupId>org.apache.hadoop</groupId>
+          <artifactId>hadoop-common</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+    <!-- 'mvn dependency:analyze' fails to detect use of this direct
+         dependency -->
+    <!-- This is needed by HBaseTestingUtility -->
+    <dependency>
+      <groupId>org.mockito</groupId>
+      <artifactId>mockito-core</artifactId>
+      <scope>test</scope>
+    </dependency>
   </dependencies>
 
   <build>
@@ -494,86 +489,14 @@
           </execution>
         </executions>
       </plugin>
+      <!-- The fork value is deliberately set to 0 to avoid VM crash while running tests
+       on Jenkins, removing this leads to tests crashing silently due to VM crash -->
+      <plugin>
+        <artifactId>maven-surefire-plugin</artifactId>
+        <configuration>
+          <forkCount>0</forkCount>
+        </configuration>
+      </plugin>
     </plugins>
   </build>
-
-  <profiles>
-    <profile>
-      <id>hbase1</id>
-      <activation>
-        <property>
-          <name>!hbase.profile</name>
-        </property>
-      </activation>
-      <dependencies>
-        <dependency>
-          <groupId>org.apache.hadoop</groupId>
-          <artifactId>hadoop-yarn-server-timelineservice-hbase-server-1</artifactId>
-          <scope>test</scope>
-          <exclusions>
-            <exclusion>
-              <groupId>org.apache.hadoop</groupId>
-              <artifactId>hadoop-common</artifactId>
-            </exclusion>
-          </exclusions>
-        </dependency>
-      </dependencies>
-    </profile>
-
-    <profile>
-      <id>hbase2</id>
-      <activation>
-        <property>
-          <name>hbase.profile</name>
-          <value>2.0</value>
-        </property>
-      </activation>
-      <dependencies>
-        <dependency>
-          <groupId>org.apache.hadoop</groupId>
-          <artifactId>hadoop-yarn-server-timelineservice-hbase-server-2</artifactId>
-          <scope>test</scope>
-          <exclusions>
-            <exclusion>
-              <groupId>org.apache.hadoop</groupId>
-              <artifactId>hadoop-common</artifactId>
-            </exclusion>
-          </exclusions>
-        </dependency>
-        <!-- 'mvn dependency:analyze' fails to detect use of this direct
-             dependency -->
-        <!-- This is needed by HBaseTestingUtility -->
-        <dependency>
-          <groupId>org.mockito</groupId>
-          <artifactId>mockito-core</artifactId>
-          <scope>test</scope>
-        </dependency>
-        <dependency>
-          <groupId>org.eclipse.jetty</groupId>
-          <artifactId>jetty-server</artifactId>
-          <scope>test</scope>
-          <version>${hbase-compatible-jetty.version}</version>
-        </dependency>
-        <dependency>
-          <groupId>org.eclipse.jetty</groupId>
-          <artifactId>jetty-servlet</artifactId>
-          <scope>test</scope>
-          <version>${hbase-compatible-jetty.version}</version>
-        </dependency>
-        <dependency>
-          <groupId>org.eclipse.jetty</groupId>
-          <artifactId>jetty-webapp</artifactId>
-          <scope>test</scope>
-          <version>${hbase-compatible-jetty.version}</version>
-        </dependency>
-        <dependency>
-          <groupId>org.eclipse.jetty</groupId>
-          <artifactId>jetty-util</artifactId>
-          <scope>test</scope>
-          <version>${hbase-compatible-jetty.version}</version>
-        </dependency>
-      </dependencies>
-    </profile>
-
-  </profiles>
 </project>
diff --git a/...java/org/apache/hadoop/yarn/server/timelineservice/reader/DummyTimelineReaderMetrics.java b/...java/org/apache/hadoop/yarn/server/timelineservice/reader/DummyTimelineReaderMetrics.java
diff --git a/.../hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesHBaseStorage.java b/.../hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesHBaseStorage.java
@@ -37,8 +37,6 @@
 
 import javax.ws.rs.core.MediaType;
 
-import mockit.Mock;
-import mockit.MockUp;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
@@ -54,7 +52,6 @@
 import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric.Type;
 import org.apache.hadoop.yarn.server.metrics.ApplicationMetricsConstants;
 import org.apache.hadoop.yarn.server.timelineservice.collector.TimelineCollectorContext;
-import org.apache.hadoop.yarn.server.timelineservice.metrics.TimelineReaderMetrics;
 import org.apache.hadoop.yarn.server.timelineservice.storage.HBaseTimelineWriterImpl;
 import org.apache.hadoop.yarn.server.timelineservice.storage.common.HBaseTimelineSchemaUtils;
 import org.apache.hadoop.yarn.server.utils.BuilderUtils;
@@ -77,17 +74,9 @@ public class TestTimelineReaderWebServicesHBaseStorage
   private static long dayTs =
       HBaseTimelineSchemaUtils.getTopOfTheDayTimestamp(ts);
   private static String doAsUser = "remoteuser";
-  private static final DummyTimelineReaderMetrics METRICS
-      = new DummyTimelineReaderMetrics();
 
   @BeforeClass
   public static void setupBeforeClass() throws Exception {
-    new MockUp<TimelineReaderMetrics>() {
-      @Mock
-      public TimelineReaderMetrics getInstance() {
-        return METRICS;
-      }
-    };
     setup();
     loadData();
     initialize();