Skip to content

Commit

Permalink
[Feat]support different hadoop version compile (apache#4110)
Browse files Browse the repository at this point in the history 
* support different hadoop version compile
  • Loading branch information
@GuoPhilipse
GuoPhilipse authored Feb 2, 2023
1 parent dedddf1 commit d75bd96
Show file tree
Hide file tree
Showing 9 changed files with 450 additions and 21 deletions.
2 changes: 1 addition & 1 deletion .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ dist/
out/
target/
.flattened-pom.xml

**/dependency-reduced-pom.xml
# log folder
*.log
logs/
Expand Down
3 changes: 2 additions & 1 deletion linkis-commons/linkis-hadoop-common/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,9 +53,10 @@
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
</dependency>

<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
<artifactId>${hadoop-hdfs-client.artifact}</artifactId>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
Expand Down
4 changes: 2 additions & 2 deletions linkis-engineconn-plugins/flink/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -162,7 +162,7 @@
</exclusion>
<exclusion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
<artifactId>${hadoop-hdfs-client.artifact}</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.hadoop</groupId>
Expand Down Expand Up @@ -362,7 +362,7 @@

<exclusion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
<artifactId>${hadoop-hdfs-client.artifact}</artifactId>
</exclusion>

<exclusion>
Expand Down
2 changes: 1 addition & 1 deletion linkis-engineconn-plugins/hive/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -284,7 +284,7 @@
</exclusion>
<exclusion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
<artifactId>${hadoop-hdfs-client.artifact}</artifactId>
</exclusion>
</exclusions>
</dependency>
Expand Down
1 change: 0 additions & 1 deletion linkis-engineconn-plugins/shell/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -110,7 +110,6 @@
<artifactId>netty-all</artifactId>
<scope>provided</scope>
</dependency>

</dependencies>

<build>
Expand Down
152 changes: 139 additions & 13 deletions linkis-engineconn-plugins/spark/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -104,20 +104,9 @@
<groupId>io.netty</groupId>
<artifactId>netty-all</artifactId>
</exclusion>

<exclusion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
</exclusion>

<exclusion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
</exclusion>

<exclusion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-auth</artifactId>
<artifactId>*</artifactId>
</exclusion>

<exclusion>
Expand Down Expand Up @@ -202,13 +191,20 @@
<artifactId>linkis-rpc</artifactId>
<version>${project.version}</version>
</dependency>

<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_${scala.binary.version}</artifactId>
<version>${spark.version}</version>
<scope>provided</scope>
<exclusions>
<exclusion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.hadoop</groupId>
<artifactId>${hadoop-hdfs-client.artifact}</artifactId>
</exclusion>
<exclusion>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-annotations</artifactId>
Expand Down Expand Up @@ -266,6 +262,14 @@
<version>${spark.version}</version>
<scope>provided</scope>
<exclusions>
<exclusion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.hadoop</groupId>
<artifactId>${hadoop-hdfs-client.artifact}</artifactId>
</exclusion>
<exclusion>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-annotations</artifactId>
Expand Down Expand Up @@ -316,6 +320,14 @@
<version>${spark.version}</version>
<scope>provided</scope>
<exclusions>
<exclusion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.hadoop</groupId>
<artifactId>${hadoop-hdfs-client.artifact}</artifactId>
</exclusion>
<exclusion>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-annotations</artifactId>
Expand Down Expand Up @@ -365,6 +377,14 @@
<version>${spark.version}</version>
<scope>provided</scope>
<exclusions>
<exclusion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.hadoop</groupId>
<artifactId>${hadoop-hdfs-client.artifact}</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
Expand Down Expand Up @@ -418,6 +438,18 @@
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>${hadoop.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
<version>${hadoop.version}</version>
<scope>provided</scope>
</dependency>
</dependencies>

<build>
Expand Down Expand Up @@ -457,4 +489,98 @@
</plugin>
</plugins>
</build>
<profiles>
<!-- spark2-hadoop3 version:spark2.4 use hadoop2.7.2 by default mvn validate -Pspark-2.4-hadoop-3.3 -->
<profile>
<id>spark-2.4-hadoop-3.3</id>
<properties>
<hadoop.version>${hadoop-hdfs-client-shade.version}</hadoop.version>
</properties>
<dependencies>
<dependency>
<groupId>org.apache.linkis</groupId>
<artifactId>linkis-hadoop-hdfs-client-shade</artifactId>
<version>${project.version}</version>
<exclusions>
<exclusion>
<groupId>commmons-logging</groupId>
<artifactId>commons-logging</artifactId>
</exclusion>
<exclusion>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
</exclusion>
<exclusion>
<groupId>org.mortbay.jetty</groupId>
<artifactId>jetty</artifactId>
</exclusion>
<exclusion>
<groupId>org.mortbay.jetty</groupId>
<artifactId>jetty-util</artifactId>
</exclusion>
<exclusion>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-core</artifactId>
</exclusion>
<exclusion>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-server</artifactId>
</exclusion>
<exclusion>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-json</artifactId>
</exclusion>
<exclusion>
<groupId>javax.ws.rs</groupId>
<artifactId>jsr311-api</artifactId>
</exclusion>
<exclusion>
<groupId>net.java.dev.jets3t</groupId>
<artifactId>jets3t</artifactId>
</exclusion>
<exclusion>
<groupId>com.jcraft</groupId>
<artifactId>jsch</artifactId>
</exclusion>
<exclusion>
<groupId>com.google.code.findbugs</groupId>
<artifactId>jsr305</artifactId>
</exclusion>
<exclusion>
<groupId>xmlenc</groupId>
<artifactId>xmlenc</artifactId>
</exclusion>
<exclusion>
<groupId>net.java.dev.jets3t</groupId>
<artifactId>jets3t</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.avro</groupId>
<artifactId>avro</artifactId>
</exclusion>
<exclusion>
<groupId>com.jcraft</groupId>
<artifactId>jsch</artifactId>
</exclusion>
<exclusion>
<groupId>com.google.code.findbugs</groupId>
<artifactId>jsr305</artifactId>
</exclusion>
<exclusion>
<groupId>javax.servlet</groupId>
<artifactId>servlet-api</artifactId>
</exclusion>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
<exclusion>
<groupId>org.eclipse.jetty</groupId>
<artifactId>*</artifactId>
</exclusion>
</exclusions>
</dependency>
</dependencies>
</profile>
</profiles>
</project>
2 changes: 1 addition & 1 deletion linkis-engineconn-plugins/sqoop/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -172,7 +172,7 @@
</exclusion>
<exclusion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
<artifactId>${hadoop-hdfs-client.artifact}</artifactId>
</exclusion>
</exclusions>
</dependency>
Expand Down
Loading

0 comments on commit d75bd96

Please sign in to comment.