Skip to content

Commit

Permalink
Closes #1426: Run IIS experiments by relying on spark 3.4 version
Browse files Browse the repository at this point in the history 
Upgrading dependencies in pom.xml files, aligning with scala 2.12 and spark 3.4.1.
  • Loading branch information
@marekhorst
marekhorst committed Oct 24, 2024
1 parent b5fb153 commit a1273c0
Show file tree
Hide file tree
Showing 11 changed files with 82 additions and 38 deletions.
8 changes: 4 additions & 4 deletions iis-common/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -79,22 +79,22 @@

<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_2.11</artifactId>
<artifactId>spark-core_2.12</artifactId>
</dependency>

<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-sql_2.11</artifactId>
<artifactId>spark-sql_2.12</artifactId>
</dependency>

<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-avro_2.11</artifactId>
<artifactId>spark-avro_2.12</artifactId>
</dependency>

<dependency>
<groupId>pl.edu.icm.spark-utils</groupId>
<artifactId>spark-utils_2.11</artifactId>
<artifactId>spark-utils_2.12</artifactId>
</dependency>

<dependency>
Expand Down
6 changes: 3 additions & 3 deletions iis-wf/iis-wf-affmatching/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,17 +35,17 @@

<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_2.11</artifactId>
<artifactId>spark-core_2.12</artifactId>
</dependency>

<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-sql_2.11</artifactId>
<artifactId>spark-sql_2.12</artifactId>
</dependency>

<dependency>
<groupId>pl.edu.icm.spark-utils</groupId>
<artifactId>spark-utils_2.11</artifactId>
<artifactId>spark-utils_2.12</artifactId>
</dependency>

<dependency>
Expand Down
6 changes: 3 additions & 3 deletions iis-wf/iis-wf-citationmatching-direct/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,17 +36,17 @@

<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_2.11</artifactId>
<artifactId>spark-core_2.12</artifactId>
</dependency>

<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-sql_2.11</artifactId>
<artifactId>spark-sql_2.12</artifactId>
</dependency>

<dependency>
<groupId>pl.edu.icm.spark-utils</groupId>
<artifactId>spark-utils_2.11</artifactId>
<artifactId>spark-utils_2.12</artifactId>
</dependency>
</dependencies>

Expand Down
44 changes: 44 additions & 0 deletions iis-wf/iis-wf-citationmatching/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,28 @@
<groupId>eu.dnetlib.iis</groupId>
<artifactId>iis-common</artifactId>
<version>${project.version}</version>
<exclusions>
<exclusion>
<!-- conflicting scala binary versions -->
<groupId>org.scala-lang</groupId>
<artifactId>scala-library</artifactId>
</exclusion>
<exclusion>
<!-- conflicting scala binary versions -->
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_2.12</artifactId>
</exclusion>
<exclusion>
<!-- conflicting scala binary versions -->
<groupId>org.apache.spark</groupId>
<artifactId>spark-sql_2.12</artifactId>
</exclusion>
<exclusion>
<!-- conflicting scala binary versions -->
<groupId>pl.edu.icm.spark-utils</groupId>
<artifactId>spark-utils_2.12</artifactId>
</exclusion>
</exclusions>
</dependency>

<dependency>
Expand All @@ -38,6 +60,28 @@
<version>${project.version}</version>
<type>test-jar</type>
<scope>test</scope>
<exclusions>
<exclusion>
<!-- conflicting scala binary versions -->
<groupId>org.scala-lang</groupId>
<artifactId>scala-library</artifactId>
</exclusion>
<exclusion>
<!-- conflicting scala binary versions -->
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_2.12</artifactId>
</exclusion>
<exclusion>
<!-- conflicting scala binary versions -->
<groupId>org.apache.spark</groupId>
<artifactId>spark-sql_2.12</artifactId>
</exclusion>
<exclusion>
<!-- conflicting scala binary versions -->
<groupId>pl.edu.icm.spark-utils</groupId>
<artifactId>spark-utils_2.12</artifactId>
</exclusion>
</exclusions>
</dependency>

<dependency>
Expand Down
6 changes: 3 additions & 3 deletions iis-wf/iis-wf-documentsclassification/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,17 +53,17 @@

<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_2.11</artifactId>
<artifactId>spark-core_2.12</artifactId>
</dependency>

<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-sql_2.11</artifactId>
<artifactId>spark-sql_2.12</artifactId>
</dependency>

<dependency>
<groupId>pl.edu.icm.spark-utils</groupId>
<artifactId>spark-utils_2.11</artifactId>
<artifactId>spark-utils_2.12</artifactId>
</dependency>

<dependency>
Expand Down
4 changes: 2 additions & 2 deletions iis-wf/iis-wf-documentssimilarity/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,13 +53,13 @@

<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_2.11</artifactId>
<artifactId>spark-core_2.12</artifactId>
<scope>test</scope>
</dependency>

<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-sql_2.11</artifactId>
<artifactId>spark-sql_2.12</artifactId>
<scope>test</scope>
</dependency>

Expand Down
8 changes: 4 additions & 4 deletions iis-wf/iis-wf-export-actionmanager/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,22 +76,22 @@

<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_2.11</artifactId>
<artifactId>spark-core_2.12</artifactId>
</dependency>

<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-sql_2.11</artifactId>
<artifactId>spark-sql_2.12</artifactId>
</dependency>

<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-avro_2.11</artifactId>
<artifactId>spark-avro_2.12</artifactId>
</dependency>

<dependency>
<groupId>pl.edu.icm.spark-utils</groupId>
<artifactId>spark-utils_2.11</artifactId>
<artifactId>spark-utils_2.12</artifactId>
</dependency>
</dependencies>

Expand Down
10 changes: 5 additions & 5 deletions iis-wf/iis-wf-import/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -115,27 +115,27 @@

<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_2.11</artifactId>
<artifactId>spark-core_2.12</artifactId>
</dependency>

<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-sql_2.11</artifactId>
<artifactId>spark-sql_2.12</artifactId>
</dependency>

<dependency>
<groupId>pl.edu.icm.spark-utils</groupId>
<artifactId>spark-utils_2.11</artifactId>
<artifactId>spark-utils_2.12</artifactId>
</dependency>

<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-avro_2.11</artifactId>
<artifactId>spark-avro_2.12</artifactId>
</dependency>

<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-hive_2.11</artifactId>
<artifactId>spark-hive_2.12</artifactId>
<exclusions>
<exclusion>
<!-- unavailable dependency in cloudera repo -->
Expand Down
8 changes: 4 additions & 4 deletions iis-wf/iis-wf-referenceextraction/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,22 +66,22 @@

<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_2.11</artifactId>
<artifactId>spark-core_2.12</artifactId>
</dependency>

<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-sql_2.11</artifactId>
<artifactId>spark-sql_2.12</artifactId>
</dependency>

<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-avro_2.11</artifactId>
<artifactId>spark-avro_2.12</artifactId>
</dependency>

<dependency>
<groupId>pl.edu.icm.spark-utils</groupId>
<artifactId>spark-utils_2.11</artifactId>
<artifactId>spark-utils_2.12</artifactId>
</dependency>

<dependency>
Expand Down
6 changes: 3 additions & 3 deletions iis-wf/iis-wf-transformers/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,17 +51,17 @@

<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_2.11</artifactId>
<artifactId>spark-core_2.12</artifactId>
</dependency>

<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-sql_2.11</artifactId>
<artifactId>spark-sql_2.12</artifactId>
</dependency>

<dependency>
<groupId>pl.edu.icm.spark-utils</groupId>
<artifactId>spark-utils_2.11</artifactId>
<artifactId>spark-utils_2.12</artifactId>
</dependency>
</dependencies>

Expand Down
14 changes: 7 additions & 7 deletions pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -321,36 +321,36 @@

<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_2.11</artifactId>
<artifactId>spark-core_2.12</artifactId>
<version>${iis.spark.version}</version>
<scope>provided</scope>
</dependency>

<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-sql_2.11</artifactId>
<artifactId>spark-sql_2.12</artifactId>
<version>${iis.spark.version}</version>
<scope>provided</scope>
</dependency>

<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-avro_2.11</artifactId>
<artifactId>spark-avro_2.12</artifactId>
<version>${iis.spark.version}</version>
<scope>provided</scope>
</dependency>

<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-hive_2.11</artifactId>
<artifactId>spark-hive_2.12</artifactId>
<version>${iis.spark.version}</version>
<scope>provided</scope>
</dependency>

<dependency>
<groupId>pl.edu.icm.spark-utils</groupId>
<artifactId>spark-utils_2.11</artifactId>
<version>1.0.1</version>
<artifactId>spark-utils_2.12</artifactId>
<version>1.0.2</version>
<scope>provided</scope>
</dependency>

Expand Down Expand Up @@ -1017,7 +1017,7 @@
<iis.pig.version>0.12.0-${iis.cdh.version}</iis.pig.version>
<iis.avro.version>1.7.6-${iis.cdh.version}</iis.avro.version>
<iis.hadoop.version>2.6.0-${iis.cdh.version}</iis.hadoop.version>
<iis.spark.version>2.4.0.cloudera2</iis.spark.version>
<iis.spark.version>3.4.1</iis.spark.version>
<iis.coansys.version>1.10</iis.coansys.version>

<junit.version>5.8.2</junit.version>
Expand Down

0 comments on commit a1273c0

Please sign in to comment.