From 383bbf11905acbfd0faec9149e832ad2bb3729c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E5=BF=97=E8=B1=AA?= Date: Wed, 18 Oct 2023 19:09:46 +0800 Subject: [PATCH 1/2] Bump delta to 3.0.0 --- spark/build.gradle | 4 ++-- .../sharpetl/spark/end2end/delta/DeltaLakeSpec.scala | 4 ++-- .../sharpdata/sharpetl/spark/end2end/delta/FlyDeltaSpec.scala | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/spark/build.gradle b/spark/build.gradle index 3fef2b9..0e2d52d 100644 --- a/spark/build.gradle +++ b/spark/build.gradle @@ -79,7 +79,7 @@ dependencies { implementation "org.apache.spark:spark-streaming-kafka-0-10_$scalaVersion:$sparkVersion" implementation "org.apache.kafka:kafka-clients:2.0.0" if (sparkVersion.startsWith("3.5")) { - implementation "io.delta:delta-core_$scalaVersion:2.4.0" + implementation "io.delta:delta-core_$scalaVersion:3.0.0" } else if (sparkVersion.startsWith("3.4")) { implementation "io.delta:delta-core_$scalaVersion:2.4.0" } else if (sparkVersion.startsWith("3.3")) { @@ -377,7 +377,7 @@ shadowJar { include dependency("net.liftweb:lift-json_$scalaVersion:3.4.3") if (sparkVersion.startsWith("3.5")) { - addDeps("io.delta:delta-core_$scalaVersion:2.4.0") + addDeps("io.delta:delta-core_$scalaVersion:3.0.0") } else if (sparkVersion.startsWith("3.4")) { addDeps("io.delta:delta-core_$scalaVersion:2.4.0") } else if (sparkVersion.startsWith("3.3")) { diff --git a/spark/src/test/scala/com/github/sharpdata/sharpetl/spark/end2end/delta/DeltaLakeSpec.scala b/spark/src/test/scala/com/github/sharpdata/sharpetl/spark/end2end/delta/DeltaLakeSpec.scala index 58e6b12..32ec537 100644 --- a/spark/src/test/scala/com/github/sharpdata/sharpetl/spark/end2end/delta/DeltaLakeSpec.scala +++ b/spark/src/test/scala/com/github/sharpdata/sharpetl/spark/end2end/delta/DeltaLakeSpec.scala @@ -38,8 +38,8 @@ class DeltaLakeSpec extends DeltaSuit { ) it("delta should works") { - if (spark.version.startsWith("2.3") || spark.version.startsWith("3.5")) { - ETLLogger.error("Delta Lake does NOT support Spark 2.3.x and Spark 3.5.x") + if (spark.version.startsWith("2.3")) { + ETLLogger.error("Delta Lake does NOT support Spark 2.3.x") } else if (spark.version.startsWith("2.4") || spark.version.startsWith("3.0") || spark.version.startsWith("3.1")) { ETLLogger.error("Delta Lake does not works well on Spark 2.4.x, " + "CREATE TABLE USING delta is not supported by Spark before 3.0.0 and Delta Lake before 0.7.0.") diff --git a/spark/src/test/scala/com/github/sharpdata/sharpetl/spark/end2end/delta/FlyDeltaSpec.scala b/spark/src/test/scala/com/github/sharpdata/sharpetl/spark/end2end/delta/FlyDeltaSpec.scala index fc34aa8..fb3ef5d 100644 --- a/spark/src/test/scala/com/github/sharpdata/sharpetl/spark/end2end/delta/FlyDeltaSpec.scala +++ b/spark/src/test/scala/com/github/sharpdata/sharpetl/spark/end2end/delta/FlyDeltaSpec.scala @@ -15,8 +15,8 @@ class FlyDeltaSpec extends AnyFunSpec with BeforeAndAfterEach { it("should just run with delta") { - if (spark.version.startsWith("2.3") || spark.version.startsWith("3.5")) { - ETLLogger.error("Delta Lake does NOT support Spark 2.3.x and Spark 3.5.x") + if (spark.version.startsWith("2.3")) { + ETLLogger.error("Delta Lake does NOT support Spark 2.3.x") } else if (spark.version.startsWith("2.4") || spark.version.startsWith("3.0") || spark.version.startsWith("3.1")) { ETLLogger.error("Delta Lake does not works well on Spark 2.4.x, " + "CREATE TABLE USING delta is not supported by Spark before 3.0.0 and Delta Lake before 0.7.0.") From 33bc2914545c7eb7a9cbc46ed4a6784f6d892be8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E5=BF=97=E8=B1=AA?= Date: Wed, 18 Oct 2023 19:21:11 +0800 Subject: [PATCH 2/2] renamed to delta-spark --- spark/build.gradle | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spark/build.gradle b/spark/build.gradle index 0e2d52d..2435f53 100644 --- a/spark/build.gradle +++ b/spark/build.gradle @@ -79,7 +79,7 @@ dependencies { implementation "org.apache.spark:spark-streaming-kafka-0-10_$scalaVersion:$sparkVersion" implementation "org.apache.kafka:kafka-clients:2.0.0" if (sparkVersion.startsWith("3.5")) { - implementation "io.delta:delta-core_$scalaVersion:3.0.0" + implementation "io.delta:delta-spark_$scalaVersion:3.0.0" } else if (sparkVersion.startsWith("3.4")) { implementation "io.delta:delta-core_$scalaVersion:2.4.0" } else if (sparkVersion.startsWith("3.3")) { @@ -377,7 +377,7 @@ shadowJar { include dependency("net.liftweb:lift-json_$scalaVersion:3.4.3") if (sparkVersion.startsWith("3.5")) { - addDeps("io.delta:delta-core_$scalaVersion:3.0.0") + addDeps("io.delta:delta-spark_$scalaVersion:3.0.0") } else if (sparkVersion.startsWith("3.4")) { addDeps("io.delta:delta-core_$scalaVersion:2.4.0") } else if (sparkVersion.startsWith("3.3")) {