From b71e9b2f1bb1d5eb38b7fddd482e9b55bf8929d3 Mon Sep 17 00:00:00 2001 From: NEUpanning Date: Mon, 18 Nov 2024 19:32:16 +0800 Subject: [PATCH 1/3] add ut --- .../spark/sql/GlutenDateFunctionsSuite.scala | 41 +++++++++++++------ 1 file changed, 29 insertions(+), 12 deletions(-) diff --git a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenDateFunctionsSuite.scala b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenDateFunctionsSuite.scala index a946e6de4345..c3513e9065c7 100644 --- a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenDateFunctionsSuite.scala +++ b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenDateFunctionsSuite.scala @@ -16,6 +16,7 @@ */ package org.apache.spark.sql +import org.apache.spark.SparkUpgradeException import org.apache.spark.sql.catalyst.util.DateTimeUtils import org.apache.spark.sql.functions._ import org.apache.spark.sql.internal.SQLConf @@ -112,18 +113,16 @@ class GlutenDateFunctionsSuite extends DateFunctionsSuite with GlutenSQLTestsTra df1.selectExpr(s"unix_timestamp(x, 'yyyy-MM-dd mm:HH:ss')"), Seq(Row(secs(ts4.getTime)), Row(null), Row(secs(ts3.getTime)), Row(null))) - // legacyParserPolicy is not respected by Gluten. - // invalid format - // val invalid = df1.selectExpr(s"unix_timestamp(x, 'yyyy-MM-dd aa:HH:ss')") - // if (legacyParserPolicy == "legacy") { - // checkAnswer(invalid, - // Seq(Row(null), Row(null), Row(null), Row(null))) - // } else { - // val e = intercept[SparkUpgradeException](invalid.collect()) - // assert(e.getCause.isInstanceOf[IllegalArgumentException]) - // assert( e.getMessage.contains( - // "You may get a different result due to the upgrading to Spark")) - // } + val invalid = df1.selectExpr(s"unix_timestamp(x, 'yyyy-MM-dd aa:HH:ss')") + if (legacyParserPolicy == "legacy") { + checkAnswer(invalid, + Seq(Row(null), Row(null), Row(null), Row(null))) + } else { + val e = intercept[SparkUpgradeException](invalid.collect()) + assert(e.getCause.isInstanceOf[IllegalArgumentException]) + assert( e.getMessage.contains( + "You may get a different result due to the upgrading to Spark")) + } // February val y1 = "2016-02-29" @@ -140,6 +139,24 @@ class GlutenDateFunctionsSuite extends DateFunctionsSuite with GlutenSQLTestsTra Row(new java.util.Date(TimeUnit.SECONDS.toMillis(now)))) } } + + val df = Seq(("2016-04-08 00:00:00")).toDF("d") + val fmt = "yyyy-MM-dd" + withSQLConf( + SQLConf.LEGACY_TIME_PARSER_POLICY.key -> "legacy") { + val result = df.selectExpr(s"unix_timestamp(d, '$fmt')") + checkAnswer(result, Seq(Row(1460041200))) + } + + withSQLConf( + SQLConf.LEGACY_TIME_PARSER_POLICY.key -> "correct") { + val result = df.selectExpr(s"unix_timestamp(d, '$fmt')") + val e = intercept[SparkUpgradeException](result.collect()) + assert(e.getCause.isInstanceOf[IllegalArgumentException]) + assert( e.getMessage.contains( + "You may get a different result due to the upgrading to Spark")) + } + } testGluten("to_unix_timestamp") { From f479227e4a42a718626f6b5e40bda56d791133a5 Mon Sep 17 00:00:00 2001 From: NEUpanning Date: Tue, 19 Nov 2024 11:33:19 +0800 Subject: [PATCH 2/3] spotless --- .../spark/sql/GlutenDateFunctionsSuite.scala | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenDateFunctionsSuite.scala b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenDateFunctionsSuite.scala index c3513e9065c7..2124005b54b2 100644 --- a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenDateFunctionsSuite.scala +++ b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenDateFunctionsSuite.scala @@ -113,16 +113,16 @@ class GlutenDateFunctionsSuite extends DateFunctionsSuite with GlutenSQLTestsTra df1.selectExpr(s"unix_timestamp(x, 'yyyy-MM-dd mm:HH:ss')"), Seq(Row(secs(ts4.getTime)), Row(null), Row(secs(ts3.getTime)), Row(null))) - val invalid = df1.selectExpr(s"unix_timestamp(x, 'yyyy-MM-dd aa:HH:ss')") - if (legacyParserPolicy == "legacy") { - checkAnswer(invalid, - Seq(Row(null), Row(null), Row(null), Row(null))) - } else { - val e = intercept[SparkUpgradeException](invalid.collect()) - assert(e.getCause.isInstanceOf[IllegalArgumentException]) - assert( e.getMessage.contains( - "You may get a different result due to the upgrading to Spark")) - } + val invalid = df1.selectExpr(s"unix_timestamp(x, 'yyyy-MM-dd aa:HH:ss')") + if (legacyParserPolicy == "legacy") { + checkAnswer(invalid, + Seq(Row(null), Row(null), Row(null), Row(null))) + } else { + val e = intercept[SparkUpgradeException](invalid.collect()) + assert(e.getCause.isInstanceOf[IllegalArgumentException]) + assert(e.getMessage.contains( + "You may get a different result due to the upgrading to Spark")) + } // February val y1 = "2016-02-29" @@ -153,10 +153,10 @@ class GlutenDateFunctionsSuite extends DateFunctionsSuite with GlutenSQLTestsTra val result = df.selectExpr(s"unix_timestamp(d, '$fmt')") val e = intercept[SparkUpgradeException](result.collect()) assert(e.getCause.isInstanceOf[IllegalArgumentException]) - assert( e.getMessage.contains( + assert(e.getMessage.contains( "You may get a different result due to the upgrading to Spark")) } - + } testGluten("to_unix_timestamp") { From ff49b6665c484c0ef8a331cfe1812c64e5ebeb39 Mon Sep 17 00:00:00 2001 From: NEUpanning Date: Tue, 19 Nov 2024 19:28:11 +0800 Subject: [PATCH 3/3] spotless --- .../spark/sql/GlutenDateFunctionsSuite.scala | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenDateFunctionsSuite.scala b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenDateFunctionsSuite.scala index 2124005b54b2..2a9262aeaf9f 100644 --- a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenDateFunctionsSuite.scala +++ b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenDateFunctionsSuite.scala @@ -26,6 +26,7 @@ import java.time.{LocalDateTime, ZoneId} import java.util.concurrent.TimeUnit class GlutenDateFunctionsSuite extends DateFunctionsSuite with GlutenSQLTestsTrait { + import testImplicits._ private def secs(millis: Long): Long = TimeUnit.MILLISECONDS.toSeconds(millis) @@ -115,13 +116,12 @@ class GlutenDateFunctionsSuite extends DateFunctionsSuite with GlutenSQLTestsTra val invalid = df1.selectExpr(s"unix_timestamp(x, 'yyyy-MM-dd aa:HH:ss')") if (legacyParserPolicy == "legacy") { - checkAnswer(invalid, - Seq(Row(null), Row(null), Row(null), Row(null))) + checkAnswer(invalid, Seq(Row(null), Row(null), Row(null), Row(null))) } else { val e = intercept[SparkUpgradeException](invalid.collect()) assert(e.getCause.isInstanceOf[IllegalArgumentException]) - assert(e.getMessage.contains( - "You may get a different result due to the upgrading to Spark")) + assert( + e.getMessage.contains("You may get a different result due to the upgrading to Spark")) } // February @@ -142,19 +142,16 @@ class GlutenDateFunctionsSuite extends DateFunctionsSuite with GlutenSQLTestsTra val df = Seq(("2016-04-08 00:00:00")).toDF("d") val fmt = "yyyy-MM-dd" - withSQLConf( - SQLConf.LEGACY_TIME_PARSER_POLICY.key -> "legacy") { + withSQLConf(SQLConf.LEGACY_TIME_PARSER_POLICY.key -> "legacy") { val result = df.selectExpr(s"unix_timestamp(d, '$fmt')") checkAnswer(result, Seq(Row(1460041200))) } - withSQLConf( - SQLConf.LEGACY_TIME_PARSER_POLICY.key -> "correct") { + withSQLConf(SQLConf.LEGACY_TIME_PARSER_POLICY.key -> "correct") { val result = df.selectExpr(s"unix_timestamp(d, '$fmt')") val e = intercept[SparkUpgradeException](result.collect()) assert(e.getCause.isInstanceOf[IllegalArgumentException]) - assert(e.getMessage.contains( - "You may get a different result due to the upgrading to Spark")) + assert(e.getMessage.contains("You may get a different result due to the upgrading to Spark")) } }