From 2f4e3d661e0957c0677be37f122cbd8fefde93c1 Mon Sep 17 00:00:00 2001 From: v-kkhuang <62878639+v-kkhuang@users.noreply.github.com> Date: Fri, 13 Dec 2024 16:30:49 +0800 Subject: [PATCH] push decimal fix code (#676) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: “v_kkhuang” <“420895376@qq.com”> --- .../scala/org/apache/linkis/storage/domain/DataType.scala | 6 ++++-- .../apache/linkis/storage/excel/StorageExcelWriter.scala | 8 ++++---- .../apache/linkis/ujes/client/utils/UJESClientUtils.scala | 4 +++- .../manager/engineplugin/jdbc/executor/JDBCHelper.java | 2 +- .../sql/execution/datasources/csv/DolphinToSpark.scala | 2 +- 5 files changed, 13 insertions(+), 9 deletions(-) diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/domain/DataType.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/domain/DataType.scala index dfd67efac8..a603e08775 100644 --- a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/domain/DataType.scala +++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/domain/DataType.scala @@ -89,7 +89,7 @@ object DataType extends Logging { case LongType | BigIntType => if (isNumberNull(newValue)) null else newValue.toLong case FloatType => if (isNumberNull(newValue)) null else newValue.toFloat case DoubleType => if (isNumberNull(newValue)) null else newValue.toDouble - case DecimalType(dataType, 3) => + case DecimalType(_, _) => if (isNumberNull(newValue)) null else new JavaBigDecimal(newValue) case DateType => if (isNumberNull(newValue)) null else Date.valueOf(newValue) case TimestampType => @@ -150,11 +150,13 @@ case object ArrayType extends DataType("array", 2003) case object MapType extends DataType("map", 2000) case object ListType extends DataType("list", 2001) case object StructType extends DataType("struct", 2002) -case object BigDecimalType extends DataType("bigdecimal", 3) case class DecimalType(override val typeName: String, override val javaSQLType: Int) extends DataType(typeName, javaSQLType) +case class BigDecimalType(override val typeName: String, override val javaSQLType: Int) + extends DataType(typeName, javaSQLType) + case class Column(columnName: String, dataType: DataType, comment: String) { def toArray: Array[Any] = { diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/excel/StorageExcelWriter.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/excel/StorageExcelWriter.scala index 89865eff38..3740dc1aaf 100644 --- a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/excel/StorageExcelWriter.scala +++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/excel/StorageExcelWriter.scala @@ -97,8 +97,8 @@ class StorageExcelWriter( case VarcharType => style.setDataFormat(format.getFormat("@")) case DateType => style.setDataFormat(format.getFormat("m/d/yy h:mm")) case TimestampType => style.setDataFormat(format.getFormat("m/d/yy h:mm")) - case DecimalType(dataType, 3) => style.setDataFormat(format.getFormat("#.000000000")) - case BigDecimalType => style.setDataFormat(format.getFormat("#.000000000")) + case DecimalType(_, _) => style.setDataFormat(format.getFormat("#.000000000")) + case BigDecimalType(_, _) => style.setDataFormat(format.getFormat("#.000000000")) case _ => style.setDataFormat(format.getFormat("@")) } } @@ -171,10 +171,10 @@ class StorageExcelWriter( case VarcharType => cell.setCellValue(DataType.valueToString(elem)) case DateType => cell.setCellValue(getDate(elem)) case TimestampType => cell.setCellValue(getDate(elem)) - case DecimalType(dataType, 3) => + case DecimalType(_, _) => doubleCheck(DataType.valueToString(elem)) cell.setCellValue(DataType.valueToString(elem).toDouble) - case BigDecimalType => + case BigDecimalType(_, _) => doubleCheck(DataType.valueToString(elem)) cell.setCellValue(DataType.valueToString(elem).toDouble) case _ => diff --git a/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/ujes/client/utils/UJESClientUtils.scala b/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/ujes/client/utils/UJESClientUtils.scala index e75929ea8f..e31136f6ae 100644 --- a/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/ujes/client/utils/UJESClientUtils.scala +++ b/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/ujes/client/utils/UJESClientUtils.scala @@ -21,6 +21,7 @@ import org.apache.linkis.ujes.client.exception.UJESClientBuilderException import org.apache.linkis.ujes.client.request.JobExecuteAction.{EngineType, RunType} import org.apache.linkis.ujes.client.response.ResultSetResult +import java.math.{BigDecimal => JavaBigDecimal} import java.util import java.util.Locale @@ -29,6 +30,7 @@ import com.google.gson.{Gson, JsonObject} object UJESClientUtils { val gson: Gson = new Gson() + val DECIMAL_REGEX = "^decimal\\(\\s*\\d*\\s*,\\s*\\d*\\s*\\)".r.unanchored def toEngineType(engineType: String): EngineType = engineType match { case "spark" => EngineType.SPARK @@ -75,7 +77,7 @@ object UJESClientUtils { case "boolean" => value.toBoolean case "byte" => value.toByte case "bigint" => value.toLong - case "decimal" => value.toDouble + case "decimal" | DECIMAL_REGEX() => new JavaBigDecimal(value) case "array" => gson.fromJson(value, classOf[util.ArrayList[Object]]) case "map" => gson.fromJson(value, classOf[util.HashMap[Object, Object]]) case "struct" => gson.fromJson(value, classOf[JsonObject]) diff --git a/linkis-engineconn-plugins/jdbc/src/main/scala/org/apache/linkis/manager/engineplugin/jdbc/executor/JDBCHelper.java b/linkis-engineconn-plugins/jdbc/src/main/scala/org/apache/linkis/manager/engineplugin/jdbc/executor/JDBCHelper.java index b3437ac129..8117c617de 100644 --- a/linkis-engineconn-plugins/jdbc/src/main/scala/org/apache/linkis/manager/engineplugin/jdbc/executor/JDBCHelper.java +++ b/linkis-engineconn-plugins/jdbc/src/main/scala/org/apache/linkis/manager/engineplugin/jdbc/executor/JDBCHelper.java @@ -146,7 +146,7 @@ public static String getTypeStr(int type) { retVal = BinaryType.typeName(); break; case Types.DECIMAL: - retVal = DecimalType.toDataType(String.valueOf(type)).toString(); + retVal = new DecimalType("decimal", 3).typeName(); break; case Types.ARRAY: retVal = ArrayType.typeName(); diff --git a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/spark/sql/execution/datasources/csv/DolphinToSpark.scala b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/spark/sql/execution/datasources/csv/DolphinToSpark.scala index e9411e45c3..dc4f1bfff0 100644 --- a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/spark/sql/execution/datasources/csv/DolphinToSpark.scala +++ b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/spark/sql/execution/datasources/csv/DolphinToSpark.scala @@ -77,7 +77,7 @@ object DolphinToSpark { case wds.BigIntType => LongType case wds.FloatType => FloatType case wds.DoubleType => DoubleType - case wds.DecimalType(dataType, 3) => DecimalType(bigDecimalPrecision, bigDecimalScale) + case wds.DecimalType(_, _) => DecimalType(bigDecimalPrecision, bigDecimalScale) case wds.DateType => DateType // case wds.TimestampType => TimestampType case wds.BinaryType => BinaryType