From 98aed905eb4ae2692ea0d6065d6eac83b5062786 Mon Sep 17 00:00:00 2001 From: Vitalii Tymchyshyn Date: Mon, 4 Dec 2023 15:16:09 -0800 Subject: [PATCH] [CDAP-20915] Use exprEnc() instead of encoder() as latter is only available in Spark 3 --- .../io/cdap/cdap/etl/spark/batch/DataframeCollection.java | 4 ++-- .../io/cdap/cdap/etl/spark/batch/OpaqueDatasetCollection.java | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cdap-app-templates/cdap-etl/hydrator-spark-core-base/src/main/java/io/cdap/cdap/etl/spark/batch/DataframeCollection.java b/cdap-app-templates/cdap-etl/hydrator-spark-core-base/src/main/java/io/cdap/cdap/etl/spark/batch/DataframeCollection.java index 5a6b0b755990..1b54d1ed9c2b 100644 --- a/cdap-app-templates/cdap-etl/hydrator-spark-core-base/src/main/java/io/cdap/cdap/etl/spark/batch/DataframeCollection.java +++ b/cdap-app-templates/cdap-etl/hydrator-spark-core-base/src/main/java/io/cdap/cdap/etl/spark/batch/DataframeCollection.java @@ -84,9 +84,9 @@ public DataframeCollection(Schema schema, Dataset dataframe, JavaSparkExecu super(sec, jsc, sqlContext, datasetContext, sinkFactory, functionCacheFactory); this.schema = Objects.requireNonNull(schema); this.dataframe = dataframe; - if (!Row.class.isAssignableFrom(dataframe.encoder().clsTag().runtimeClass())) { + if (!Row.class.isAssignableFrom(dataframe.exprEnc().clsTag().runtimeClass())) { throw new IllegalArgumentException( - "Dataframe collection received dataset of " + dataframe.encoder().clsTag() + "Dataframe collection received dataset of " + dataframe.exprEnc().clsTag() .runtimeClass()); } } diff --git a/cdap-app-templates/cdap-etl/hydrator-spark-core-base/src/main/java/io/cdap/cdap/etl/spark/batch/OpaqueDatasetCollection.java b/cdap-app-templates/cdap-etl/hydrator-spark-core-base/src/main/java/io/cdap/cdap/etl/spark/batch/OpaqueDatasetCollection.java index f20a1b4d12de..371bbef134e2 100644 --- a/cdap-app-templates/cdap-etl/hydrator-spark-core-base/src/main/java/io/cdap/cdap/etl/spark/batch/OpaqueDatasetCollection.java +++ b/cdap-app-templates/cdap-etl/hydrator-spark-core-base/src/main/java/io/cdap/cdap/etl/spark/batch/OpaqueDatasetCollection.java @@ -56,9 +56,9 @@ private OpaqueDatasetCollection(Dataset dataset, FunctionCache.Factory functionCacheFactory) { super(sec, jsc, sqlContext, datasetContext, sinkFactory, functionCacheFactory); this.dataset = dataset; - if (Row.class.isAssignableFrom(dataset.encoder().clsTag().runtimeClass())) { + if (Row.class.isAssignableFrom(dataset.exprEnc().clsTag().runtimeClass())) { throw new IllegalArgumentException( - "Opaque collection received dataset of Row (" + dataset.encoder().clsTag() + "Opaque collection received dataset of Row (" + dataset.exprEnc().clsTag() .runtimeClass() + "). DataframeCollection should be used."); } }