From 3be4e246c8a8fec1eedad6d5996a9f0f2804d763 Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Tue, 6 Aug 2024 14:11:04 +0800 Subject: [PATCH 1/6] Rename tests --- ...ffHeapSizingTest.scala => DynamicOffHeapSizingSuite.scala} | 2 +- ...nctionsValidateTest.scala => FunctionsValidateSuite.scala} | 2 +- .../gluten/execution/ScalarFunctionsValidateSuite.scala | 2 +- .../gluten/execution/WindowFunctionsValidateSuite.scala | 2 +- .../gluten/fuzzer/{FuzzerTestBase.scala => FuzzerBase.scala} | 4 ++-- .../fuzzer/{FuzzerTestResult.scala => FuzzerResult.scala} | 2 +- ...owToColumnarFuzzerTest.scala => RowToColumnarFuzzer.scala} | 4 ++-- ...huffleWriterFuzzerTest.scala => ShuffleWriterFuzzer.scala} | 4 ++-- 8 files changed, 11 insertions(+), 11 deletions(-) rename backends-velox/src/test/scala/org/apache/gluten/execution/{DynamicOffHeapSizingTest.scala => DynamicOffHeapSizingSuite.scala} (96%) rename backends-velox/src/test/scala/org/apache/gluten/execution/{FunctionsValidateTest.scala => FunctionsValidateSuite.scala} (97%) rename backends-velox/src/test/scala/org/apache/gluten/fuzzer/{FuzzerTestBase.scala => FuzzerBase.scala} (96%) rename backends-velox/src/test/scala/org/apache/gluten/fuzzer/{FuzzerTestResult.scala => FuzzerResult.scala} (97%) rename backends-velox/src/test/scala/org/apache/gluten/fuzzer/{RowToColumnarFuzzerTest.scala => RowToColumnarFuzzer.scala} (95%) rename backends-velox/src/test/scala/org/apache/gluten/fuzzer/{ShuffleWriterFuzzerTest.scala => ShuffleWriterFuzzer.scala} (96%) diff --git a/backends-velox/src/test/scala/org/apache/gluten/execution/DynamicOffHeapSizingTest.scala b/backends-velox/src/test/scala/org/apache/gluten/execution/DynamicOffHeapSizingSuite.scala similarity index 96% rename from backends-velox/src/test/scala/org/apache/gluten/execution/DynamicOffHeapSizingTest.scala rename to backends-velox/src/test/scala/org/apache/gluten/execution/DynamicOffHeapSizingSuite.scala index 56fc6eac3e11..51d06110ed60 100644 --- a/backends-velox/src/test/scala/org/apache/gluten/execution/DynamicOffHeapSizingTest.scala +++ b/backends-velox/src/test/scala/org/apache/gluten/execution/DynamicOffHeapSizingSuite.scala @@ -22,7 +22,7 @@ import org.apache.gluten.tags.SkipTestTags import org.apache.spark.SparkConf @SkipTestTags -class DynamicOffHeapSizingTest extends VeloxWholeStageTransformerSuite { +class DynamicOffHeapSizingSuite extends VeloxWholeStageTransformerSuite { override protected val resourcePath: String = "/tpch-data-parquet-velox" override protected val fileFormat: String = "parquet" diff --git a/backends-velox/src/test/scala/org/apache/gluten/execution/FunctionsValidateTest.scala b/backends-velox/src/test/scala/org/apache/gluten/execution/FunctionsValidateSuite.scala similarity index 97% rename from backends-velox/src/test/scala/org/apache/gluten/execution/FunctionsValidateTest.scala rename to backends-velox/src/test/scala/org/apache/gluten/execution/FunctionsValidateSuite.scala index 12f66278f70a..a9c5cf70da64 100644 --- a/backends-velox/src/test/scala/org/apache/gluten/execution/FunctionsValidateTest.scala +++ b/backends-velox/src/test/scala/org/apache/gluten/execution/FunctionsValidateSuite.scala @@ -25,7 +25,7 @@ import java.nio.file.Files import scala.collection.JavaConverters._ -class FunctionsValidateTest extends WholeStageTransformerSuite { +class FunctionsValidateSuite extends WholeStageTransformerSuite { override protected val resourcePath: String = "/tpch-data-parquet-velox" override protected val fileFormat: String = "parquet" private var parquetPath: String = _ diff --git a/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala b/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala index a0f7d22e4a54..b8de30b1b06f 100644 --- a/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala +++ b/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala @@ -67,7 +67,7 @@ class ScalarFunctionsValidateSuiteRasOn extends ScalarFunctionsValidateSuite { } } -abstract class ScalarFunctionsValidateSuite extends FunctionsValidateTest { +abstract class ScalarFunctionsValidateSuite extends FunctionsValidateSuite { disableFallbackCheck import testImplicits._ diff --git a/backends-velox/src/test/scala/org/apache/gluten/execution/WindowFunctionsValidateSuite.scala b/backends-velox/src/test/scala/org/apache/gluten/execution/WindowFunctionsValidateSuite.scala index ef9c028be46d..04d0d2c56b94 100644 --- a/backends-velox/src/test/scala/org/apache/gluten/execution/WindowFunctionsValidateSuite.scala +++ b/backends-velox/src/test/scala/org/apache/gluten/execution/WindowFunctionsValidateSuite.scala @@ -16,7 +16,7 @@ */ package org.apache.gluten.execution -class WindowFunctionsValidateSuite extends FunctionsValidateTest { +class WindowFunctionsValidateSuite extends FunctionsValidateSuite { test("lag/lead window function with negative input offset") { runQueryAndCompare( diff --git a/backends-velox/src/test/scala/org/apache/gluten/fuzzer/FuzzerTestBase.scala b/backends-velox/src/test/scala/org/apache/gluten/fuzzer/FuzzerBase.scala similarity index 96% rename from backends-velox/src/test/scala/org/apache/gluten/fuzzer/FuzzerTestBase.scala rename to backends-velox/src/test/scala/org/apache/gluten/fuzzer/FuzzerBase.scala index 1ee79a2ade87..07996bc4b266 100644 --- a/backends-velox/src/test/scala/org/apache/gluten/fuzzer/FuzzerTestBase.scala +++ b/backends-velox/src/test/scala/org/apache/gluten/fuzzer/FuzzerBase.scala @@ -18,12 +18,12 @@ package org.apache.gluten.fuzzer import org.apache.gluten.benchmarks.RandomParquetDataGenerator import org.apache.gluten.execution.VeloxWholeStageTransformerSuite -import org.apache.gluten.fuzzer.FuzzerTestResult.{Failed, OOM, Successful, TestResult} +import org.apache.gluten.fuzzer.FuzzerResult.{Failed, OOM, Successful, TestResult} import org.apache.gluten.memory.memtarget.ThrowOnOomMemoryTarget import org.apache.spark.SparkConf -abstract class FuzzerTestBase extends VeloxWholeStageTransformerSuite { +abstract class FuzzerBase extends VeloxWholeStageTransformerSuite { override protected val resourcePath: String = "/tpch-data-parquet-velox" override protected val fileFormat: String = "parquet" diff --git a/backends-velox/src/test/scala/org/apache/gluten/fuzzer/FuzzerTestResult.scala b/backends-velox/src/test/scala/org/apache/gluten/fuzzer/FuzzerResult.scala similarity index 97% rename from backends-velox/src/test/scala/org/apache/gluten/fuzzer/FuzzerTestResult.scala rename to backends-velox/src/test/scala/org/apache/gluten/fuzzer/FuzzerResult.scala index 42f901b068b4..cff9ffec8fd5 100644 --- a/backends-velox/src/test/scala/org/apache/gluten/fuzzer/FuzzerTestResult.scala +++ b/backends-velox/src/test/scala/org/apache/gluten/fuzzer/FuzzerResult.scala @@ -16,7 +16,7 @@ */ package org.apache.gluten.fuzzer -object FuzzerTestResult { +object FuzzerResult { trait TestResult { val seed: Long diff --git a/backends-velox/src/test/scala/org/apache/gluten/fuzzer/RowToColumnarFuzzerTest.scala b/backends-velox/src/test/scala/org/apache/gluten/fuzzer/RowToColumnarFuzzer.scala similarity index 95% rename from backends-velox/src/test/scala/org/apache/gluten/fuzzer/RowToColumnarFuzzerTest.scala rename to backends-velox/src/test/scala/org/apache/gluten/fuzzer/RowToColumnarFuzzer.scala index b6d986047e2c..d60e577aca09 100644 --- a/backends-velox/src/test/scala/org/apache/gluten/fuzzer/RowToColumnarFuzzerTest.scala +++ b/backends-velox/src/test/scala/org/apache/gluten/fuzzer/RowToColumnarFuzzer.scala @@ -17,7 +17,7 @@ package org.apache.gluten.fuzzer import org.apache.gluten.execution.RowToVeloxColumnarExec -import org.apache.gluten.fuzzer.FuzzerTestResult.Successful +import org.apache.gluten.fuzzer.FuzzerResult.Successful import org.apache.gluten.tags.{FuzzerTest, SkipTestTags} import org.apache.spark.SparkConf @@ -25,7 +25,7 @@ import org.apache.spark.sql.DataFrame @FuzzerTest @SkipTestTags -class RowToColumnarFuzzerTest extends FuzzerTestBase { +class RowToColumnarFuzzer extends FuzzerBase { override protected def sparkConf: SparkConf = { super.sparkConf diff --git a/backends-velox/src/test/scala/org/apache/gluten/fuzzer/ShuffleWriterFuzzerTest.scala b/backends-velox/src/test/scala/org/apache/gluten/fuzzer/ShuffleWriterFuzzer.scala similarity index 96% rename from backends-velox/src/test/scala/org/apache/gluten/fuzzer/ShuffleWriterFuzzerTest.scala rename to backends-velox/src/test/scala/org/apache/gluten/fuzzer/ShuffleWriterFuzzer.scala index 7d8fc56d9728..44f25b517b46 100644 --- a/backends-velox/src/test/scala/org/apache/gluten/fuzzer/ShuffleWriterFuzzerTest.scala +++ b/backends-velox/src/test/scala/org/apache/gluten/fuzzer/ShuffleWriterFuzzer.scala @@ -16,7 +16,7 @@ */ package org.apache.gluten.fuzzer -import org.apache.gluten.fuzzer.FuzzerTestResult.Successful +import org.apache.gluten.fuzzer.FuzzerResult.Successful import org.apache.gluten.tags.{FuzzerTest, SkipTestTags} import org.apache.spark.sql.DataFrame @@ -24,7 +24,7 @@ import org.apache.spark.sql.execution.ColumnarShuffleExchangeExec @FuzzerTest @SkipTestTags -class ShuffleWriterFuzzerTest extends FuzzerTestBase { +class ShuffleWriterFuzzer extends FuzzerBase { private val REPARTITION_SQL = (numPartitions: Int) => s"select /*+ REPARTITION($numPartitions) */ * from tbl" private val AGG_REPARTITION_SQL = From 30ef67dd11a70d50fa7d0ccb575742974c4194be Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Tue, 6 Aug 2024 13:33:21 +0800 Subject: [PATCH 2/6] rename package --- .../org/apache/gluten/utils/VeloxBatchResizer.java | 4 ++-- .../gluten/utils/VeloxBatchResizerJniWrapper.java | 4 ++-- .../java/org/apache/gluten/utils/VeloxBloomFilter.java | 2 +- .../gluten/utils/VeloxBloomFilterJniWrapper.java | 4 ++-- .../gluten/backendsapi/velox/VeloxTransformerApi.scala | 2 +- .../gluten/execution/RowToVeloxColumnarExec.scala | 2 +- .../gluten/execution/VeloxColumnarToRowExec.scala | 2 +- .../scala/org/apache/gluten/utils/DatasourceUtil.scala | 2 +- .../apache/spark/sql/execution/BroadcastUtils.scala | 2 +- .../sql/execution/ColumnarCachedBatchSerializer.scala | 2 +- .../datasources/velox/VeloxFormatWriterInjects.scala | 2 +- cpp/core/jni/JniCommon.cc | 2 +- cpp/core/jni/JniWrapper.cc | 10 +++++----- .../gluten/columnarbatch/ColumnarBatchJniWrapper.java | 4 ++-- .../apache/gluten/columnarbatch/ColumnarBatches.java | 4 ++-- .../gluten/columnarbatch/IndicatorVectorBase.java | 2 +- .../apache/gluten/datasource/DatasourceJniWrapper.java | 4 ++-- .../apache/gluten/{exec => runtime}/RuntimeAware.java | 2 +- .../gluten/{exec => runtime}/RuntimeJniWrapper.java | 2 +- .../gluten/vectorized/ColumnarBatchInIterator.java | 2 +- .../gluten/vectorized/ColumnarBatchOutIterator.java | 4 ++-- .../vectorized/ColumnarBatchSerializerJniWrapper.java | 4 ++-- .../vectorized/NativeColumnarToRowJniWrapper.java | 4 ++-- .../apache/gluten/vectorized/NativePlanEvaluator.java | 4 ++-- .../vectorized/NativeRowToColumnarJniWrapper.java | 4 ++-- .../gluten/vectorized/PlanEvaluatorJniWrapper.java | 4 ++-- .../gluten/vectorized/ShuffleReaderJniWrapper.java | 4 ++-- .../gluten/vectorized/ShuffleWriterJniWrapper.java | 4 ++-- .../org/apache/gluten/{exec => runtime}/Runtime.scala | 4 ++-- .../org/apache/gluten/{exec => runtime}/Runtimes.scala | 2 +- .../gluten/vectorized/ColumnarBatchSerializer.scala | 2 +- .../apache/spark/shuffle/ColumnarShuffleWriter.scala | 2 +- .../sql/execution/ColumnarBuildSideRelation.scala | 2 +- .../apache/spark/sql/execution/utils/ExecUtil.scala | 2 +- .../writer/VeloxUniffleColumnarShuffleWriter.java | 4 ++-- 35 files changed, 55 insertions(+), 55 deletions(-) rename gluten-data/src/main/java/org/apache/gluten/{exec => runtime}/RuntimeAware.java (96%) rename gluten-data/src/main/java/org/apache/gluten/{exec => runtime}/RuntimeJniWrapper.java (97%) rename gluten-data/src/main/scala/org/apache/gluten/{exec => runtime}/Runtime.scala (97%) rename gluten-data/src/main/scala/org/apache/gluten/{exec => runtime}/Runtimes.scala (97%) diff --git a/backends-velox/src/main/java/org/apache/gluten/utils/VeloxBatchResizer.java b/backends-velox/src/main/java/org/apache/gluten/utils/VeloxBatchResizer.java index e2f11cd3510b..2a6dcb43a052 100644 --- a/backends-velox/src/main/java/org/apache/gluten/utils/VeloxBatchResizer.java +++ b/backends-velox/src/main/java/org/apache/gluten/utils/VeloxBatchResizer.java @@ -16,8 +16,8 @@ */ package org.apache.gluten.utils; -import org.apache.gluten.exec.Runtime; -import org.apache.gluten.exec.Runtimes; +import org.apache.gluten.runtime.Runtime; +import org.apache.gluten.runtime.Runtimes; import org.apache.gluten.vectorized.ColumnarBatchInIterator; import org.apache.gluten.vectorized.ColumnarBatchOutIterator; diff --git a/backends-velox/src/main/java/org/apache/gluten/utils/VeloxBatchResizerJniWrapper.java b/backends-velox/src/main/java/org/apache/gluten/utils/VeloxBatchResizerJniWrapper.java index 3011ced2ab37..8855dd2acbbd 100644 --- a/backends-velox/src/main/java/org/apache/gluten/utils/VeloxBatchResizerJniWrapper.java +++ b/backends-velox/src/main/java/org/apache/gluten/utils/VeloxBatchResizerJniWrapper.java @@ -16,8 +16,8 @@ */ package org.apache.gluten.utils; -import org.apache.gluten.exec.Runtime; -import org.apache.gluten.exec.RuntimeAware; +import org.apache.gluten.runtime.Runtime; +import org.apache.gluten.runtime.RuntimeAware; import org.apache.gluten.vectorized.ColumnarBatchInIterator; public class VeloxBatchResizerJniWrapper implements RuntimeAware { diff --git a/backends-velox/src/main/java/org/apache/gluten/utils/VeloxBloomFilter.java b/backends-velox/src/main/java/org/apache/gluten/utils/VeloxBloomFilter.java index f23426d7da9d..10179d63edb3 100644 --- a/backends-velox/src/main/java/org/apache/gluten/utils/VeloxBloomFilter.java +++ b/backends-velox/src/main/java/org/apache/gluten/utils/VeloxBloomFilter.java @@ -16,7 +16,7 @@ */ package org.apache.gluten.utils; -import org.apache.gluten.exec.Runtimes; +import org.apache.gluten.runtime.Runtimes; import org.apache.commons.io.IOUtils; import org.apache.spark.util.sketch.BloomFilter; diff --git a/backends-velox/src/main/java/org/apache/gluten/utils/VeloxBloomFilterJniWrapper.java b/backends-velox/src/main/java/org/apache/gluten/utils/VeloxBloomFilterJniWrapper.java index 3ddfd2c02ed8..d5ee78415086 100644 --- a/backends-velox/src/main/java/org/apache/gluten/utils/VeloxBloomFilterJniWrapper.java +++ b/backends-velox/src/main/java/org/apache/gluten/utils/VeloxBloomFilterJniWrapper.java @@ -16,8 +16,8 @@ */ package org.apache.gluten.utils; -import org.apache.gluten.exec.Runtime; -import org.apache.gluten.exec.RuntimeAware; +import org.apache.gluten.runtime.Runtime; +import org.apache.gluten.runtime.RuntimeAware; public class VeloxBloomFilterJniWrapper implements RuntimeAware { private final Runtime runtime; diff --git a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxTransformerApi.scala b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxTransformerApi.scala index 4cbde635e9bd..f1a0cd56a922 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxTransformerApi.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxTransformerApi.scala @@ -17,7 +17,7 @@ package org.apache.gluten.backendsapi.velox import org.apache.gluten.backendsapi.TransformerApi -import org.apache.gluten.exec.Runtimes +import org.apache.gluten.runtime.Runtimes import org.apache.gluten.expression.ConverterUtils import org.apache.gluten.substrait.expression.{ExpressionBuilder, ExpressionNode} import org.apache.gluten.utils.InputPartitionsUtil diff --git a/backends-velox/src/main/scala/org/apache/gluten/execution/RowToVeloxColumnarExec.scala b/backends-velox/src/main/scala/org/apache/gluten/execution/RowToVeloxColumnarExec.scala index 2f3e88f9af9c..4cc8c182b798 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/execution/RowToVeloxColumnarExec.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/execution/RowToVeloxColumnarExec.scala @@ -18,7 +18,7 @@ package org.apache.gluten.execution import org.apache.gluten.GlutenConfig import org.apache.gluten.columnarbatch.ColumnarBatches -import org.apache.gluten.exec.Runtimes +import org.apache.gluten.runtime.Runtimes import org.apache.gluten.memory.arrow.alloc.ArrowBufferAllocators import org.apache.gluten.utils.ArrowAbiUtil import org.apache.gluten.utils.iterator.Iterators diff --git a/backends-velox/src/main/scala/org/apache/gluten/execution/VeloxColumnarToRowExec.scala b/backends-velox/src/main/scala/org/apache/gluten/execution/VeloxColumnarToRowExec.scala index 993a888b91df..2596b876af7a 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/execution/VeloxColumnarToRowExec.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/execution/VeloxColumnarToRowExec.scala @@ -18,7 +18,7 @@ package org.apache.gluten.execution import org.apache.gluten.columnarbatch.ColumnarBatches import org.apache.gluten.exception.GlutenNotSupportException -import org.apache.gluten.exec.Runtimes +import org.apache.gluten.runtime.Runtimes import org.apache.gluten.extension.ValidationResult import org.apache.gluten.utils.iterator.Iterators import org.apache.gluten.vectorized.NativeColumnarToRowJniWrapper diff --git a/backends-velox/src/main/scala/org/apache/gluten/utils/DatasourceUtil.scala b/backends-velox/src/main/scala/org/apache/gluten/utils/DatasourceUtil.scala index 3d7725655bb1..5a795daa2fe8 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/utils/DatasourceUtil.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/utils/DatasourceUtil.scala @@ -17,7 +17,7 @@ package org.apache.gluten.utils import org.apache.gluten.datasource.DatasourceJniWrapper -import org.apache.gluten.exec.Runtimes +import org.apache.gluten.runtime.Runtimes import org.apache.gluten.memory.arrow.alloc.ArrowBufferAllocators import org.apache.spark.sql.types.StructType diff --git a/backends-velox/src/main/scala/org/apache/spark/sql/execution/BroadcastUtils.scala b/backends-velox/src/main/scala/org/apache/spark/sql/execution/BroadcastUtils.scala index 3d532133a053..29a12f532bea 100644 --- a/backends-velox/src/main/scala/org/apache/spark/sql/execution/BroadcastUtils.scala +++ b/backends-velox/src/main/scala/org/apache/spark/sql/execution/BroadcastUtils.scala @@ -17,7 +17,7 @@ package org.apache.spark.sql.execution import org.apache.gluten.columnarbatch.ColumnarBatches -import org.apache.gluten.exec.Runtimes +import org.apache.gluten.runtime.Runtimes import org.apache.gluten.sql.shims.SparkShimLoader import org.apache.gluten.vectorized.{ColumnarBatchSerializeResult, ColumnarBatchSerializerJniWrapper} diff --git a/backends-velox/src/main/scala/org/apache/spark/sql/execution/ColumnarCachedBatchSerializer.scala b/backends-velox/src/main/scala/org/apache/spark/sql/execution/ColumnarCachedBatchSerializer.scala index 15fd51abef48..493a894dc81d 100644 --- a/backends-velox/src/main/scala/org/apache/spark/sql/execution/ColumnarCachedBatchSerializer.scala +++ b/backends-velox/src/main/scala/org/apache/spark/sql/execution/ColumnarCachedBatchSerializer.scala @@ -19,7 +19,7 @@ package org.apache.spark.sql.execution import org.apache.gluten.GlutenConfig import org.apache.gluten.backendsapi.BackendsApiManager import org.apache.gluten.columnarbatch.ColumnarBatches -import org.apache.gluten.exec.Runtimes +import org.apache.gluten.runtime.Runtimes import org.apache.gluten.execution.{RowToVeloxColumnarExec, VeloxColumnarToRowExec} import org.apache.gluten.memory.arrow.alloc.ArrowBufferAllocators import org.apache.gluten.utils.ArrowAbiUtil diff --git a/backends-velox/src/main/scala/org/apache/spark/sql/execution/datasources/velox/VeloxFormatWriterInjects.scala b/backends-velox/src/main/scala/org/apache/spark/sql/execution/datasources/velox/VeloxFormatWriterInjects.scala index 7da4da5f0784..6cfe754f8424 100644 --- a/backends-velox/src/main/scala/org/apache/spark/sql/execution/datasources/velox/VeloxFormatWriterInjects.scala +++ b/backends-velox/src/main/scala/org/apache/spark/sql/execution/datasources/velox/VeloxFormatWriterInjects.scala @@ -19,7 +19,7 @@ package org.apache.spark.sql.execution.datasources.velox import org.apache.gluten.columnarbatch.{ColumnarBatches, ColumnarBatchJniWrapper} import org.apache.gluten.datasource.DatasourceJniWrapper import org.apache.gluten.exception.GlutenException -import org.apache.gluten.exec.Runtimes +import org.apache.gluten.runtime.Runtimes import org.apache.gluten.execution.datasource.GlutenRowSplitter import org.apache.gluten.memory.arrow.alloc.ArrowBufferAllocators import org.apache.gluten.utils.{ArrowAbiUtil, DatasourceUtil} diff --git a/cpp/core/jni/JniCommon.cc b/cpp/core/jni/JniCommon.cc index 759a9d121f91..0d2b5d874dd2 100644 --- a/cpp/core/jni/JniCommon.cc +++ b/cpp/core/jni/JniCommon.cc @@ -38,7 +38,7 @@ jmethodID gluten::JniCommonState::runtimeAwareCtxHandle() { } void gluten::JniCommonState::initialize(JNIEnv* env) { - runtimeAwareClass_ = createGlobalClassReference(env, "Lorg/apache/gluten/exec/RuntimeAware;"); + runtimeAwareClass_ = createGlobalClassReference(env, "Lorg/apache/gluten/runtime/RuntimeAware;"); runtimeAwareCtxHandle_ = getMethodIdOrError(env, runtimeAwareClass_, "handle", "()J"); JavaVM* vm; if (env->GetJavaVM(&vm) != JNI_OK) { diff --git a/cpp/core/jni/JniWrapper.cc b/cpp/core/jni/JniWrapper.cc index 60f367fd72d1..3e583f20b519 100644 --- a/cpp/core/jni/JniWrapper.cc +++ b/cpp/core/jni/JniWrapper.cc @@ -222,7 +222,7 @@ namespace { const std::string kBacktraceAllocation = "spark.gluten.memory.backtrace.allocation"; } -JNIEXPORT jlong JNICALL Java_org_apache_gluten_exec_RuntimeJniWrapper_createRuntime( // NOLINT +JNIEXPORT jlong JNICALL Java_org_apache_gluten_runtime_RuntimeJniWrapper_createRuntime( // NOLINT JNIEnv* env, jclass, jstring jbackendType, @@ -249,7 +249,7 @@ JNIEXPORT jlong JNICALL Java_org_apache_gluten_exec_RuntimeJniWrapper_createRunt JNI_METHOD_END(kInvalidObjectHandle) } -JNIEXPORT jbyteArray JNICALL Java_org_apache_gluten_exec_RuntimeJniWrapper_collectMemoryUsage( // NOLINT +JNIEXPORT jbyteArray JNICALL Java_org_apache_gluten_runtime_RuntimeJniWrapper_collectMemoryUsage( // NOLINT JNIEnv* env, jclass, jlong ctxHandle) { @@ -268,7 +268,7 @@ JNIEXPORT jbyteArray JNICALL Java_org_apache_gluten_exec_RuntimeJniWrapper_colle JNI_METHOD_END(nullptr) } -JNIEXPORT jlong JNICALL Java_org_apache_gluten_exec_RuntimeJniWrapper_shrinkMemory( // NOLINT +JNIEXPORT jlong JNICALL Java_org_apache_gluten_runtime_RuntimeJniWrapper_shrinkMemory( // NOLINT JNIEnv* env, jclass, jlong ctxHandle, @@ -279,7 +279,7 @@ JNIEXPORT jlong JNICALL Java_org_apache_gluten_exec_RuntimeJniWrapper_shrinkMemo JNI_METHOD_END(kInvalidObjectHandle) } -JNIEXPORT void JNICALL Java_org_apache_gluten_exec_RuntimeJniWrapper_holdMemory( // NOLINT +JNIEXPORT void JNICALL Java_org_apache_gluten_runtime_RuntimeJniWrapper_holdMemory( // NOLINT JNIEnv* env, jclass, jlong ctxHandle) { @@ -289,7 +289,7 @@ JNIEXPORT void JNICALL Java_org_apache_gluten_exec_RuntimeJniWrapper_holdMemory( JNI_METHOD_END() } -JNIEXPORT void JNICALL Java_org_apache_gluten_exec_RuntimeJniWrapper_releaseRuntime( // NOLINT +JNIEXPORT void JNICALL Java_org_apache_gluten_runtime_RuntimeJniWrapper_releaseRuntime( // NOLINT JNIEnv* env, jclass, jlong ctxHandle) { diff --git a/gluten-data/src/main/java/org/apache/gluten/columnarbatch/ColumnarBatchJniWrapper.java b/gluten-data/src/main/java/org/apache/gluten/columnarbatch/ColumnarBatchJniWrapper.java index e71e9d7bee1b..37376951c543 100644 --- a/gluten-data/src/main/java/org/apache/gluten/columnarbatch/ColumnarBatchJniWrapper.java +++ b/gluten-data/src/main/java/org/apache/gluten/columnarbatch/ColumnarBatchJniWrapper.java @@ -16,8 +16,8 @@ */ package org.apache.gluten.columnarbatch; -import org.apache.gluten.exec.Runtime; -import org.apache.gluten.exec.RuntimeAware; +import org.apache.gluten.runtime.Runtime; +import org.apache.gluten.runtime.RuntimeAware; public class ColumnarBatchJniWrapper implements RuntimeAware { private final Runtime runtime; diff --git a/gluten-data/src/main/java/org/apache/gluten/columnarbatch/ColumnarBatches.java b/gluten-data/src/main/java/org/apache/gluten/columnarbatch/ColumnarBatches.java index a72eaafae4d9..543e6d4cf97b 100644 --- a/gluten-data/src/main/java/org/apache/gluten/columnarbatch/ColumnarBatches.java +++ b/gluten-data/src/main/java/org/apache/gluten/columnarbatch/ColumnarBatches.java @@ -17,8 +17,8 @@ package org.apache.gluten.columnarbatch; import org.apache.gluten.exception.GlutenException; -import org.apache.gluten.exec.Runtime; -import org.apache.gluten.exec.Runtimes; +import org.apache.gluten.runtime.Runtime; +import org.apache.gluten.runtime.Runtimes; import org.apache.gluten.utils.ArrowAbiUtil; import org.apache.gluten.utils.ArrowUtil; import org.apache.gluten.utils.ImplicitClass; diff --git a/gluten-data/src/main/java/org/apache/gluten/columnarbatch/IndicatorVectorBase.java b/gluten-data/src/main/java/org/apache/gluten/columnarbatch/IndicatorVectorBase.java index 1bc685bd5ceb..700eb3cadeee 100644 --- a/gluten-data/src/main/java/org/apache/gluten/columnarbatch/IndicatorVectorBase.java +++ b/gluten-data/src/main/java/org/apache/gluten/columnarbatch/IndicatorVectorBase.java @@ -16,7 +16,7 @@ */ package org.apache.gluten.columnarbatch; -import org.apache.gluten.exec.Runtimes; +import org.apache.gluten.runtime.Runtimes; import org.apache.spark.sql.types.DataTypes; import org.apache.spark.sql.types.Decimal; diff --git a/gluten-data/src/main/java/org/apache/gluten/datasource/DatasourceJniWrapper.java b/gluten-data/src/main/java/org/apache/gluten/datasource/DatasourceJniWrapper.java index 11ed3fb7df8c..fd36c85acc25 100644 --- a/gluten-data/src/main/java/org/apache/gluten/datasource/DatasourceJniWrapper.java +++ b/gluten-data/src/main/java/org/apache/gluten/datasource/DatasourceJniWrapper.java @@ -16,8 +16,8 @@ */ package org.apache.gluten.datasource; -import org.apache.gluten.exec.Runtime; -import org.apache.gluten.exec.RuntimeAware; +import org.apache.gluten.runtime.Runtime; +import org.apache.gluten.runtime.RuntimeAware; import org.apache.gluten.init.JniUtils; import org.apache.spark.sql.execution.datasources.BlockStripes; diff --git a/gluten-data/src/main/java/org/apache/gluten/exec/RuntimeAware.java b/gluten-data/src/main/java/org/apache/gluten/runtime/RuntimeAware.java similarity index 96% rename from gluten-data/src/main/java/org/apache/gluten/exec/RuntimeAware.java rename to gluten-data/src/main/java/org/apache/gluten/runtime/RuntimeAware.java index ca96ace64492..5caef9a69187 100644 --- a/gluten-data/src/main/java/org/apache/gluten/exec/RuntimeAware.java +++ b/gluten-data/src/main/java/org/apache/gluten/runtime/RuntimeAware.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.gluten.exec; +package org.apache.gluten.runtime; /** * This defines the base abstraction for the contextual objects that can be transmitted to C++ side diff --git a/gluten-data/src/main/java/org/apache/gluten/exec/RuntimeJniWrapper.java b/gluten-data/src/main/java/org/apache/gluten/runtime/RuntimeJniWrapper.java similarity index 97% rename from gluten-data/src/main/java/org/apache/gluten/exec/RuntimeJniWrapper.java rename to gluten-data/src/main/java/org/apache/gluten/runtime/RuntimeJniWrapper.java index d2a18e9b4930..80f9509d9eaa 100644 --- a/gluten-data/src/main/java/org/apache/gluten/exec/RuntimeJniWrapper.java +++ b/gluten-data/src/main/java/org/apache/gluten/runtime/RuntimeJniWrapper.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.gluten.exec; +package org.apache.gluten.runtime; import org.apache.gluten.memory.listener.ReservationListener; diff --git a/gluten-data/src/main/java/org/apache/gluten/vectorized/ColumnarBatchInIterator.java b/gluten-data/src/main/java/org/apache/gluten/vectorized/ColumnarBatchInIterator.java index c147862d0139..3892c442648b 100644 --- a/gluten-data/src/main/java/org/apache/gluten/vectorized/ColumnarBatchInIterator.java +++ b/gluten-data/src/main/java/org/apache/gluten/vectorized/ColumnarBatchInIterator.java @@ -18,7 +18,7 @@ import org.apache.gluten.columnarbatch.ColumnarBatchJniWrapper; import org.apache.gluten.columnarbatch.ColumnarBatches; -import org.apache.gluten.exec.Runtimes; +import org.apache.gluten.runtime.Runtimes; import org.apache.gluten.memory.arrow.alloc.ArrowBufferAllocators; import org.apache.spark.sql.vectorized.ColumnarBatch; diff --git a/gluten-data/src/main/java/org/apache/gluten/vectorized/ColumnarBatchOutIterator.java b/gluten-data/src/main/java/org/apache/gluten/vectorized/ColumnarBatchOutIterator.java index 9dd0404384ad..f6eb337a95a5 100644 --- a/gluten-data/src/main/java/org/apache/gluten/vectorized/ColumnarBatchOutIterator.java +++ b/gluten-data/src/main/java/org/apache/gluten/vectorized/ColumnarBatchOutIterator.java @@ -17,8 +17,8 @@ package org.apache.gluten.vectorized; import org.apache.gluten.columnarbatch.ColumnarBatches; -import org.apache.gluten.exec.Runtime; -import org.apache.gluten.exec.RuntimeAware; +import org.apache.gluten.runtime.Runtime; +import org.apache.gluten.runtime.RuntimeAware; import org.apache.gluten.metrics.IMetrics; import org.apache.spark.sql.vectorized.ColumnarBatch; diff --git a/gluten-data/src/main/java/org/apache/gluten/vectorized/ColumnarBatchSerializerJniWrapper.java b/gluten-data/src/main/java/org/apache/gluten/vectorized/ColumnarBatchSerializerJniWrapper.java index bfe0d756112f..d78cbdab2403 100644 --- a/gluten-data/src/main/java/org/apache/gluten/vectorized/ColumnarBatchSerializerJniWrapper.java +++ b/gluten-data/src/main/java/org/apache/gluten/vectorized/ColumnarBatchSerializerJniWrapper.java @@ -16,8 +16,8 @@ */ package org.apache.gluten.vectorized; -import org.apache.gluten.exec.Runtime; -import org.apache.gluten.exec.RuntimeAware; +import org.apache.gluten.runtime.Runtime; +import org.apache.gluten.runtime.RuntimeAware; public class ColumnarBatchSerializerJniWrapper implements RuntimeAware { private final Runtime runtime; diff --git a/gluten-data/src/main/java/org/apache/gluten/vectorized/NativeColumnarToRowJniWrapper.java b/gluten-data/src/main/java/org/apache/gluten/vectorized/NativeColumnarToRowJniWrapper.java index ffcb77ad32c8..947f4c8166ce 100644 --- a/gluten-data/src/main/java/org/apache/gluten/vectorized/NativeColumnarToRowJniWrapper.java +++ b/gluten-data/src/main/java/org/apache/gluten/vectorized/NativeColumnarToRowJniWrapper.java @@ -16,8 +16,8 @@ */ package org.apache.gluten.vectorized; -import org.apache.gluten.exec.Runtime; -import org.apache.gluten.exec.RuntimeAware; +import org.apache.gluten.runtime.Runtime; +import org.apache.gluten.runtime.RuntimeAware; public class NativeColumnarToRowJniWrapper implements RuntimeAware { private final Runtime runtime; diff --git a/gluten-data/src/main/java/org/apache/gluten/vectorized/NativePlanEvaluator.java b/gluten-data/src/main/java/org/apache/gluten/vectorized/NativePlanEvaluator.java index e5eea029b2b3..2fcf6ba99e50 100644 --- a/gluten-data/src/main/java/org/apache/gluten/vectorized/NativePlanEvaluator.java +++ b/gluten-data/src/main/java/org/apache/gluten/vectorized/NativePlanEvaluator.java @@ -17,8 +17,8 @@ package org.apache.gluten.vectorized; import org.apache.gluten.backendsapi.BackendsApiManager; -import org.apache.gluten.exec.Runtime; -import org.apache.gluten.exec.Runtimes; +import org.apache.gluten.runtime.Runtime; +import org.apache.gluten.runtime.Runtimes; import org.apache.gluten.memory.memtarget.MemoryTarget; import org.apache.gluten.memory.memtarget.Spiller; import org.apache.gluten.memory.memtarget.Spillers; diff --git a/gluten-data/src/main/java/org/apache/gluten/vectorized/NativeRowToColumnarJniWrapper.java b/gluten-data/src/main/java/org/apache/gluten/vectorized/NativeRowToColumnarJniWrapper.java index 1185d52fee77..c561174b2ba2 100644 --- a/gluten-data/src/main/java/org/apache/gluten/vectorized/NativeRowToColumnarJniWrapper.java +++ b/gluten-data/src/main/java/org/apache/gluten/vectorized/NativeRowToColumnarJniWrapper.java @@ -16,8 +16,8 @@ */ package org.apache.gluten.vectorized; -import org.apache.gluten.exec.Runtime; -import org.apache.gluten.exec.RuntimeAware; +import org.apache.gluten.runtime.Runtime; +import org.apache.gluten.runtime.RuntimeAware; public class NativeRowToColumnarJniWrapper implements RuntimeAware { private final Runtime runtime; diff --git a/gluten-data/src/main/java/org/apache/gluten/vectorized/PlanEvaluatorJniWrapper.java b/gluten-data/src/main/java/org/apache/gluten/vectorized/PlanEvaluatorJniWrapper.java index eecd7c9e2879..ee6a26a41a36 100644 --- a/gluten-data/src/main/java/org/apache/gluten/vectorized/PlanEvaluatorJniWrapper.java +++ b/gluten-data/src/main/java/org/apache/gluten/vectorized/PlanEvaluatorJniWrapper.java @@ -16,8 +16,8 @@ */ package org.apache.gluten.vectorized; -import org.apache.gluten.exec.Runtime; -import org.apache.gluten.exec.RuntimeAware; +import org.apache.gluten.runtime.Runtime; +import org.apache.gluten.runtime.RuntimeAware; import org.apache.gluten.validate.NativePlanValidationInfo; /** diff --git a/gluten-data/src/main/java/org/apache/gluten/vectorized/ShuffleReaderJniWrapper.java b/gluten-data/src/main/java/org/apache/gluten/vectorized/ShuffleReaderJniWrapper.java index 515486e45a5b..3d2f9e119732 100644 --- a/gluten-data/src/main/java/org/apache/gluten/vectorized/ShuffleReaderJniWrapper.java +++ b/gluten-data/src/main/java/org/apache/gluten/vectorized/ShuffleReaderJniWrapper.java @@ -16,8 +16,8 @@ */ package org.apache.gluten.vectorized; -import org.apache.gluten.exec.Runtime; -import org.apache.gluten.exec.RuntimeAware; +import org.apache.gluten.runtime.Runtime; +import org.apache.gluten.runtime.RuntimeAware; public class ShuffleReaderJniWrapper implements RuntimeAware { private final Runtime runtime; diff --git a/gluten-data/src/main/java/org/apache/gluten/vectorized/ShuffleWriterJniWrapper.java b/gluten-data/src/main/java/org/apache/gluten/vectorized/ShuffleWriterJniWrapper.java index 1d622d491eb5..23c7118afc6e 100644 --- a/gluten-data/src/main/java/org/apache/gluten/vectorized/ShuffleWriterJniWrapper.java +++ b/gluten-data/src/main/java/org/apache/gluten/vectorized/ShuffleWriterJniWrapper.java @@ -16,8 +16,8 @@ */ package org.apache.gluten.vectorized; -import org.apache.gluten.exec.Runtime; -import org.apache.gluten.exec.RuntimeAware; +import org.apache.gluten.runtime.Runtime; +import org.apache.gluten.runtime.RuntimeAware; import java.io.IOException; diff --git a/gluten-data/src/main/scala/org/apache/gluten/exec/Runtime.scala b/gluten-data/src/main/scala/org/apache/gluten/runtime/Runtime.scala similarity index 97% rename from gluten-data/src/main/scala/org/apache/gluten/exec/Runtime.scala rename to gluten-data/src/main/scala/org/apache/gluten/runtime/Runtime.scala index 1f632659eadf..8edceea447fc 100644 --- a/gluten-data/src/main/scala/org/apache/gluten/exec/Runtime.scala +++ b/gluten-data/src/main/scala/org/apache/gluten/runtime/Runtime.scala @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.gluten.exec +package org.apache.gluten.runtime import org.apache.gluten.GlutenConfig import org.apache.gluten.backendsapi.BackendsApiManager @@ -44,7 +44,7 @@ trait Runtime { } object Runtime { - private[exec] def apply(name: String): Runtime with TaskResource = { + private[runtime] def apply(name: String): Runtime with TaskResource = { new RuntimeImpl(name) } diff --git a/gluten-data/src/main/scala/org/apache/gluten/exec/Runtimes.scala b/gluten-data/src/main/scala/org/apache/gluten/runtime/Runtimes.scala similarity index 97% rename from gluten-data/src/main/scala/org/apache/gluten/exec/Runtimes.scala rename to gluten-data/src/main/scala/org/apache/gluten/runtime/Runtimes.scala index 3614fe05f6b6..6d5e11afeff5 100644 --- a/gluten-data/src/main/scala/org/apache/gluten/exec/Runtimes.scala +++ b/gluten-data/src/main/scala/org/apache/gluten/runtime/Runtimes.scala @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.gluten.exec +package org.apache.gluten.runtime import org.apache.spark.util.{TaskResource, TaskResources} diff --git a/gluten-data/src/main/scala/org/apache/gluten/vectorized/ColumnarBatchSerializer.scala b/gluten-data/src/main/scala/org/apache/gluten/vectorized/ColumnarBatchSerializer.scala index 379fb4885fb3..38d4acd96d04 100644 --- a/gluten-data/src/main/scala/org/apache/gluten/vectorized/ColumnarBatchSerializer.scala +++ b/gluten-data/src/main/scala/org/apache/gluten/vectorized/ColumnarBatchSerializer.scala @@ -17,7 +17,7 @@ package org.apache.gluten.vectorized import org.apache.gluten.GlutenConfig -import org.apache.gluten.exec.Runtimes +import org.apache.gluten.runtime.Runtimes import org.apache.gluten.memory.arrow.alloc.ArrowBufferAllocators import org.apache.gluten.utils.ArrowAbiUtil diff --git a/gluten-data/src/main/scala/org/apache/spark/shuffle/ColumnarShuffleWriter.scala b/gluten-data/src/main/scala/org/apache/spark/shuffle/ColumnarShuffleWriter.scala index d62ff1d68d6d..8662dbbfd2c5 100644 --- a/gluten-data/src/main/scala/org/apache/spark/shuffle/ColumnarShuffleWriter.scala +++ b/gluten-data/src/main/scala/org/apache/spark/shuffle/ColumnarShuffleWriter.scala @@ -18,7 +18,7 @@ package org.apache.spark.shuffle import org.apache.gluten.GlutenConfig import org.apache.gluten.columnarbatch.ColumnarBatches -import org.apache.gluten.exec.Runtimes +import org.apache.gluten.runtime.Runtimes import org.apache.gluten.memory.memtarget.{MemoryTarget, Spiller, Spillers} import org.apache.gluten.vectorized._ diff --git a/gluten-data/src/main/scala/org/apache/spark/sql/execution/ColumnarBuildSideRelation.scala b/gluten-data/src/main/scala/org/apache/spark/sql/execution/ColumnarBuildSideRelation.scala index 9f13ea967a8d..499c80d71c8d 100644 --- a/gluten-data/src/main/scala/org/apache/spark/sql/execution/ColumnarBuildSideRelation.scala +++ b/gluten-data/src/main/scala/org/apache/spark/sql/execution/ColumnarBuildSideRelation.scala @@ -17,7 +17,7 @@ package org.apache.spark.sql.execution import org.apache.gluten.columnarbatch.ColumnarBatches -import org.apache.gluten.exec.Runtimes +import org.apache.gluten.runtime.Runtimes import org.apache.gluten.memory.arrow.alloc.ArrowBufferAllocators import org.apache.gluten.sql.shims.SparkShimLoader import org.apache.gluten.utils.ArrowAbiUtil diff --git a/gluten-data/src/main/scala/org/apache/spark/sql/execution/utils/ExecUtil.scala b/gluten-data/src/main/scala/org/apache/spark/sql/execution/utils/ExecUtil.scala index 94bdc73a5b50..fc8f947f4984 100644 --- a/gluten-data/src/main/scala/org/apache/spark/sql/execution/utils/ExecUtil.scala +++ b/gluten-data/src/main/scala/org/apache/spark/sql/execution/utils/ExecUtil.scala @@ -17,7 +17,7 @@ package org.apache.spark.sql.execution.utils import org.apache.gluten.columnarbatch.ColumnarBatches -import org.apache.gluten.exec.Runtimes +import org.apache.gluten.runtime.Runtimes import org.apache.gluten.memory.arrow.alloc.ArrowBufferAllocators import org.apache.gluten.utils.iterator.Iterators import org.apache.gluten.vectorized.{ArrowWritableColumnVector, NativeColumnarToRowInfo, NativeColumnarToRowJniWrapper, NativePartitioning} diff --git a/gluten-uniffle/velox/src/main/java/org/apache/spark/shuffle/writer/VeloxUniffleColumnarShuffleWriter.java b/gluten-uniffle/velox/src/main/java/org/apache/spark/shuffle/writer/VeloxUniffleColumnarShuffleWriter.java index d2032fa48564..856da1e56756 100644 --- a/gluten-uniffle/velox/src/main/java/org/apache/spark/shuffle/writer/VeloxUniffleColumnarShuffleWriter.java +++ b/gluten-uniffle/velox/src/main/java/org/apache/spark/shuffle/writer/VeloxUniffleColumnarShuffleWriter.java @@ -18,8 +18,8 @@ import org.apache.gluten.GlutenConfig; import org.apache.gluten.columnarbatch.ColumnarBatches; -import org.apache.gluten.exec.Runtime; -import org.apache.gluten.exec.Runtimes; +import org.apache.gluten.runtime.Runtime; +import org.apache.gluten.runtime.Runtimes; import org.apache.gluten.memory.memtarget.MemoryTarget; import org.apache.gluten.memory.memtarget.Spiller; import org.apache.gluten.memory.memtarget.Spillers; From 9fe5bb7f3cdcbb34b4f44ac563ec50dd84ce7370 Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Thu, 8 Aug 2024 11:39:10 +0800 Subject: [PATCH 3/6] fixup --- dev/build_arrow.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dev/build_arrow.sh b/dev/build_arrow.sh index 8e1eefc21038..e7496350f988 100755 --- a/dev/build_arrow.sh +++ b/dev/build_arrow.sh @@ -14,6 +14,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +set -exu + CURRENT_DIR=$(cd "$(dirname "$BASH_SOURCE")"; pwd) export SUDO=sudo source ${CURRENT_DIR}/build_helper_functions.sh From b2635f9405e21bd45e767de7ad386d22ab3601ca Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Thu, 8 Aug 2024 13:42:10 +0800 Subject: [PATCH 4/6] fixup --- .../apache/gluten/backendsapi/velox/VeloxTransformerApi.scala | 2 +- .../org/apache/gluten/execution/RowToVeloxColumnarExec.scala | 2 +- .../org/apache/gluten/execution/VeloxColumnarToRowExec.scala | 2 +- .../main/scala/org/apache/gluten/utils/DatasourceUtil.scala | 2 +- .../spark/sql/execution/ColumnarCachedBatchSerializer.scala | 2 +- .../datasources/velox/VeloxFormatWriterInjects.scala | 2 +- .../org/apache/gluten/datasource/DatasourceJniWrapper.java | 2 +- .../org/apache/gluten/vectorized/ColumnarBatchInIterator.java | 2 +- .../apache/gluten/vectorized/ColumnarBatchOutIterator.java | 2 +- .../org/apache/gluten/vectorized/NativePlanEvaluator.java | 4 ++-- .../apache/gluten/vectorized/ColumnarBatchSerializer.scala | 2 +- .../org/apache/spark/shuffle/ColumnarShuffleWriter.scala | 2 +- .../spark/sql/execution/ColumnarBuildSideRelation.scala | 2 +- .../scala/org/apache/spark/sql/execution/utils/ExecUtil.scala | 2 +- .../shuffle/writer/VeloxUniffleColumnarShuffleWriter.java | 4 ++-- 15 files changed, 17 insertions(+), 17 deletions(-) diff --git a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxTransformerApi.scala b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxTransformerApi.scala index f1a0cd56a922..37d8acc7f2b7 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxTransformerApi.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxTransformerApi.scala @@ -17,8 +17,8 @@ package org.apache.gluten.backendsapi.velox import org.apache.gluten.backendsapi.TransformerApi -import org.apache.gluten.runtime.Runtimes import org.apache.gluten.expression.ConverterUtils +import org.apache.gluten.runtime.Runtimes import org.apache.gluten.substrait.expression.{ExpressionBuilder, ExpressionNode} import org.apache.gluten.utils.InputPartitionsUtil import org.apache.gluten.vectorized.PlanEvaluatorJniWrapper diff --git a/backends-velox/src/main/scala/org/apache/gluten/execution/RowToVeloxColumnarExec.scala b/backends-velox/src/main/scala/org/apache/gluten/execution/RowToVeloxColumnarExec.scala index 4cc8c182b798..9ceb6b5b6e52 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/execution/RowToVeloxColumnarExec.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/execution/RowToVeloxColumnarExec.scala @@ -18,8 +18,8 @@ package org.apache.gluten.execution import org.apache.gluten.GlutenConfig import org.apache.gluten.columnarbatch.ColumnarBatches -import org.apache.gluten.runtime.Runtimes import org.apache.gluten.memory.arrow.alloc.ArrowBufferAllocators +import org.apache.gluten.runtime.Runtimes import org.apache.gluten.utils.ArrowAbiUtil import org.apache.gluten.utils.iterator.Iterators import org.apache.gluten.vectorized._ diff --git a/backends-velox/src/main/scala/org/apache/gluten/execution/VeloxColumnarToRowExec.scala b/backends-velox/src/main/scala/org/apache/gluten/execution/VeloxColumnarToRowExec.scala index 2596b876af7a..d3fb9c3ffc70 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/execution/VeloxColumnarToRowExec.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/execution/VeloxColumnarToRowExec.scala @@ -18,8 +18,8 @@ package org.apache.gluten.execution import org.apache.gluten.columnarbatch.ColumnarBatches import org.apache.gluten.exception.GlutenNotSupportException -import org.apache.gluten.runtime.Runtimes import org.apache.gluten.extension.ValidationResult +import org.apache.gluten.runtime.Runtimes import org.apache.gluten.utils.iterator.Iterators import org.apache.gluten.vectorized.NativeColumnarToRowJniWrapper diff --git a/backends-velox/src/main/scala/org/apache/gluten/utils/DatasourceUtil.scala b/backends-velox/src/main/scala/org/apache/gluten/utils/DatasourceUtil.scala index 5a795daa2fe8..8963ce93c1f6 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/utils/DatasourceUtil.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/utils/DatasourceUtil.scala @@ -17,8 +17,8 @@ package org.apache.gluten.utils import org.apache.gluten.datasource.DatasourceJniWrapper -import org.apache.gluten.runtime.Runtimes import org.apache.gluten.memory.arrow.alloc.ArrowBufferAllocators +import org.apache.gluten.runtime.Runtimes import org.apache.spark.sql.types.StructType import org.apache.spark.sql.utils.SparkSchemaUtil diff --git a/backends-velox/src/main/scala/org/apache/spark/sql/execution/ColumnarCachedBatchSerializer.scala b/backends-velox/src/main/scala/org/apache/spark/sql/execution/ColumnarCachedBatchSerializer.scala index 493a894dc81d..db9e75a05681 100644 --- a/backends-velox/src/main/scala/org/apache/spark/sql/execution/ColumnarCachedBatchSerializer.scala +++ b/backends-velox/src/main/scala/org/apache/spark/sql/execution/ColumnarCachedBatchSerializer.scala @@ -19,9 +19,9 @@ package org.apache.spark.sql.execution import org.apache.gluten.GlutenConfig import org.apache.gluten.backendsapi.BackendsApiManager import org.apache.gluten.columnarbatch.ColumnarBatches -import org.apache.gluten.runtime.Runtimes import org.apache.gluten.execution.{RowToVeloxColumnarExec, VeloxColumnarToRowExec} import org.apache.gluten.memory.arrow.alloc.ArrowBufferAllocators +import org.apache.gluten.runtime.Runtimes import org.apache.gluten.utils.ArrowAbiUtil import org.apache.gluten.utils.iterator.Iterators import org.apache.gluten.vectorized.ColumnarBatchSerializerJniWrapper diff --git a/backends-velox/src/main/scala/org/apache/spark/sql/execution/datasources/velox/VeloxFormatWriterInjects.scala b/backends-velox/src/main/scala/org/apache/spark/sql/execution/datasources/velox/VeloxFormatWriterInjects.scala index 6cfe754f8424..e25d3c663d69 100644 --- a/backends-velox/src/main/scala/org/apache/spark/sql/execution/datasources/velox/VeloxFormatWriterInjects.scala +++ b/backends-velox/src/main/scala/org/apache/spark/sql/execution/datasources/velox/VeloxFormatWriterInjects.scala @@ -19,9 +19,9 @@ package org.apache.spark.sql.execution.datasources.velox import org.apache.gluten.columnarbatch.{ColumnarBatches, ColumnarBatchJniWrapper} import org.apache.gluten.datasource.DatasourceJniWrapper import org.apache.gluten.exception.GlutenException -import org.apache.gluten.runtime.Runtimes import org.apache.gluten.execution.datasource.GlutenRowSplitter import org.apache.gluten.memory.arrow.alloc.ArrowBufferAllocators +import org.apache.gluten.runtime.Runtimes import org.apache.gluten.utils.{ArrowAbiUtil, DatasourceUtil} import org.apache.spark.sql.SparkSession diff --git a/gluten-data/src/main/java/org/apache/gluten/datasource/DatasourceJniWrapper.java b/gluten-data/src/main/java/org/apache/gluten/datasource/DatasourceJniWrapper.java index fd36c85acc25..4550dbd9a4d8 100644 --- a/gluten-data/src/main/java/org/apache/gluten/datasource/DatasourceJniWrapper.java +++ b/gluten-data/src/main/java/org/apache/gluten/datasource/DatasourceJniWrapper.java @@ -16,9 +16,9 @@ */ package org.apache.gluten.datasource; +import org.apache.gluten.init.JniUtils; import org.apache.gluten.runtime.Runtime; import org.apache.gluten.runtime.RuntimeAware; -import org.apache.gluten.init.JniUtils; import org.apache.spark.sql.execution.datasources.BlockStripes; diff --git a/gluten-data/src/main/java/org/apache/gluten/vectorized/ColumnarBatchInIterator.java b/gluten-data/src/main/java/org/apache/gluten/vectorized/ColumnarBatchInIterator.java index 3892c442648b..ef7d7167c80d 100644 --- a/gluten-data/src/main/java/org/apache/gluten/vectorized/ColumnarBatchInIterator.java +++ b/gluten-data/src/main/java/org/apache/gluten/vectorized/ColumnarBatchInIterator.java @@ -18,8 +18,8 @@ import org.apache.gluten.columnarbatch.ColumnarBatchJniWrapper; import org.apache.gluten.columnarbatch.ColumnarBatches; -import org.apache.gluten.runtime.Runtimes; import org.apache.gluten.memory.arrow.alloc.ArrowBufferAllocators; +import org.apache.gluten.runtime.Runtimes; import org.apache.spark.sql.vectorized.ColumnarBatch; diff --git a/gluten-data/src/main/java/org/apache/gluten/vectorized/ColumnarBatchOutIterator.java b/gluten-data/src/main/java/org/apache/gluten/vectorized/ColumnarBatchOutIterator.java index f6eb337a95a5..4f436e58d071 100644 --- a/gluten-data/src/main/java/org/apache/gluten/vectorized/ColumnarBatchOutIterator.java +++ b/gluten-data/src/main/java/org/apache/gluten/vectorized/ColumnarBatchOutIterator.java @@ -17,9 +17,9 @@ package org.apache.gluten.vectorized; import org.apache.gluten.columnarbatch.ColumnarBatches; +import org.apache.gluten.metrics.IMetrics; import org.apache.gluten.runtime.Runtime; import org.apache.gluten.runtime.RuntimeAware; -import org.apache.gluten.metrics.IMetrics; import org.apache.spark.sql.vectorized.ColumnarBatch; diff --git a/gluten-data/src/main/java/org/apache/gluten/vectorized/NativePlanEvaluator.java b/gluten-data/src/main/java/org/apache/gluten/vectorized/NativePlanEvaluator.java index 2fcf6ba99e50..8fb18e439a60 100644 --- a/gluten-data/src/main/java/org/apache/gluten/vectorized/NativePlanEvaluator.java +++ b/gluten-data/src/main/java/org/apache/gluten/vectorized/NativePlanEvaluator.java @@ -17,11 +17,11 @@ package org.apache.gluten.vectorized; import org.apache.gluten.backendsapi.BackendsApiManager; -import org.apache.gluten.runtime.Runtime; -import org.apache.gluten.runtime.Runtimes; import org.apache.gluten.memory.memtarget.MemoryTarget; import org.apache.gluten.memory.memtarget.Spiller; import org.apache.gluten.memory.memtarget.Spillers; +import org.apache.gluten.runtime.Runtime; +import org.apache.gluten.runtime.Runtimes; import org.apache.gluten.utils.DebugUtil; import org.apache.gluten.validate.NativePlanValidationInfo; diff --git a/gluten-data/src/main/scala/org/apache/gluten/vectorized/ColumnarBatchSerializer.scala b/gluten-data/src/main/scala/org/apache/gluten/vectorized/ColumnarBatchSerializer.scala index 38d4acd96d04..0bd78cb92c5c 100644 --- a/gluten-data/src/main/scala/org/apache/gluten/vectorized/ColumnarBatchSerializer.scala +++ b/gluten-data/src/main/scala/org/apache/gluten/vectorized/ColumnarBatchSerializer.scala @@ -17,8 +17,8 @@ package org.apache.gluten.vectorized import org.apache.gluten.GlutenConfig -import org.apache.gluten.runtime.Runtimes import org.apache.gluten.memory.arrow.alloc.ArrowBufferAllocators +import org.apache.gluten.runtime.Runtimes import org.apache.gluten.utils.ArrowAbiUtil import org.apache.spark.SparkEnv diff --git a/gluten-data/src/main/scala/org/apache/spark/shuffle/ColumnarShuffleWriter.scala b/gluten-data/src/main/scala/org/apache/spark/shuffle/ColumnarShuffleWriter.scala index 8662dbbfd2c5..08535a39351c 100644 --- a/gluten-data/src/main/scala/org/apache/spark/shuffle/ColumnarShuffleWriter.scala +++ b/gluten-data/src/main/scala/org/apache/spark/shuffle/ColumnarShuffleWriter.scala @@ -18,8 +18,8 @@ package org.apache.spark.shuffle import org.apache.gluten.GlutenConfig import org.apache.gluten.columnarbatch.ColumnarBatches -import org.apache.gluten.runtime.Runtimes import org.apache.gluten.memory.memtarget.{MemoryTarget, Spiller, Spillers} +import org.apache.gluten.runtime.Runtimes import org.apache.gluten.vectorized._ import org.apache.spark._ diff --git a/gluten-data/src/main/scala/org/apache/spark/sql/execution/ColumnarBuildSideRelation.scala b/gluten-data/src/main/scala/org/apache/spark/sql/execution/ColumnarBuildSideRelation.scala index 499c80d71c8d..cb65dbca4db0 100644 --- a/gluten-data/src/main/scala/org/apache/spark/sql/execution/ColumnarBuildSideRelation.scala +++ b/gluten-data/src/main/scala/org/apache/spark/sql/execution/ColumnarBuildSideRelation.scala @@ -17,8 +17,8 @@ package org.apache.spark.sql.execution import org.apache.gluten.columnarbatch.ColumnarBatches -import org.apache.gluten.runtime.Runtimes import org.apache.gluten.memory.arrow.alloc.ArrowBufferAllocators +import org.apache.gluten.runtime.Runtimes import org.apache.gluten.sql.shims.SparkShimLoader import org.apache.gluten.utils.ArrowAbiUtil import org.apache.gluten.utils.iterator.Iterators diff --git a/gluten-data/src/main/scala/org/apache/spark/sql/execution/utils/ExecUtil.scala b/gluten-data/src/main/scala/org/apache/spark/sql/execution/utils/ExecUtil.scala index fc8f947f4984..65b06214e73b 100644 --- a/gluten-data/src/main/scala/org/apache/spark/sql/execution/utils/ExecUtil.scala +++ b/gluten-data/src/main/scala/org/apache/spark/sql/execution/utils/ExecUtil.scala @@ -17,8 +17,8 @@ package org.apache.spark.sql.execution.utils import org.apache.gluten.columnarbatch.ColumnarBatches -import org.apache.gluten.runtime.Runtimes import org.apache.gluten.memory.arrow.alloc.ArrowBufferAllocators +import org.apache.gluten.runtime.Runtimes import org.apache.gluten.utils.iterator.Iterators import org.apache.gluten.vectorized.{ArrowWritableColumnVector, NativeColumnarToRowInfo, NativeColumnarToRowJniWrapper, NativePartitioning} diff --git a/gluten-uniffle/velox/src/main/java/org/apache/spark/shuffle/writer/VeloxUniffleColumnarShuffleWriter.java b/gluten-uniffle/velox/src/main/java/org/apache/spark/shuffle/writer/VeloxUniffleColumnarShuffleWriter.java index 856da1e56756..d505260b8412 100644 --- a/gluten-uniffle/velox/src/main/java/org/apache/spark/shuffle/writer/VeloxUniffleColumnarShuffleWriter.java +++ b/gluten-uniffle/velox/src/main/java/org/apache/spark/shuffle/writer/VeloxUniffleColumnarShuffleWriter.java @@ -18,11 +18,11 @@ import org.apache.gluten.GlutenConfig; import org.apache.gluten.columnarbatch.ColumnarBatches; -import org.apache.gluten.runtime.Runtime; -import org.apache.gluten.runtime.Runtimes; import org.apache.gluten.memory.memtarget.MemoryTarget; import org.apache.gluten.memory.memtarget.Spiller; import org.apache.gluten.memory.memtarget.Spillers; +import org.apache.gluten.runtime.Runtime; +import org.apache.gluten.runtime.Runtimes; import org.apache.gluten.vectorized.ShuffleWriterJniWrapper; import org.apache.gluten.vectorized.SplitResult; From bf8159b16856e63c6c87ea9581cff05b8ea6b5a8 Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Thu, 8 Aug 2024 14:02:01 +0800 Subject: [PATCH 5/6] fixup --- .../spark/shuffle/VeloxCelebornColumnarBatchSerializer.scala | 2 +- .../spark/shuffle/VeloxCelebornColumnarShuffleWriter.scala | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/gluten-celeborn/velox/src/main/scala/org/apache/spark/shuffle/VeloxCelebornColumnarBatchSerializer.scala b/gluten-celeborn/velox/src/main/scala/org/apache/spark/shuffle/VeloxCelebornColumnarBatchSerializer.scala index 6f21b528f1c2..2f713d06ebd3 100644 --- a/gluten-celeborn/velox/src/main/scala/org/apache/spark/shuffle/VeloxCelebornColumnarBatchSerializer.scala +++ b/gluten-celeborn/velox/src/main/scala/org/apache/spark/shuffle/VeloxCelebornColumnarBatchSerializer.scala @@ -18,7 +18,7 @@ package org.apache.spark.shuffle import org.apache.gluten.GlutenConfig import org.apache.gluten.GlutenConfig.{GLUTEN_RSS_SORT_SHUFFLE_WRITER, GLUTEN_SORT_SHUFFLE_WRITER} -import org.apache.gluten.exec.Runtimes +import org.apache.gluten.runtime.Runtimes import org.apache.gluten.memory.arrow.alloc.ArrowBufferAllocators import org.apache.gluten.utils.ArrowAbiUtil import org.apache.gluten.vectorized._ diff --git a/gluten-celeborn/velox/src/main/scala/org/apache/spark/shuffle/VeloxCelebornColumnarShuffleWriter.scala b/gluten-celeborn/velox/src/main/scala/org/apache/spark/shuffle/VeloxCelebornColumnarShuffleWriter.scala index c14d46a52c12..99c6578f9b78 100644 --- a/gluten-celeborn/velox/src/main/scala/org/apache/spark/shuffle/VeloxCelebornColumnarShuffleWriter.scala +++ b/gluten-celeborn/velox/src/main/scala/org/apache/spark/shuffle/VeloxCelebornColumnarShuffleWriter.scala @@ -18,7 +18,7 @@ package org.apache.spark.shuffle import org.apache.gluten.GlutenConfig import org.apache.gluten.columnarbatch.ColumnarBatches -import org.apache.gluten.exec.Runtimes +import org.apache.gluten.runtime.Runtimes import org.apache.gluten.memory.memtarget.{MemoryTarget, Spiller, Spillers} import org.apache.gluten.vectorized._ From da0c9ee3b8ff53c4750b9010b1f4c9bba0af9b27 Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Thu, 8 Aug 2024 15:16:08 +0800 Subject: [PATCH 6/6] fixup --- .../spark/shuffle/VeloxCelebornColumnarBatchSerializer.scala | 2 +- .../spark/shuffle/VeloxCelebornColumnarShuffleWriter.scala | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/gluten-celeborn/velox/src/main/scala/org/apache/spark/shuffle/VeloxCelebornColumnarBatchSerializer.scala b/gluten-celeborn/velox/src/main/scala/org/apache/spark/shuffle/VeloxCelebornColumnarBatchSerializer.scala index 2f713d06ebd3..696a3c3438b0 100644 --- a/gluten-celeborn/velox/src/main/scala/org/apache/spark/shuffle/VeloxCelebornColumnarBatchSerializer.scala +++ b/gluten-celeborn/velox/src/main/scala/org/apache/spark/shuffle/VeloxCelebornColumnarBatchSerializer.scala @@ -18,8 +18,8 @@ package org.apache.spark.shuffle import org.apache.gluten.GlutenConfig import org.apache.gluten.GlutenConfig.{GLUTEN_RSS_SORT_SHUFFLE_WRITER, GLUTEN_SORT_SHUFFLE_WRITER} -import org.apache.gluten.runtime.Runtimes import org.apache.gluten.memory.arrow.alloc.ArrowBufferAllocators +import org.apache.gluten.runtime.Runtimes import org.apache.gluten.utils.ArrowAbiUtil import org.apache.gluten.vectorized._ diff --git a/gluten-celeborn/velox/src/main/scala/org/apache/spark/shuffle/VeloxCelebornColumnarShuffleWriter.scala b/gluten-celeborn/velox/src/main/scala/org/apache/spark/shuffle/VeloxCelebornColumnarShuffleWriter.scala index 99c6578f9b78..b7a0beae704b 100644 --- a/gluten-celeborn/velox/src/main/scala/org/apache/spark/shuffle/VeloxCelebornColumnarShuffleWriter.scala +++ b/gluten-celeborn/velox/src/main/scala/org/apache/spark/shuffle/VeloxCelebornColumnarShuffleWriter.scala @@ -18,8 +18,8 @@ package org.apache.spark.shuffle import org.apache.gluten.GlutenConfig import org.apache.gluten.columnarbatch.ColumnarBatches -import org.apache.gluten.runtime.Runtimes import org.apache.gluten.memory.memtarget.{MemoryTarget, Spiller, Spillers} +import org.apache.gluten.runtime.Runtimes import org.apache.gluten.vectorized._ import org.apache.spark._