From 880a99fb01f360a3d3aac0a42d242b285a4fb5a9 Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Mon, 26 Aug 2024 14:47:40 +0800 Subject: [PATCH] fixup --- .../apache/gluten/backendsapi/velox/VeloxListenerApi.scala | 7 +++++++ .../src/main/scala/org/apache/gluten/GlutenPlugin.scala | 7 ------- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala index 2cfc4e9a9099..56b439a37c2c 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala @@ -42,6 +42,13 @@ class VeloxListenerApi extends ListenerApi with Logging { override def onDriverStart(sc: SparkContext, pc: PluginContext): Unit = { val conf = pc.conf() + // FIXME: The following is a workaround. Remove once the causes are fixed. + conf.set(GlutenConfig.GLUTEN_OVERHEAD_SIZE_IN_BYTES_KEY, Long.MaxValue.toString) + logWarning( + "Setting overhead memory that Gluten can use to UNLIMITED. This is currently a" + + " temporary solution to avoid OOM by Velox's global memory pools." + + " See GLUTEN-6960 for more information.") + // Sql table cache serializer. if (conf.getBoolean(GlutenConfig.COLUMNAR_TABLE_CACHE_ENABLED.key, defaultValue = false)) { conf.set( diff --git a/gluten-core/src/main/scala/org/apache/gluten/GlutenPlugin.scala b/gluten-core/src/main/scala/org/apache/gluten/GlutenPlugin.scala index f775d78a15ac..5e0c1758d0b4 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/GlutenPlugin.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/GlutenPlugin.scala @@ -184,13 +184,6 @@ private[gluten] class GlutenDriverPlugin extends DriverPlugin with Logging { val overheadSize: Long = SparkResourceUtil.getMemoryOverheadSize(conf) conf.set(GlutenConfig.GLUTEN_OVERHEAD_SIZE_IN_BYTES_KEY, overheadSize.toString) - // FIXME: The following is a workaround. Remove once the causes are fixed. - conf.set(GlutenConfig.GLUTEN_OVERHEAD_SIZE_IN_BYTES_KEY, Long.MaxValue.toString) - logWarning( - "Setting overhead memory that Gluten can use to UNLIMITED. This is currently a" + - " temporary solution to avoid OOM by Velox's global memory pools." + - " See GLUTEN-6960 for more information.") - // If dynamic off-heap sizing is enabled, the off-heap size is calculated based on the on-heap // size. Otherwise, the off-heap size is set to the value specified by the user (if any). // Note that this means that we will IGNORE the off-heap size specified by the user if the