diff --git a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala
index 2cfc4e9a9099..56b439a37c2c 100644
--- a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala
+++ b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala
@@ -42,6 +42,13 @@ class VeloxListenerApi extends ListenerApi with Logging {
   override def onDriverStart(sc: SparkContext, pc: PluginContext): Unit = {
     val conf = pc.conf()
 
+    // FIXME: The following is a workaround. Remove once the causes are fixed.
+    conf.set(GlutenConfig.GLUTEN_OVERHEAD_SIZE_IN_BYTES_KEY, Long.MaxValue.toString)
+    logWarning(
+      "Setting overhead memory that Gluten can use to UNLIMITED. This is currently a" +
+        " temporary solution to avoid OOM by Velox's global memory pools." +
+        " See GLUTEN-6960 for more information.")
+
     // Sql table cache serializer.
     if (conf.getBoolean(GlutenConfig.COLUMNAR_TABLE_CACHE_ENABLED.key, defaultValue = false)) {
       conf.set(
diff --git a/gluten-core/src/main/scala/org/apache/gluten/GlutenPlugin.scala b/gluten-core/src/main/scala/org/apache/gluten/GlutenPlugin.scala
index f775d78a15ac..5e0c1758d0b4 100644
--- a/gluten-core/src/main/scala/org/apache/gluten/GlutenPlugin.scala
+++ b/gluten-core/src/main/scala/org/apache/gluten/GlutenPlugin.scala
@@ -184,13 +184,6 @@ private[gluten] class GlutenDriverPlugin extends DriverPlugin with Logging {
     val overheadSize: Long = SparkResourceUtil.getMemoryOverheadSize(conf)
     conf.set(GlutenConfig.GLUTEN_OVERHEAD_SIZE_IN_BYTES_KEY, overheadSize.toString)
 
-    // FIXME: The following is a workaround. Remove once the causes are fixed.
-    conf.set(GlutenConfig.GLUTEN_OVERHEAD_SIZE_IN_BYTES_KEY, Long.MaxValue.toString)
-    logWarning(
-      "Setting overhead memory that Gluten can use to UNLIMITED. This is currently a" +
-        " temporary solution to avoid OOM by Velox's global memory pools." +
-        " See GLUTEN-6960 for more information.")
-
     // If dynamic off-heap sizing is enabled, the off-heap size is calculated based on the on-heap
     // size. Otherwise, the off-heap size is set to the value specified by the user (if any).
     // Note that this means that we will IGNORE the off-heap size specified by the user if the