[core] Increase write-max-writers-to-spill and logging

apache · Aug 5, 2024 · 68ed999 · 68ed999
1 parent da4fbb9
commit 68ed999
Show file tree

Hide file tree

Showing 3 changed files with 10 additions and 2 deletions.
diff --git a/docs/layouts/shortcodes/generated/core_configuration.html b/docs/layouts/shortcodes/generated/core_configuration.html
@@ -854,7 +854,7 @@
         </tr>
         <tr>
             <td><h5>write-max-writers-to-spill</h5></td>
-            <td style="word-wrap: break-word;">5</td>
+            <td style="word-wrap: break-word;">10</td>
             <td>Integer</td>
             <td>When in batch append inserting, if the writer number is greater than this option, we open the buffer cache and spill function to avoid out-of-memory. </td>
         </tr>

diff --git a/paimon-common/src/main/java/org/apache/paimon/CoreOptions.java b/paimon-common/src/main/java/org/apache/paimon/CoreOptions.java
@@ -410,7 +410,7 @@ public class CoreOptions implements Serializable {
     public static final ConfigOption<Integer> WRITE_MAX_WRITERS_TO_SPILL =
             key("write-max-writers-to-spill")
                     .intType()
-                    .defaultValue(5)
+                    .defaultValue(10)
                     .withDescription(
                             "When in batch append inserting, if the writer number is greater than this option, we open the buffer cache and spill function to avoid out-of-memory. ");
 

diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/AppendOnlyFileStoreWrite.java b/paimon-core/src/main/java/org/apache/paimon/operation/AppendOnlyFileStoreWrite.java
@@ -47,6 +47,9 @@
 import org.apache.paimon.utils.SnapshotManager;
 import org.apache.paimon.utils.StatsCollectorFactories;
 
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 import javax.annotation.Nullable;
 
 import java.io.IOException;
@@ -58,6 +61,8 @@
 /** {@link FileStoreWrite} for {@link AppendOnlyFileStore}. */
 public class AppendOnlyFileStoreWrite extends MemoryFileStoreWrite<InternalRow> {
 
+    private static final Logger LOG = LoggerFactory.getLogger(AppendOnlyFileStoreWrite.class);
+
     private final FileIO fileIO;
     private final RawFileSplitRead read;
     private final long schemaId;
@@ -228,6 +233,9 @@ protected void forceBufferSpill() throws Exception {
             return;
         }
         forceBufferSpill = true;
+        LOG.info(
+                "Force buffer spill for append-only file store write, writer number is: {}",
+                writers.size());
         for (Map<Integer, WriterContainer<InternalRow>> bucketWriters : writers.values()) {
             for (WriterContainer<InternalRow> writerContainer : bucketWriters.values()) {
                 ((AppendOnlyWriter) writerContainer.writer).toBufferedWriter();