[core] Increase write-max-writers-to-spill and logging (#3897)

apache · Aug 5, 2024 · fdaff8c · fdaff8c
1 parent eb8b113
commit fdaff8c
Show file tree

Hide file tree

Showing 4 changed files with 14 additions and 2 deletions.
diff --git a/docs/layouts/shortcodes/generated/core_configuration.html b/docs/layouts/shortcodes/generated/core_configuration.html
@@ -854,7 +854,7 @@
         </tr>
         <tr>
             <td><h5>write-max-writers-to-spill</h5></td>
-            <td style="word-wrap: break-word;">5</td>
+            <td style="word-wrap: break-word;">10</td>
             <td>Integer</td>
             <td>When in batch append inserting, if the writer number is greater than this option, we open the buffer cache and spill function to avoid out-of-memory. </td>
         </tr>

diff --git a/paimon-common/src/main/java/org/apache/paimon/CoreOptions.java b/paimon-common/src/main/java/org/apache/paimon/CoreOptions.java
@@ -410,7 +410,7 @@ public class CoreOptions implements Serializable {
     public static final ConfigOption<Integer> WRITE_MAX_WRITERS_TO_SPILL =
             key("write-max-writers-to-spill")
                     .intType()
-                    .defaultValue(5)
+                    .defaultValue(10)
                     .withDescription(
                             "When in batch append inserting, if the writer number is greater than this option, we open the buffer cache and spill function to avoid out-of-memory. ");
 

diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/AppendOnlyFileStoreWrite.java b/paimon-core/src/main/java/org/apache/paimon/operation/AppendOnlyFileStoreWrite.java
@@ -47,6 +47,9 @@
 import org.apache.paimon.utils.SnapshotManager;
 import org.apache.paimon.utils.StatsCollectorFactories;
 
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 import javax.annotation.Nullable;
 
 import java.io.IOException;
@@ -58,6 +61,8 @@
 /** {@link FileStoreWrite} for {@link AppendOnlyFileStore}. */
 public class AppendOnlyFileStoreWrite extends MemoryFileStoreWrite<InternalRow> {
 
+    private static final Logger LOG = LoggerFactory.getLogger(AppendOnlyFileStoreWrite.class);
+
     private final FileIO fileIO;
     private final RawFileSplitRead read;
     private final long schemaId;
@@ -228,6 +233,9 @@ protected void forceBufferSpill() throws Exception {
             return;
         }
         forceBufferSpill = true;
+        LOG.info(
+                "Force buffer spill for append-only file store write, writer number is: {}",
+                writers.size());
         for (Map<Integer, WriterContainer<InternalRow>> bucketWriters : writers.values()) {
             for (WriterContainer<InternalRow> writerContainer : bucketWriters.values()) {
                 ((AppendOnlyWriter) writerContainer.writer).toBufferedWriter();

diff --git a/paimon-core/src/test/java/org/apache/paimon/operation/AppendOnlyFileStoreWriteTest.java b/paimon-core/src/test/java/org/apache/paimon/operation/AppendOnlyFileStoreWriteTest.java
@@ -44,10 +44,13 @@
 import org.junit.jupiter.api.io.TempDir;
 
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.Random;
 
+import static org.apache.paimon.CoreOptions.WRITE_MAX_WRITERS_TO_SPILL;
+
 /** Tests for {@link AppendOnlyFileStoreWrite}. */
 public class AppendOnlyFileStoreWriteTest {
 
@@ -58,6 +61,7 @@ public class AppendOnlyFileStoreWriteTest {
     @Test
     public void testWritesInBatch() throws Exception {
         FileStoreTable table = createFileStoreTable();
+        table = table.copy(Collections.singletonMap(WRITE_MAX_WRITERS_TO_SPILL.key(), "5"));
 
         AppendOnlyFileStoreWrite write = (AppendOnlyFileStoreWrite) table.store().newWrite("ss");
         write.withIOManager(IOManager.create(tempDir.toString()));