diff --git a/docs/layouts/shortcodes/generated/core_configuration.html b/docs/layouts/shortcodes/generated/core_configuration.html index 446931acacda..bc6f8b5c4efc 100644 --- a/docs/layouts/shortcodes/generated/core_configuration.html +++ b/docs/layouts/shortcodes/generated/core_configuration.html @@ -854,7 +854,7 @@
write-max-writers-to-spill
- 5 + 10 Integer When in batch append inserting, if the writer number is greater than this option, we open the buffer cache and spill function to avoid out-of-memory. diff --git a/paimon-common/src/main/java/org/apache/paimon/CoreOptions.java b/paimon-common/src/main/java/org/apache/paimon/CoreOptions.java index 17bff3653229..156915e2365d 100644 --- a/paimon-common/src/main/java/org/apache/paimon/CoreOptions.java +++ b/paimon-common/src/main/java/org/apache/paimon/CoreOptions.java @@ -410,7 +410,7 @@ public class CoreOptions implements Serializable { public static final ConfigOption WRITE_MAX_WRITERS_TO_SPILL = key("write-max-writers-to-spill") .intType() - .defaultValue(5) + .defaultValue(10) .withDescription( "When in batch append inserting, if the writer number is greater than this option, we open the buffer cache and spill function to avoid out-of-memory. "); diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/AppendOnlyFileStoreWrite.java b/paimon-core/src/main/java/org/apache/paimon/operation/AppendOnlyFileStoreWrite.java index 12757df9b272..94744887bbc7 100644 --- a/paimon-core/src/main/java/org/apache/paimon/operation/AppendOnlyFileStoreWrite.java +++ b/paimon-core/src/main/java/org/apache/paimon/operation/AppendOnlyFileStoreWrite.java @@ -47,6 +47,9 @@ import org.apache.paimon.utils.SnapshotManager; import org.apache.paimon.utils.StatsCollectorFactories; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import javax.annotation.Nullable; import java.io.IOException; @@ -58,6 +61,8 @@ /** {@link FileStoreWrite} for {@link AppendOnlyFileStore}. */ public class AppendOnlyFileStoreWrite extends MemoryFileStoreWrite { + private static final Logger LOG = LoggerFactory.getLogger(AppendOnlyFileStoreWrite.class); + private final FileIO fileIO; private final RawFileSplitRead read; private final long schemaId; @@ -228,6 +233,9 @@ protected void forceBufferSpill() throws Exception { return; } forceBufferSpill = true; + LOG.info( + "Force buffer spill for append-only file store write, writer number is: {}", + writers.size()); for (Map> bucketWriters : writers.values()) { for (WriterContainer writerContainer : bucketWriters.values()) { ((AppendOnlyWriter) writerContainer.writer).toBufferedWriter(); diff --git a/paimon-core/src/test/java/org/apache/paimon/operation/AppendOnlyFileStoreWriteTest.java b/paimon-core/src/test/java/org/apache/paimon/operation/AppendOnlyFileStoreWriteTest.java index 4d95a91a936b..6384729b8184 100644 --- a/paimon-core/src/test/java/org/apache/paimon/operation/AppendOnlyFileStoreWriteTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/operation/AppendOnlyFileStoreWriteTest.java @@ -44,10 +44,13 @@ import org.junit.jupiter.api.io.TempDir; import java.util.Arrays; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Random; +import static org.apache.paimon.CoreOptions.WRITE_MAX_WRITERS_TO_SPILL; + /** Tests for {@link AppendOnlyFileStoreWrite}. */ public class AppendOnlyFileStoreWriteTest { @@ -58,6 +61,7 @@ public class AppendOnlyFileStoreWriteTest { @Test public void testWritesInBatch() throws Exception { FileStoreTable table = createFileStoreTable(); + table = table.copy(Collections.singletonMap(WRITE_MAX_WRITERS_TO_SPILL.key(), "5")); AppendOnlyFileStoreWrite write = (AppendOnlyFileStoreWrite) table.store().newWrite("ss"); write.withIOManager(IOManager.create(tempDir.toString()));