[core] modify the default value of "target-file-size" (apache#3721)

askwang · Jul 11, 2024 · 15e6ad4 · 15e6ad4
1 parent afda354
commit 15e6ad4
Show file tree

Hide file tree

Showing 10 changed files with 67 additions and 22 deletions.
diff --git a/docs/layouts/shortcodes/generated/core_configuration.html b/docs/layouts/shortcodes/generated/core_configuration.html
@@ -782,7 +782,7 @@
         </tr>
         <tr>
             <td><h5>target-file-size</h5></td>
-            <td style="word-wrap: break-word;">128 mb</td>
+            <td style="word-wrap: break-word;">(none)</td>
             <td>MemorySize</td>
             <td>Target size of a file.</td>
         </tr>

diff --git a/paimon-common/src/main/java/org/apache/paimon/CoreOptions.java b/paimon-common/src/main/java/org/apache/paimon/CoreOptions.java
@@ -438,7 +438,7 @@ public class CoreOptions implements Serializable {
     public static final ConfigOption<MemorySize> TARGET_FILE_SIZE =
             key("target-file-size")
                     .memoryType()
-                    .defaultValue(MemorySize.ofMebiBytes(128))
+                    .noDefaultValue()
                     .withDescription("Target size of a file.");
 
     public static final ConfigOption<Integer> NUM_SORTED_RUNS_COMPACTION_TRIGGER =
@@ -1567,15 +1567,19 @@ public MemorySize lookupCacheMaxMemory() {
         return options.get(LOOKUP_CACHE_MAX_MEMORY_SIZE);
     }
 
-    public long targetFileSize() {
-        return options.get(TARGET_FILE_SIZE).getBytes();
+    public long targetFileSize(TableType tableType) {
+        MemorySize memorySize = options.get(TARGET_FILE_SIZE);
+        if (memorySize == null) {
+            memorySize = tableType.getDefaultMemorySize();
+        }
+        return memorySize.getBytes();
     }
 
-    public long compactionFileSize() {
+    public long compactionFileSize(TableType tableType) {
         // file size to join the compaction, we don't process on middle file size to avoid
         // compact a same file twice (the compression is not calculate so accurately. the output
         // file maybe be less than target file generated by rolling file write).
-        return options.get(TARGET_FILE_SIZE).getBytes() / 10 * 7;
+        return targetFileSize(tableType) / 10 * 7;
     }
 
     public int numSortedRunCompactionTrigger() {
@@ -2549,4 +2553,36 @@ public InlineElement getDescription() {
             return text(description);
         }
     }
+
+    /** Specifies the table type. */
+    public enum TableType implements DescribedEnum {
+        PRIMARY_KEY_TABLE(
+                "primaryKeyTable", MemorySize.ofMebiBytes(128), "The table of primaryKey."),
+        APPEND_ONLY_TABLE(
+                "appendOnlyTable", MemorySize.ofMebiBytes(256), "The table of appendOnly.");
+
+        private final String name;
+        private final MemorySize defaultMemorySize;
+        private final String description;
+
+        TableType(String name, MemorySize defaultMemorySize, String description) {
+            this.name = name;
+            this.defaultMemorySize = defaultMemorySize;
+            this.description = description;
+        }
+
+        @Override
+        public String toString() {
+            return name;
+        }
+
+        @Override
+        public InlineElement getDescription() {
+            return text(description);
+        }
+
+        public MemorySize getDefaultMemorySize() {
+            return defaultMemorySize;
+        }
+    }
 }
diff --git a/paimon-core/src/main/java/org/apache/paimon/append/AppendOnlyTableCompactionCoordinator.java b/paimon-core/src/main/java/org/apache/paimon/append/AppendOnlyTableCompactionCoordinator.java
@@ -92,8 +92,9 @@ public AppendOnlyTableCompactionCoordinator(
         }
         this.streamingMode = isStreaming;
         CoreOptions coreOptions = table.coreOptions();
-        this.targetFileSize = coreOptions.targetFileSize();
-        this.compactionFileSize = coreOptions.compactionFileSize();
+        this.targetFileSize = coreOptions.targetFileSize(CoreOptions.TableType.APPEND_ONLY_TABLE);
+        this.compactionFileSize =
+                coreOptions.compactionFileSize(CoreOptions.TableType.APPEND_ONLY_TABLE);
         this.minFileNum = coreOptions.compactionMinFileNum();
         this.maxFileNum = coreOptions.compactionMaxFileNum();
     }

diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/AppendOnlyFileStoreWrite.java b/paimon-core/src/main/java/org/apache/paimon/operation/AppendOnlyFileStoreWrite.java
@@ -98,7 +98,7 @@ public AppendOnlyFileStoreWrite(
         this.rowType = rowType;
         this.fileFormat = options.fileFormat();
         this.pathFactory = pathFactory;
-        this.targetFileSize = options.targetFileSize();
+        this.targetFileSize = options.targetFileSize(CoreOptions.TableType.APPEND_ONLY_TABLE);
         this.compactionMinFileNum = options.compactionMinFileNum();
         this.compactionMaxFileNum = options.compactionMaxFileNum();
         this.commitForceCompact = options.commitForceCompact();

diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/KeyValueFileStoreWrite.java b/paimon-core/src/main/java/org/apache/paimon/operation/KeyValueFileStoreWrite.java
@@ -154,7 +154,7 @@ public KeyValueFileStoreWrite(
                         valueType,
                         options.fileFormat(),
                         format2PathFactory,
-                        options.targetFileSize());
+                        options.targetFileSize(CoreOptions.TableType.PRIMARY_KEY_TABLE));
         this.keyComparatorSupplier = keyComparatorSupplier;
         this.valueEqualiserSupplier = valueEqualiserSupplier;
         this.mfFactory = mfFactory;
@@ -243,7 +243,7 @@ private CompactManager createCompactManager(
                     levels,
                     compactStrategy,
                     keyComparator,
-                    options.compactionFileSize(),
+                    options.compactionFileSize(CoreOptions.TableType.PRIMARY_KEY_TABLE),
                     options.numSortedRunStopTrigger(),
                     rewriter,
                     compactionMetrics == null

diff --git a/...core/src/test/java/org/apache/paimon/append/AppendOnlyTableCompactionCoordinatorTest.java b/...core/src/test/java/org/apache/paimon/append/AppendOnlyTableCompactionCoordinatorTest.java
@@ -69,15 +69,26 @@ public void testNoCompactTask() {
     public void testMinSizeCompactTask() {
         List<DataFileMeta> files =
                 generateNewFiles(
-                        100, appendOnlyFileStoreTable.coreOptions().targetFileSize() / 3 + 1);
+                        100,
+                        appendOnlyFileStoreTable
+                                                .coreOptions()
+                                                .targetFileSize(
+                                                        CoreOptions.TableType.APPEND_ONLY_TABLE)
+                                        / 3
+                                + 1);
         assertTasks(files, 100 / 3);
     }
 
     @Test
     public void testFilterMiddleFile() {
         List<DataFileMeta> files =
                 generateNewFiles(
-                        100, appendOnlyFileStoreTable.coreOptions().targetFileSize() / 10 * 8);
+                        100,
+                        appendOnlyFileStoreTable
+                                        .coreOptions()
+                                        .targetFileSize(CoreOptions.TableType.APPEND_ONLY_TABLE)
+                                / 10
+                                * 8);
         assertTasks(files, 0);
     }
 

diff --git a/paimon-core/src/test/java/org/apache/paimon/mergetree/ChangelogMergeTreeRewriterTest.java b/paimon-core/src/test/java/org/apache/paimon/mergetree/ChangelogMergeTreeRewriterTest.java
@@ -69,7 +69,6 @@
 import java.util.UUID;
 import java.util.stream.Collectors;
 
-import static org.apache.paimon.CoreOptions.TARGET_FILE_SIZE;
 import static org.apache.paimon.utils.FileStorePathFactoryTest.createNonPartFactory;
 import static org.junit.jupiter.api.Assertions.fail;
 
@@ -201,7 +200,7 @@ private KeyValueFileWriterFactory createWriterFactory(
                         valueType,
                         new FlushingFileFormat(formatIdentifier),
                         Collections.singletonMap(formatIdentifier, createNonPartFactory(path)),
-                        TARGET_FILE_SIZE.defaultValue().getBytes())
+                        CoreOptions.TableType.PRIMARY_KEY_TABLE.getDefaultMemorySize().getBytes())
                 .build(BinaryRow.EMPTY_ROW, 0, new CoreOptions(new Options()));
     }
 

diff --git a/paimon-core/src/test/java/org/apache/paimon/mergetree/ContainsLevelsTest.java b/paimon-core/src/test/java/org/apache/paimon/mergetree/ContainsLevelsTest.java
@@ -62,7 +62,6 @@
 import java.util.Map;
 import java.util.UUID;
 
-import static org.apache.paimon.CoreOptions.TARGET_FILE_SIZE;
 import static org.apache.paimon.io.DataFileTestUtils.row;
 import static org.apache.paimon.utils.FileStorePathFactoryTest.createNonPartFactory;
 import static org.assertj.core.api.Assertions.assertThat;
@@ -227,7 +226,7 @@ private KeyValueFileWriterFactory createWriterFactory() {
                         rowType,
                         new FlushingFileFormat(identifier),
                         pathFactoryMap,
-                        TARGET_FILE_SIZE.defaultValue().getBytes())
+                        CoreOptions.TableType.PRIMARY_KEY_TABLE.getDefaultMemorySize().getBytes())
                 .build(BinaryRow.EMPTY_ROW, 0, new CoreOptions(new Options()));
     }
 

diff --git a/paimon-core/src/test/java/org/apache/paimon/mergetree/LookupLevelsTest.java b/paimon-core/src/test/java/org/apache/paimon/mergetree/LookupLevelsTest.java
@@ -62,7 +62,6 @@
 import java.util.Map;
 import java.util.UUID;
 
-import static org.apache.paimon.CoreOptions.TARGET_FILE_SIZE;
 import static org.apache.paimon.KeyValue.UNKNOWN_SEQUENCE;
 import static org.apache.paimon.io.DataFileTestUtils.row;
 import static org.apache.paimon.utils.FileStorePathFactoryTest.createNonPartFactory;
@@ -307,7 +306,7 @@ private KeyValueFileWriterFactory createWriterFactory() {
                         rowType,
                         new FlushingFileFormat(identifier),
                         pathFactoryMap,
-                        TARGET_FILE_SIZE.defaultValue().getBytes())
+                        CoreOptions.TableType.PRIMARY_KEY_TABLE.getDefaultMemorySize().getBytes())
                 .build(BinaryRow.EMPTY_ROW, 0, new CoreOptions(new Options()));
     }
 

diff --git a/paimon-core/src/test/java/org/apache/paimon/mergetree/MergeTreeTestBase.java b/paimon-core/src/test/java/org/apache/paimon/mergetree/MergeTreeTestBase.java
@@ -192,7 +192,7 @@ public List<DataField> valueFields(TableSchema schema) {
                         valueType,
                         flushingAvro,
                         pathFactoryMap,
-                        this.options.targetFileSize());
+                        this.options.targetFileSize(CoreOptions.TableType.PRIMARY_KEY_TABLE));
         writerFactory = writerFactoryBuilder.build(BinaryRow.EMPTY_ROW, 0, this.options);
         compactWriterFactory = writerFactoryBuilder.build(BinaryRow.EMPTY_ROW, 0, this.options);
         writer = createMergeTreeWriter(Collections.emptyList());
@@ -289,7 +289,7 @@ public void testPrepareCommitRecycleReference() throws Exception {
                                 options.sortedRunSizeRatio(),
                                 options.numSortedRunCompactionTrigger()),
                         comparator,
-                        options.targetFileSize(),
+                        options.targetFileSize(CoreOptions.TableType.PRIMARY_KEY_TABLE),
                         options.numSortedRunStopTrigger(),
                         new TestRewriter());
         writer = createMergeTreeWriter(dataFileMetas, mockFailResultCompactionManager);
@@ -542,7 +542,7 @@ private MergeTreeCompactManager createCompactManager(
                 new Levels(comparator, files, options.numLevels()),
                 strategy,
                 comparator,
-                options.compactionFileSize(),
+                options.compactionFileSize(CoreOptions.TableType.PRIMARY_KEY_TABLE),
                 options.numSortedRunStopTrigger(),
                 new TestRewriter(),
                 null,