[core] Make default of 'lookup.local-file-type' to sort (#4622)

apache · Dec 2, 2024 · 3c82082 · 3c82082
1 parent 6fb887f
commit 3c82082
Show file tree

Hide file tree

Showing 5 changed files with 11 additions and 4 deletions.
diff --git a/docs/layouts/shortcodes/generated/core_configuration.html b/docs/layouts/shortcodes/generated/core_configuration.html
@@ -443,7 +443,7 @@
         </tr>
         <tr>
             <td><h5>lookup.local-file-type</h5></td>
-            <td style="word-wrap: break-word;">hash</td>
+            <td style="word-wrap: break-word;">sort</td>
             <td><p>Enum</p></td>
             <td>The local file type for lookup.<br /><br />Possible values:<ul><li>"sort": Construct a sorted file for lookup.</li><li>"hash": Construct a hash file for lookup.</li></ul></td>
         </tr>

diff --git a/...-benchmarks/src/test/java/org/apache/paimon/benchmark/lookup/AbstractLookupBenchmark.java b/...-benchmarks/src/test/java/org/apache/paimon/benchmark/lookup/AbstractLookupBenchmark.java
@@ -102,7 +102,10 @@ protected Pair<String, LookupStoreFactory.Context> writeData(
                         new CacheManager(MemorySize.ofMebiBytes(10)),
                         keySerializer.createSliceComparator());
 
-        File file = new File(tempDir.toFile(), UUID.randomUUID().toString());
+        String name =
+                String.format(
+                        "%s-%s-%s", options.lookupLocalFileType(), valueLength, bloomFilterEnabled);
+        File file = new File(tempDir.toFile(), UUID.randomUUID() + "-" + name);
         LookupStoreWriter writer = factory.createWriter(file, createBloomFiler(bloomFilterEnabled));
         int i = 0;
         for (byte[] input : inputs) {

diff --git a/...ro-benchmarks/src/test/java/org/apache/paimon/benchmark/lookup/LookupReaderBenchmark.java b/...ro-benchmarks/src/test/java/org/apache/paimon/benchmark/lookup/LookupReaderBenchmark.java
@@ -128,7 +128,7 @@ private void readData(
         LookupStoreFactory factory =
                 LookupStoreFactory.create(
                         options,
-                        new CacheManager(MemorySize.ofMebiBytes(10)),
+                        new CacheManager(MemorySize.ofMebiBytes(20), 0.5),
                         new RowCompactedSerializer(RowType.of(new IntType()))
                                 .createSliceComparator());
 

diff --git a/paimon-common/src/main/java/org/apache/paimon/CoreOptions.java b/paimon-common/src/main/java/org/apache/paimon/CoreOptions.java
@@ -891,7 +891,7 @@ public class CoreOptions implements Serializable {
     public static final ConfigOption<LookupLocalFileType> LOOKUP_LOCAL_FILE_TYPE =
             key("lookup.local-file-type")
                     .enumType(LookupLocalFileType.class)
-                    .defaultValue(LookupLocalFileType.HASH)
+                    .defaultValue(LookupLocalFileType.SORT)
                     .withDescription("The local file type for lookup.");
 
     public static final ConfigOption<Float> LOOKUP_HASH_LOAD_FACTOR =

diff --git a/paimon-common/src/main/java/org/apache/paimon/io/cache/CacheBuilder.java b/paimon-common/src/main/java/org/apache/paimon/io/cache/CacheBuilder.java
@@ -72,6 +72,10 @@ public Cache build() {
                     org.apache.paimon.shade.guava30.com.google.common.cache.CacheBuilder
                             .newBuilder()
                             .weigher(CacheBuilder::weigh)
+                            // The concurrency level determines the number of segment caches in
+                            // Guava,limiting the maximum block entries held in cache. Since we do
+                            // not access this cache concurrently, it is set to 1.
+                            .concurrencyLevel(1)
                             .maximumWeight(memorySize.getBytes())
                             .removalListener(this::onRemoval)
                             .build());