[core] Pass null value to file index (apache#3766)

herefree · Jul 19, 2024 · b7e11d7 · b7e11d7
1 parent 375ffb8
commit b7e11d7
Show file tree

Hide file tree

Showing 3 changed files with 19 additions and 5 deletions.
diff --git a/paimon-common/src/main/java/org/apache/paimon/fileindex/FileIndexWriter.java b/paimon-common/src/main/java/org/apache/paimon/fileindex/FileIndexWriter.java
@@ -24,10 +24,8 @@ public abstract class FileIndexWriter {
     private boolean empty = true;
 
     public void writeRecord(Object key) {
-        if (key != null) {
-            empty = false;
-            write(key);
-        }
+        empty = false;
+        write(key);
     }
 
     public abstract void write(Object key);

diff --git a/...on-common/src/main/java/org/apache/paimon/fileindex/bloomfilter/BloomFilterFileIndex.java b/...on-common/src/main/java/org/apache/paimon/fileindex/bloomfilter/BloomFilterFileIndex.java
@@ -92,7 +92,9 @@ public Writer(DataType type, int items, double fpp) {
 
         @Override
         public void write(Object key) {
-            filter.addHash(hashFunction.hash(key));
+            if (key != null) {
+                filter.addHash(hashFunction.hash(key));
+            }
         }
 
         @Override

diff --git a/paimon-core/src/main/java/org/apache/paimon/io/DataFileIndexWriter.java b/paimon-core/src/main/java/org/apache/paimon/io/DataFileIndexWriter.java
@@ -43,8 +43,10 @@
 import java.io.OutputStream;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 
 /** Index file writer for a data file. */
 public final class DataFileIndexWriter implements Closeable {
@@ -303,18 +305,30 @@ public MapFileIndexMaintainer(
         }
 
         public void write(InternalRow row) {
+            if (row.isNullAt(position)) {
+                indexWritersMap.values().forEach(write -> write.writeRecord(null));
+                return;
+            }
             InternalMap internalMap = row.getMap(position);
             InternalArray keyArray = internalMap.keyArray();
             InternalArray valueArray = internalMap.valueArray();
 
+            Set<String> writedKeys = new HashSet<>();
             for (int i = 0; i < keyArray.size(); i++) {
                 String key = keyArray.getString(i).toString();
                 org.apache.paimon.fileindex.FileIndexWriter writer =
                         indexWritersMap.getOrDefault(key, null);
                 if (writer != null) {
+                    writedKeys.add(key);
                     writer.writeRecord(valueElementGetter.getElementOrNull(valueArray, i));
                 }
             }
+
+            for (Map.Entry<String, FileIndexWriter> writerEntry : indexWritersMap.entrySet()) {
+                if (!writedKeys.contains(writerEntry.getKey())) {
+                    writerEntry.getValue().writeRecord(null);
+                }
+            }
         }
 
         public void add(String nestedKey, Options nestedOptions) {