From 67a77820ed4589bbe77fd306232b6fd459d46241 Mon Sep 17 00:00:00 2001 From: guanshi Date: Mon, 5 Aug 2024 17:24:39 +0800 Subject: [PATCH] [common] fix bitmap index --- .../fileindex/bitmap/BitmapFileIndex.java | 6 ++++- .../bitmapindex/TestBitmapFileIndex.java | 26 +++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/paimon-common/src/main/java/org/apache/paimon/fileindex/bitmap/BitmapFileIndex.java b/paimon-common/src/main/java/org/apache/paimon/fileindex/bitmap/BitmapFileIndex.java index ef2c2b207bdb..22375e763f79 100644 --- a/paimon-common/src/main/java/org/apache/paimon/fileindex/bitmap/BitmapFileIndex.java +++ b/paimon-common/src/main/java/org/apache/paimon/fileindex/bitmap/BitmapFileIndex.java @@ -116,7 +116,11 @@ public byte[] serializedBytes() { // 2.build bitmap file index meta LinkedHashMap bitmapOffsets = new LinkedHashMap<>(); LinkedList serializeBitmaps = new LinkedList<>(); - int[] offsetRef = {nullBitmap.isEmpty() ? 0 : nullBitmapBytes.length}; + int[] offsetRef = { + nullBitmap.isEmpty() || nullBitmap.getCardinality() == 1 + ? 0 + : nullBitmapBytes.length + }; id2bitmap.forEach( (k, v) -> { if (v.getCardinality() == 1) { diff --git a/paimon-common/src/test/java/org/apache/paimon/fileindex/bitmapindex/TestBitmapFileIndex.java b/paimon-common/src/test/java/org/apache/paimon/fileindex/bitmapindex/TestBitmapFileIndex.java index 923cc5c7b731..84e2af304f7d 100644 --- a/paimon-common/src/test/java/org/apache/paimon/fileindex/bitmapindex/TestBitmapFileIndex.java +++ b/paimon-common/src/test/java/org/apache/paimon/fileindex/bitmapindex/TestBitmapFileIndex.java @@ -112,4 +112,30 @@ public void testBitmapIndex2() { (BitmapIndexResultLazy) reader.visitNotIn(fieldRef, Arrays.asList(1, 0)); assert result5.get().equals(RoaringBitmap32.bitmapOf(2)); } + + @Test + public void testBitmapIndex3() { + + IntType intType = new IntType(); + FieldRef fieldRef = new FieldRef(0, "", intType); + BitmapFileIndex bitmapFileIndex = new BitmapFileIndex(intType, null); + FileIndexWriter writer = bitmapFileIndex.createWriter(); + + // test only one null-value + Object[] arr = {1, 2, 1, 2, 1, 3, null}; + + for (Object o : arr) { + writer.write(o); + } + byte[] bytes = writer.serializedBytes(); + ByteArraySeekableStream seekableStream = new ByteArraySeekableStream(bytes); + FileIndexReader reader = bitmapFileIndex.createReader(seekableStream, 0, bytes.length); + + BitmapIndexResultLazy result1 = (BitmapIndexResultLazy) reader.visitEqual(fieldRef, 1); + assert result1.get().equals(RoaringBitmap32.bitmapOf(0, 2, 4)); + + // test read singleton bitmap + BitmapIndexResultLazy result2 = (BitmapIndexResultLazy) reader.visitIsNull(fieldRef); + assert result2.get().equals(RoaringBitmap32.bitmapOf(6)); + } }