[opt](inverted index) Add null document interface to optimize empty s…

…tring indexing (apache#28661)
seawinde · Dec 20, 2023 · 3296487 · 3296487
1 parent afd5512
commit 3296487
Show file tree

Hide file tree

Showing 2 changed files with 17 additions and 4 deletions.
diff --git a/be/src/clucene b/be/src/clucene
diff --git a/be/src/olap/rowset/segment_v2/inverted_index_writer.cpp b/be/src/olap/rowset/segment_v2/inverted_index_writer.cpp
@@ -230,6 +230,17 @@ class InvertedIndexColumnWriterImpl : public InvertedIndexColumnWriter {
         return Status::OK();
     }
 
+    Status add_null_document() {
+        try {
+            _index_writer->addNullDocument(_doc.get());
+        } catch (const CLuceneError& e) {
+            _dir->deleteDirectory();
+            return Status::Error<ErrorCode::INVERTED_INDEX_CLUCENE_ERROR>(
+                    "CLuceneError add_null_document: {}", e.what());
+        }
+        return Status::OK();
+    }
+
     Status add_nulls(uint32_t count) override {
         _null_bitmap.addRange(_rid, _rid + count);
         _rid += count;
@@ -242,7 +253,7 @@ class InvertedIndexColumnWriterImpl : public InvertedIndexColumnWriter {
 
             for (int i = 0; i < count; ++i) {
                 new_fulltext_field(empty_value.c_str(), 0);
-                RETURN_IF_ERROR(add_document());
+                RETURN_IF_ERROR(add_null_document());
             }
         }
         return Status::OK();
@@ -292,10 +303,11 @@ class InvertedIndexColumnWriterImpl : public InvertedIndexColumnWriter {
                     VLOG_DEBUG << "fulltext index value length can be at most 256, but got "
                                << "value length:" << v->get_size() << ", ignore this value";
                     new_fulltext_field(empty_value.c_str(), 0);
+                    RETURN_IF_ERROR(add_null_document());
                 } else {
                     new_fulltext_field(v->get_data(), v->get_size());
+                    RETURN_IF_ERROR(add_document());
                 }
-                RETURN_IF_ERROR(add_document());
                 ++v;
                 _rid++;
             }
@@ -341,11 +353,12 @@ class InvertedIndexColumnWriterImpl : public InvertedIndexColumnWriter {
                     VLOG_DEBUG << "fulltext index value length can be at most 256, but got "
                                << "value length:" << value.length() << ", ignore this value";
                     new_fulltext_field(empty_value.c_str(), 0);
+                    RETURN_IF_ERROR(add_null_document());
                 } else {
                     new_fulltext_field(value.c_str(), value.length());
+                    RETURN_IF_ERROR(add_document());
                 }
                 _rid++;
-                RETURN_IF_ERROR(add_document());
             }
         } else if constexpr (field_is_numeric_type(field_type)) {
             for (int i = 0; i < count; ++i) {
+5 −0		src/core/CLucene/index/IndexWriter.cpp
+4 −0		src/core/CLucene/index/IndexWriter.h
+9 −0		src/core/CLucene/index/SDocumentWriter.cpp
+4 −0		src/core/CLucene/index/SDocumentWriter.h
+1 −0		src/core/CLucene/index/_DocumentsWriter.h