[improve](load) add switch for vertical segment writer (apache#26996)

seawinde · Nov 15, 2023 · 598b07d · 598b07d
1 parent dfa6e44
commit 598b07d
Show file tree

Hide file tree

Showing 3 changed files with 17 additions and 4 deletions.
diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp
@@ -351,6 +351,9 @@ DEFINE_Int32(vertical_compaction_max_row_source_memory_mb, "200");
 // In vertical compaction, max dest segment file size
 DEFINE_mInt64(vertical_compaction_max_segment_size, "268435456");
 
+// If enabled, segments will be flushed column by column
+DEFINE_mBool(enable_vertical_segment_writer, "true");
+
 // In ordered data compaction, min segment size for input rowset
 DEFINE_mInt32(ordered_data_compaction_min_segment_size, "10485760");
 

diff --git a/be/src/common/config.h b/be/src/common/config.h
@@ -404,6 +404,9 @@ DECLARE_Int32(vertical_compaction_max_row_source_memory_mb);
 // In vertical compaction, max dest segment file size
 DECLARE_mInt64(vertical_compaction_max_segment_size);
 
+// If enabled, segments will be flushed column by column
+DECLARE_mBool(enable_vertical_segment_writer);
+
 // In ordered data compaction, min segment size for input rowset
 DECLARE_mInt32(ordered_data_compaction_min_segment_size);
 

diff --git a/be/src/olap/rowset/segment_creator.cpp b/be/src/olap/rowset/segment_creator.cpp
@@ -50,11 +50,18 @@ Status SegmentFlusher::flush_single_block(const vectorized::Block* block, int32_
     if (block->rows() == 0) {
         return Status::OK();
     }
-    std::unique_ptr<segment_v2::VerticalSegmentWriter> writer;
     bool no_compression = block->bytes() <= config::segment_compression_threshold_kb * 1024;
-    RETURN_IF_ERROR(_create_segment_writer(writer, segment_id, no_compression, flush_schema));
-    RETURN_IF_ERROR(_add_rows(writer, block, 0, block->rows()));
-    RETURN_IF_ERROR(_flush_segment_writer(writer, flush_size));
+    if (config::enable_vertical_segment_writer) {
+        std::unique_ptr<segment_v2::VerticalSegmentWriter> writer;
+        RETURN_IF_ERROR(_create_segment_writer(writer, segment_id, no_compression, flush_schema));
+        RETURN_IF_ERROR(_add_rows(writer, block, 0, block->rows()));
+        RETURN_IF_ERROR(_flush_segment_writer(writer, flush_size));
+    } else {
+        std::unique_ptr<segment_v2::SegmentWriter> writer;
+        RETURN_IF_ERROR(_create_segment_writer(writer, segment_id, no_compression, flush_schema));
+        RETURN_IF_ERROR(_add_rows(writer, block, 0, block->rows()));
+        RETURN_IF_ERROR(_flush_segment_writer(writer, flush_size));
+    }
     return Status::OK();
 }