Merge branch 'master' into alter_catalog_rename

.asf.yaml

@@ -63,6 +63,7 @@ github:
           - COMPILE (DORIS_COMPILE)
           - Need_2_Approval
           - Cloud UT (Doris Cloud UT)
+          - performance (Doris Performance)
 
       required_pull_request_reviews:
         dismiss_stale_reviews: true

README.md

@@ -59,12 +59,9 @@ Apache Doris is an easy-to-use, high-performance and real-time analytical databa
 
 All this makes Apache Doris an ideal tool for scenarios including report analysis, ad-hoc query, unified data warehouse, and data lake query acceleration. On Apache Doris, users can build various applications, such as user behavior analysis, AB test platform, log retrieval analysis, user portrait analysis, and order analysis.
 
-🎉 Version 2.1.4 released now. Check out the 🔗[Release Notes](https://doris.apache.org/docs/releasenotes/release-2.1.4) here. The 2.1 verison delivers exceptional performance with 100% higher out-of-the-box queries proven by TPC-DS 1TB tests, enhanced data lake analytics that are 4-6 times speedier than Trino and Spark, solid support for semi-structured data analysis with new Variant types and suite of analytical functions, asynchronous materialized views for query acceleration, optimized real-time writing at scale, and better workload management with stability and runtime SQL resource tracking.
+🎉 Check out the 🔗[All releases](https://doris.apache.org/docs/releasenotes/all-release), where you'll find a chronological summary of Apache Doris versions released over the past year.
 
-
-🎉 Version 2.0.12 is now released ! This fully evolved and stable release is ready for all users to upgrade. Check out the 🔗[Release Notes](https://doris.apache.org/docs/2.0/releasenotes/release-2.0.12) here. 
-
-👀 Have a look at the 🔗[Official Website](https://doris.apache.org/) for a comprehensive list of Apache Doris's core features, blogs and user cases.
+👀 Explore the 🔗[Official Website](https://doris.apache.org/) to discover Apache Doris's core features, blogs, and user cases in detail.
 
 ## 📈 Usage Scenarios
 

be/src/cloud/cloud_base_compaction.cpp

@@ -268,8 +268,9 @@ Status CloudBaseCompaction::execute_compact() {
                      << ", output_version=" << _output_version;
         return res;
     }
-    LOG_INFO("finish CloudBaseCompaction, tablet_id={}, cost={}ms", _tablet->tablet_id(),
-             duration_cast<milliseconds>(steady_clock::now() - start).count())
+    LOG_INFO("finish CloudBaseCompaction, tablet_id={}, cost={}ms range=[{}-{}]",
+             _tablet->tablet_id(), duration_cast<milliseconds>(steady_clock::now() - start).count(),
+             _input_rowsets.front()->start_version(), _input_rowsets.back()->end_version())
             .tag("job_id", _uuid)
             .tag("input_rowsets", _input_rowsets.size())
             .tag("input_rows", _input_row_num)
@@ -343,7 +344,7 @@ Status CloudBaseCompaction::modify_rowsets() {
                 .tag("input_rowsets", _input_rowsets.size())
                 .tag("input_rows", _input_row_num)
                 .tag("input_segments", _input_segments)
-                .tag("update_bitmap_size", output_rowset_delete_bitmap->delete_bitmap.size());
+                .tag("num_output_delete_bitmap", output_rowset_delete_bitmap->delete_bitmap.size());
         compaction_job->set_delete_bitmap_lock_initiator(initiator);
     }
 

be/src/cloud/cloud_cumulative_compaction.cpp

@@ -204,8 +204,9 @@ Status CloudCumulativeCompaction::execute_compact() {
                      << ", output_version=" << _output_version;
         return res;
     }
-    LOG_INFO("finish CloudCumulativeCompaction, tablet_id={}, cost={}ms", _tablet->tablet_id(),
-             duration_cast<milliseconds>(steady_clock::now() - start).count())
+    LOG_INFO("finish CloudCumulativeCompaction, tablet_id={}, cost={}ms, range=[{}-{}]",
+             _tablet->tablet_id(), duration_cast<milliseconds>(steady_clock::now() - start).count(),
+             _input_rowsets.front()->start_version(), _input_rowsets.back()->end_version())
             .tag("job_id", _uuid)
             .tag("input_rowsets", _input_rowsets.size())
             .tag("input_rows", _input_row_num)
@@ -299,7 +300,8 @@ Status CloudCumulativeCompaction::modify_rowsets() {
                 .tag("input_rowsets", _input_rowsets.size())
                 .tag("input_rows", _input_row_num)
                 .tag("input_segments", _input_segments)
-                .tag("update_bitmap_size", output_rowset_delete_bitmap->delete_bitmap.size());
+                .tag("number_output_delete_bitmap",
+                     output_rowset_delete_bitmap->delete_bitmap.size());
         compaction_job->set_delete_bitmap_lock_initiator(initiator);
     }
 

be/src/cloud/cloud_storage_engine.h

@@ -75,7 +75,7 @@ class CloudStorageEngine final : public BaseStorageEngine {
     void _check_file_cache_ttl_block_valid();
 
     std::optional<StorageResource> get_storage_resource(const std::string& vault_id) {
-        LOG(INFO) << "Getting storage resource for vault_id: " << vault_id;
+        VLOG_DEBUG << "Getting storage resource for vault_id: " << vault_id;
 
         bool synced = false;
         do {

be/src/cloud/cloud_tablet.cpp

@@ -33,6 +33,7 @@
 #include "cloud/cloud_meta_mgr.h"
 #include "cloud/cloud_storage_engine.h"
 #include "cloud/cloud_tablet_mgr.h"
+#include "common/logging.h"
 #include "io/cache/block_file_cache_downloader.h"
 #include "io/cache/block_file_cache_factory.h"
 #include "olap/cumulative_compaction_time_series_policy.h"
@@ -408,6 +409,9 @@ uint64_t CloudTablet::delete_expired_stale_rowsets() {
                 auto rs_it = _stale_rs_version_map.find(v_ts->version());
                 if (rs_it != _stale_rs_version_map.end()) {
                     expired_rowsets.push_back(rs_it->second);
+                    LOG(INFO) << "erase stale rowset, tablet_id=" << tablet_id()
+                              << " rowset_id=" << rs_it->second->rowset_id().to_string()
+                              << " version=" << rs_it->first.to_string();
                     _stale_rs_version_map.erase(rs_it);
                 } else {
                     LOG(WARNING) << "cannot find stale rowset " << v_ts->version() << " in tablet "
@@ -657,11 +661,14 @@ void CloudTablet::get_compaction_status(std::string* json_result) {
 }
 
 void CloudTablet::set_cumulative_layer_point(int64_t new_point) {
+    if (new_point == Tablet::K_INVALID_CUMULATIVE_POINT || new_point >= _cumulative_point) {
+        _cumulative_point = new_point;
+        return;
+    }
     // cumulative point should only be reset to -1, or be increased
-    CHECK(new_point == Tablet::K_INVALID_CUMULATIVE_POINT || new_point >= _cumulative_point)
-            << "Unexpected cumulative point: " << new_point
-            << ", origin: " << _cumulative_point.load();
-    _cumulative_point = new_point;
+    // FIXME: could happen in currently unresolved race conditions
+    LOG(WARNING) << "Unexpected cumulative point: " << new_point
+                 << ", origin: " << _cumulative_point.load();
 }
 
 std::vector<RowsetSharedPtr> CloudTablet::pick_candidate_rowsets_to_base_compaction() {

be/src/exec/table_connector.cpp

@@ -118,16 +118,17 @@ Status TableConnector::convert_column_data(const vectorized::ColumnPtr& column_p
             fmt::format_to(_insert_stmt_buffer, "\"{}\"", str);
         }
     };
-    const vectorized::IColumn* column = column_ptr;
+    const vectorized::IColumn* column = column_ptr.get();
     if (type_ptr->is_nullable()) {
-        auto nullable_column = assert_cast<const vectorized::ColumnNullable*>(column_ptr.get());
+        const auto* nullable_column =
+                assert_cast<const vectorized::ColumnNullable*>(column_ptr.get());
         if (nullable_column->is_null_at(row)) {
             fmt::format_to(_insert_stmt_buffer, "{}", "NULL");
             return Status::OK();
         }
         column = nullable_column->get_nested_column_ptr().get();
     } else {
-        column = column_ptr;
+        column = column_ptr.get();
     }
     auto [item, size] = column->get_data_at(row);
     switch (type.type) {

be/src/http/http_channel.cpp

@@ -123,7 +123,8 @@ void HttpChannel::send_files(HttpRequest* request, const std::string& root_dir,
         VLOG_DEBUG << "http channel send file " << file_path << ", size: " << file_size;
 
         evbuffer_add_printf(evb.get(), "File-Name: %s\r\n", file.c_str());
-        evbuffer_add_printf(evb.get(), "Content-Length: %ld\r\n", file_size);
+        evbuffer_add_printf(evb.get(), "Content-Length: %" PRIi64 "\r\n", file_size);
+
         evbuffer_add_printf(evb.get(), "\r\n");
         if (file_size > 0) {
             evbuffer_add_file(evb.get(), fd, 0, file_size);

be/src/olap/base_tablet.cpp

@@ -28,6 +28,7 @@
 #include "common/status.h"
 #include "olap/calc_delete_bitmap_executor.h"
 #include "olap/delete_bitmap_calculator.h"
+#include "olap/iterators.h"
 #include "olap/memtable.h"
 #include "olap/partial_update_info.h"
 #include "olap/primary_key_index.h"
@@ -81,7 +82,9 @@ Status _get_segment_column_iterator(const BetaRowsetSharedPtr& rowset, uint32_t
                                             rowset->rowset_id().to_string(), segid));
     }
     segment_v2::SegmentSharedPtr segment = *it;
-    RETURN_IF_ERROR(segment->new_column_iterator(target_column, column_iterator, nullptr));
+    StorageReadOptions opts;
+    opts.stats = stats;
+    RETURN_IF_ERROR(segment->new_column_iterator(target_column, column_iterator, &opts));
     segment_v2::ColumnIteratorOptions opt {
             .use_page_cache = !config::disable_storage_page_cache,
             .file_reader = segment->file_reader().get(),
@@ -496,7 +499,7 @@ Status BaseTablet::lookup_row_key(const Slice& encoded_key, TabletSchema* latest
 
         for (auto id : picked_segments) {
             Status s = segments[id]->lookup_row_key(encoded_key, schema, with_seq_col, with_rowid,
-                                                    &loc, encoded_seq_value, stats);
+                                                    &loc, stats, encoded_seq_value);
             if (s.is<KEY_NOT_FOUND>()) {
                 continue;
             }
@@ -612,7 +615,7 @@ Status BaseTablet::calc_segment_delete_bitmap(RowsetSharedPtr rowset,
     vectorized::Block ordered_block = block.clone_empty();
     uint32_t pos = 0;
 
-    RETURN_IF_ERROR(seg->load_pk_index_and_bf()); // We need index blocks to iterate
+    RETURN_IF_ERROR(seg->load_pk_index_and_bf(nullptr)); // We need index blocks to iterate
     const auto* pk_idx = seg->get_primary_key_index();
     int total = pk_idx->num_rows();
     uint32_t row_id = 0;
@@ -626,7 +629,7 @@ Status BaseTablet::calc_segment_delete_bitmap(RowsetSharedPtr rowset,
     std::vector<std::unique_ptr<SegmentCacheHandle>> segment_caches(specified_rowsets.size());
     while (remaining > 0) {
         std::unique_ptr<segment_v2::IndexedColumnIterator> iter;
-        RETURN_IF_ERROR(pk_idx->new_iterator(&iter));
+        RETURN_IF_ERROR(pk_idx->new_iterator(&iter, nullptr));
 
         size_t num_to_read = std::min(batch_size, remaining);
         auto index_type = vectorized::DataTypeFactory::instance().create_data_type(

be/src/olap/delete_bitmap_calculator.cpp

@@ -145,12 +145,11 @@ Status MergeIndexDeleteBitmapCalculator::init(RowsetId rowset_id,
                 MergeIndexDeleteBitmapCalculatorContext::Comparator(seq_col_length, _rowid_length);
         _contexts.reserve(segments.size());
         _heap = std::make_unique<Heap>(_comparator);
-
         for (auto& segment : segments) {
-            RETURN_IF_ERROR(segment->load_index());
+            RETURN_IF_ERROR(segment->load_index(nullptr));
             auto pk_idx = segment->get_primary_key_index();
             std::unique_ptr<segment_v2::IndexedColumnIterator> index;
-            RETURN_IF_ERROR(pk_idx->new_iterator(&index));
+            RETURN_IF_ERROR(pk_idx->new_iterator(&index, nullptr));
             auto index_type = vectorized::DataTypeFactory::instance().create_data_type(
                     pk_idx->type_info()->type(), 1, 0);
             _contexts.emplace_back(std::move(index), index_type, segment->id(), pk_idx->num_rows());

be/src/olap/lru_cache.cpp

@@ -604,12 +604,12 @@ ShardedLRUCache::ShardedLRUCache(const std::string& name, size_t capacity, LRUCa
     INT_GAUGE_METRIC_REGISTER(_entity, cache_capacity);
     INT_GAUGE_METRIC_REGISTER(_entity, cache_usage);
     INT_GAUGE_METRIC_REGISTER(_entity, cache_element_count);
-    INT_DOUBLE_METRIC_REGISTER(_entity, cache_usage_ratio);
-    INT_ATOMIC_COUNTER_METRIC_REGISTER(_entity, cache_lookup_count);
-    INT_ATOMIC_COUNTER_METRIC_REGISTER(_entity, cache_hit_count);
-    INT_ATOMIC_COUNTER_METRIC_REGISTER(_entity, cache_stampede_count);
-    INT_ATOMIC_COUNTER_METRIC_REGISTER(_entity, cache_miss_count);
-    INT_DOUBLE_METRIC_REGISTER(_entity, cache_hit_ratio);
+    DOUBLE_GAUGE_METRIC_REGISTER(_entity, cache_usage_ratio);
+    INT_COUNTER_METRIC_REGISTER(_entity, cache_lookup_count);
+    INT_COUNTER_METRIC_REGISTER(_entity, cache_hit_count);
+    INT_COUNTER_METRIC_REGISTER(_entity, cache_stampede_count);
+    INT_COUNTER_METRIC_REGISTER(_entity, cache_miss_count);
+    DOUBLE_GAUGE_METRIC_REGISTER(_entity, cache_hit_ratio);
 
     _hit_count_bvar.reset(new bvar::Adder<uint64_t>("doris_cache", _name));
     _hit_count_per_second.reset(new bvar::PerSecond<bvar::Adder<uint64_t>>(

be/src/olap/lru_cache.h

@@ -447,10 +447,10 @@ class ShardedLRUCache : public Cache {
     IntGauge* cache_usage = nullptr;
     IntGauge* cache_element_count = nullptr;
     DoubleGauge* cache_usage_ratio = nullptr;
-    IntAtomicCounter* cache_lookup_count = nullptr;
-    IntAtomicCounter* cache_hit_count = nullptr;
-    IntAtomicCounter* cache_miss_count = nullptr;
-    IntAtomicCounter* cache_stampede_count = nullptr;
+    IntCounter* cache_lookup_count = nullptr;
+    IntCounter* cache_hit_count = nullptr;
+    IntCounter* cache_miss_count = nullptr;
+    IntCounter* cache_stampede_count = nullptr;
     DoubleGauge* cache_hit_ratio = nullptr;
     // bvars
     std::unique_ptr<bvar::Adder<uint64_t>> _hit_count_bvar;

be/src/olap/olap_common.h

@@ -389,6 +389,30 @@ struct OlapReaderStatistics {
     int64_t collect_iterator_merge_next_timer = 0;
     int64_t collect_iterator_normal_next_timer = 0;
     int64_t delete_bitmap_get_agg_ns = 0;
+
+    int64_t tablet_reader_init_timer_ns = 0;
+    int64_t tablet_reader_capture_rs_readers_timer_ns = 0;
+    int64_t tablet_reader_init_return_columns_timer_ns = 0;
+    int64_t tablet_reader_init_keys_param_timer_ns = 0;
+    int64_t tablet_reader_init_orderby_keys_param_timer_ns = 0;
+    int64_t tablet_reader_init_conditions_param_timer_ns = 0;
+    int64_t tablet_reader_init_delete_condition_param_timer_ns = 0;
+    int64_t block_reader_vcollect_iter_init_timer_ns = 0;
+    int64_t block_reader_rs_readers_init_timer_ns = 0;
+    int64_t block_reader_build_heap_init_timer_ns = 0;
+
+    int64_t rowset_reader_get_segment_iterators_timer_ns = 0;
+    int64_t rowset_reader_create_iterators_timer_ns = 0;
+    int64_t rowset_reader_init_iterators_timer_ns = 0;
+    int64_t rowset_reader_load_segments_timer_ns = 0;
+
+    int64_t segment_iterator_init_timer_ns = 0;
+    int64_t segment_iterator_init_return_column_iterators_timer_ns = 0;
+    int64_t segment_iterator_init_bitmap_index_iterators_timer_ns = 0;
+    int64_t segment_iterator_init_inverted_index_iterators_timer_ns = 0;
+
+    int64_t segment_create_column_readers_timer_ns = 0;
+    int64_t segment_load_index_timer_ns = 0;
 };
 
 using ColumnId = uint32_t;

be/src/olap/primary_key_index.cpp

@@ -95,27 +95,29 @@ Status PrimaryKeyIndexBuilder::finalize(segment_v2::PrimaryKeyIndexMetaPB* meta)
 }
 
 Status PrimaryKeyIndexReader::parse_index(io::FileReaderSPtr file_reader,
-                                          const segment_v2::PrimaryKeyIndexMetaPB& meta) {
+                                          const segment_v2::PrimaryKeyIndexMetaPB& meta,
+                                          OlapReaderStatistics* pk_index_load_stats) {
     // parse primary key index
     _index_reader.reset(new segment_v2::IndexedColumnReader(file_reader, meta.primary_key_index()));
     _index_reader->set_is_pk_index(true);
     RETURN_IF_ERROR(_index_reader->load(!config::disable_pk_storage_page_cache, false,
-                                        _pk_index_load_stats));
+                                        pk_index_load_stats));
 
     _index_parsed = true;
     return Status::OK();
 }
 
 Status PrimaryKeyIndexReader::parse_bf(io::FileReaderSPtr file_reader,
-                                       const segment_v2::PrimaryKeyIndexMetaPB& meta) {
+                                       const segment_v2::PrimaryKeyIndexMetaPB& meta,
+                                       OlapReaderStatistics* pk_index_load_stats) {
     // parse bloom filter
     segment_v2::ColumnIndexMetaPB column_index_meta = meta.bloom_filter_index();
     segment_v2::BloomFilterIndexReader bf_index_reader(std::move(file_reader),
                                                        column_index_meta.bloom_filter_index());
     RETURN_IF_ERROR(bf_index_reader.load(!config::disable_pk_storage_page_cache, false,
-                                         _pk_index_load_stats));
+                                         pk_index_load_stats));
     std::unique_ptr<segment_v2::BloomFilterIndexIterator> bf_iter;
-    RETURN_IF_ERROR(bf_index_reader.new_iterator(&bf_iter));
+    RETURN_IF_ERROR(bf_index_reader.new_iterator(&bf_iter, pk_index_load_stats));
     RETURN_IF_ERROR(bf_iter->read_bloom_filter(0, &_bf));
     segment_v2::g_pk_total_bloom_filter_num << 1;
     segment_v2::g_pk_total_bloom_filter_total_bytes << _bf->size();

be/src/olap/primary_key_index.h

@@ -98,8 +98,7 @@ class PrimaryKeyIndexBuilder {
 
 class PrimaryKeyIndexReader {
 public:
-    PrimaryKeyIndexReader(OlapReaderStatistics* pk_index_load_stats = nullptr)
-            : _index_parsed(false), _bf_parsed(false), _pk_index_load_stats(pk_index_load_stats) {}
+    PrimaryKeyIndexReader() : _index_parsed(false), _bf_parsed(false) {}
 
     ~PrimaryKeyIndexReader() {
         segment_v2::g_pk_total_bloom_filter_num << -static_cast<int64_t>(_bf_num);
@@ -109,12 +108,14 @@ class PrimaryKeyIndexReader {
     }
 
     Status parse_index(io::FileReaderSPtr file_reader,
-                       const segment_v2::PrimaryKeyIndexMetaPB& meta);
+                       const segment_v2::PrimaryKeyIndexMetaPB& meta,
+                       OlapReaderStatistics* pk_index_load_stats);
 
-    Status parse_bf(io::FileReaderSPtr file_reader, const segment_v2::PrimaryKeyIndexMetaPB& meta);
+    Status parse_bf(io::FileReaderSPtr file_reader, const segment_v2::PrimaryKeyIndexMetaPB& meta,
+                    OlapReaderStatistics* pk_index_load_stats);
 
     Status new_iterator(std::unique_ptr<segment_v2::IndexedColumnIterator>* index_iterator,
-                        OlapReaderStatistics* stats = nullptr) const {
+                        OlapReaderStatistics* stats) const {
         DCHECK(_index_parsed);
         index_iterator->reset(new segment_v2::IndexedColumnIterator(_index_reader.get(), stats));
         return Status::OK();
@@ -155,7 +156,6 @@ class PrimaryKeyIndexReader {
     std::unique_ptr<segment_v2::BloomFilter> _bf;
     size_t _bf_num = 0;
     uint64 _bf_bytes = 0;
-    OlapReaderStatistics* _pk_index_load_stats = nullptr;
 };
 
 } // namespace doris

be/src/olap/push_handler.cpp

@@ -518,7 +518,7 @@ Status PushBrokerReader::_convert_to_output_block(vectorized::Block* block) {
         column_ptr = _src_block.get_by_position(result_column_id).column;
         // column_ptr maybe a ColumnConst, convert it to a normal column
         column_ptr = column_ptr->convert_to_full_column_if_const();
-        DCHECK(column_ptr != nullptr);
+        DCHECK(column_ptr);
 
         // because of src_slot_desc is always be nullable, so the column_ptr after do dest_expr
         // is likely to be nullable