Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[opt](profile) add index page profile for io #45675

Merged
merged 2 commits into from
Dec 20, 2024
+164 −132
Merged

[opt](profile) add index page profile for io #45675

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
51747fe
[opt](profile) add index page profile for io
airborne12 Dec 20, 2024
0129a80
[opt](profile) add index page profile for io
airborne12 Dec 20, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions be/src/olap/base_tablet.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -499,7 +499,7 @@ Status BaseTablet::lookup_row_key(const Slice& encoded_key, TabletSchema* latest

for (auto id : picked_segments) {
Status s = segments[id]->lookup_row_key(encoded_key, schema, with_seq_col, with_rowid,
&loc, encoded_seq_value, stats);
&loc, stats, encoded_seq_value);
if (s.is<KEY_NOT_FOUND>()) {
continue;
}
Expand Down Expand Up @@ -615,7 +615,7 @@ Status BaseTablet::calc_segment_delete_bitmap(RowsetSharedPtr rowset,
vectorized::Block ordered_block = block.clone_empty();
uint32_t pos = 0;

RETURN_IF_ERROR(seg->load_pk_index_and_bf()); // We need index blocks to iterate
RETURN_IF_ERROR(seg->load_pk_index_and_bf(nullptr)); // We need index blocks to iterate
const auto* pk_idx = seg->get_primary_key_index();
int total = pk_idx->num_rows();
uint32_t row_id = 0;
Expand All @@ -629,7 +629,7 @@ Status BaseTablet::calc_segment_delete_bitmap(RowsetSharedPtr rowset,
std::vector<std::unique_ptr<SegmentCacheHandle>> segment_caches(specified_rowsets.size());
while (remaining > 0) {
std::unique_ptr<segment_v2::IndexedColumnIterator> iter;
RETURN_IF_ERROR(pk_idx->new_iterator(&iter));
RETURN_IF_ERROR(pk_idx->new_iterator(&iter, nullptr));

size_t num_to_read = std::min(batch_size, remaining);
auto index_type = vectorized::DataTypeFactory::instance().create_data_type(
Expand Down
5 changes: 2 additions & 3 deletions be/src/olap/delete_bitmap_calculator.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -145,12 +145,11 @@ Status MergeIndexDeleteBitmapCalculator::init(RowsetId rowset_id,
MergeIndexDeleteBitmapCalculatorContext::Comparator(seq_col_length, _rowid_length);
_contexts.reserve(segments.size());
_heap = std::make_unique<Heap>(_comparator);

for (auto& segment : segments) {
RETURN_IF_ERROR(segment->load_index());
RETURN_IF_ERROR(segment->load_index(nullptr));
auto pk_idx = segment->get_primary_key_index();
std::unique_ptr<segment_v2::IndexedColumnIterator> index;
RETURN_IF_ERROR(pk_idx->new_iterator(&index));
RETURN_IF_ERROR(pk_idx->new_iterator(&index, nullptr));
auto index_type = vectorized::DataTypeFactory::instance().create_data_type(
pk_idx->type_info()->type(), 1, 0);
_contexts.emplace_back(std::move(index), index_type, segment->id(), pk_idx->num_rows());
Expand Down
12 changes: 7 additions & 5 deletions be/src/olap/primary_key_index.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -95,27 +95,29 @@ Status PrimaryKeyIndexBuilder::finalize(segment_v2::PrimaryKeyIndexMetaPB* meta)
}

Status PrimaryKeyIndexReader::parse_index(io::FileReaderSPtr file_reader,
const segment_v2::PrimaryKeyIndexMetaPB& meta) {
const segment_v2::PrimaryKeyIndexMetaPB& meta,
OlapReaderStatistics* pk_index_load_stats) {
// parse primary key index
_index_reader.reset(new segment_v2::IndexedColumnReader(file_reader, meta.primary_key_index()));
_index_reader->set_is_pk_index(true);
RETURN_IF_ERROR(_index_reader->load(!config::disable_pk_storage_page_cache, false,
_pk_index_load_stats));
pk_index_load_stats));

_index_parsed = true;
return Status::OK();
}

Status PrimaryKeyIndexReader::parse_bf(io::FileReaderSPtr file_reader,
const segment_v2::PrimaryKeyIndexMetaPB& meta) {
const segment_v2::PrimaryKeyIndexMetaPB& meta,
OlapReaderStatistics* pk_index_load_stats) {
// parse bloom filter
segment_v2::ColumnIndexMetaPB column_index_meta = meta.bloom_filter_index();
segment_v2::BloomFilterIndexReader bf_index_reader(std::move(file_reader),
column_index_meta.bloom_filter_index());
RETURN_IF_ERROR(bf_index_reader.load(!config::disable_pk_storage_page_cache, false,
_pk_index_load_stats));
pk_index_load_stats));
std::unique_ptr<segment_v2::BloomFilterIndexIterator> bf_iter;
RETURN_IF_ERROR(bf_index_reader.new_iterator(&bf_iter));
RETURN_IF_ERROR(bf_index_reader.new_iterator(&bf_iter, pk_index_load_stats));
RETURN_IF_ERROR(bf_iter->read_bloom_filter(0, &_bf));
segment_v2::g_pk_total_bloom_filter_num << 1;
segment_v2::g_pk_total_bloom_filter_total_bytes << _bf->size();
Expand Down
12 changes: 6 additions & 6 deletions be/src/olap/primary_key_index.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -98,8 +98,7 @@ class PrimaryKeyIndexBuilder {

class PrimaryKeyIndexReader {
public:
PrimaryKeyIndexReader(OlapReaderStatistics* pk_index_load_stats = nullptr)
: _index_parsed(false), _bf_parsed(false), _pk_index_load_stats(pk_index_load_stats) {}
PrimaryKeyIndexReader() : _index_parsed(false), _bf_parsed(false) {}

~PrimaryKeyIndexReader() {
segment_v2::g_pk_total_bloom_filter_num << -static_cast<int64_t>(_bf_num);
Expand All @@ -109,12 +108,14 @@ class PrimaryKeyIndexReader {
}

Status parse_index(io::FileReaderSPtr file_reader,
const segment_v2::PrimaryKeyIndexMetaPB& meta);
const segment_v2::PrimaryKeyIndexMetaPB& meta,
OlapReaderStatistics* pk_index_load_stats);

Status parse_bf(io::FileReaderSPtr file_reader, const segment_v2::PrimaryKeyIndexMetaPB& meta);
Status parse_bf(io::FileReaderSPtr file_reader, const segment_v2::PrimaryKeyIndexMetaPB& meta,
OlapReaderStatistics* pk_index_load_stats);

Status new_iterator(std::unique_ptr<segment_v2::IndexedColumnIterator>* index_iterator,
OlapReaderStatistics* stats = nullptr) const {
OlapReaderStatistics* stats) const {
DCHECK(_index_parsed);
index_iterator->reset(new segment_v2::IndexedColumnIterator(_index_reader.get(), stats));
return Status::OK();
Expand Down Expand Up @@ -155,7 +156,6 @@ class PrimaryKeyIndexReader {
std::unique_ptr<segment_v2::BloomFilter> _bf;
size_t _bf_num = 0;
uint64 _bf_bytes = 0;
OlapReaderStatistics* _pk_index_load_stats = nullptr;
};

} // namespace doris
15 changes: 8 additions & 7 deletions be/src/olap/rowset/segment_v2/bloom_filter_index_reader.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,9 +34,8 @@ namespace segment_v2 {
Status BloomFilterIndexReader::load(bool use_page_cache, bool kept_in_memory,
OlapReaderStatistics* index_load_stats) {
// TODO yyq: implement a new once flag to avoid status construct.
_index_load_stats = index_load_stats;
return _load_once.call([this, use_page_cache, kept_in_memory] {
return _load(use_page_cache, kept_in_memory);
return _load_once.call([this, use_page_cache, kept_in_memory, index_load_stats] {
return _load(use_page_cache, kept_in_memory, index_load_stats);
});
}

Expand All @@ -45,20 +44,22 @@ int64_t BloomFilterIndexReader::get_metadata_size() const {
(_bloom_filter_index_meta ? _bloom_filter_index_meta->ByteSizeLong() : 0);
}

Status BloomFilterIndexReader::_load(bool use_page_cache, bool kept_in_memory) {
Status BloomFilterIndexReader::_load(bool use_page_cache, bool kept_in_memory,
OlapReaderStatistics* index_load_stats) {
const IndexedColumnMetaPB& bf_index_meta = _bloom_filter_index_meta->bloom_filter();

_bloom_filter_reader.reset(new IndexedColumnReader(_file_reader, bf_index_meta));
RETURN_IF_ERROR(_bloom_filter_reader->load(use_page_cache, kept_in_memory, _index_load_stats));
RETURN_IF_ERROR(_bloom_filter_reader->load(use_page_cache, kept_in_memory, index_load_stats));
update_metadata_size();
airborne12 marked this conversation as resolved.
Show resolved Hide resolved
return Status::OK();
}

Status BloomFilterIndexReader::new_iterator(std::unique_ptr<BloomFilterIndexIterator>* iterator) {
Status BloomFilterIndexReader::new_iterator(std::unique_ptr<BloomFilterIndexIterator>* iterator,
OlapReaderStatistics* index_load_stats) {
DBUG_EXECUTE_IF("BloomFilterIndexReader::new_iterator.fail", {
return Status::InternalError("new_iterator for bloom filter index failed");
});
iterator->reset(new BloomFilterIndexIterator(this));
iterator->reset(new BloomFilterIndexIterator(this, index_load_stats));
return Status::OK();
}

Expand Down
12 changes: 6 additions & 6 deletions be/src/olap/rowset/segment_v2/bloom_filter_index_reader.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,17 +48,18 @@ class BloomFilterIndexReader : public MetadataAdder<BloomFilterIndexReader> {
}

Status load(bool use_page_cache, bool kept_in_memory,
OlapReaderStatistics* _bf_index_load_stats = nullptr);
OlapReaderStatistics* bf_index_load_stats);

BloomFilterAlgorithmPB algorithm() { return _bloom_filter_index_meta->algorithm(); }

// create a new column iterator.
Status new_iterator(std::unique_ptr<BloomFilterIndexIterator>* iterator);
Status new_iterator(std::unique_ptr<BloomFilterIndexIterator>* iterator,
OlapReaderStatistics* index_load_stats);

const TypeInfo* type_info() const { return _type_info; }

private:
Status _load(bool use_page_cache, bool kept_in_memory);
Status _load(bool use_page_cache, bool kept_in_memory, OlapReaderStatistics* index_load_stats);

int64_t get_metadata_size() const override;

Expand All @@ -70,13 +71,12 @@ class BloomFilterIndexReader : public MetadataAdder<BloomFilterIndexReader> {
const TypeInfo* _type_info = nullptr;
std::unique_ptr<BloomFilterIndexPB> _bloom_filter_index_meta = nullptr;
std::unique_ptr<IndexedColumnReader> _bloom_filter_reader;
OlapReaderStatistics* _index_load_stats = nullptr;
};

class BloomFilterIndexIterator {
public:
explicit BloomFilterIndexIterator(BloomFilterIndexReader* reader)
: _reader(reader), _bloom_filter_iter(reader->_bloom_filter_reader.get()) {}
explicit BloomFilterIndexIterator(BloomFilterIndexReader* reader, OlapReaderStatistics* stats)
: _reader(reader), _bloom_filter_iter(reader->_bloom_filter_reader.get(), stats) {}

// Read bloom filter at the given ordinal into `bf`.
Status read_bloom_filter(rowid_t ordinal, std::unique_ptr<BloomFilter>* bf);
Expand Down
62 changes: 36 additions & 26 deletions be/src/olap/rowset/segment_v2/column_reader.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -374,10 +374,12 @@ Status ColumnReader::read_page(const ColumnIteratorOptions& iter_opts, const Pag

Status ColumnReader::get_row_ranges_by_zone_map(
const AndBlockColumnPredicate* col_predicates,
const std::vector<const ColumnPredicate*>* delete_predicates, RowRanges* row_ranges) {
const std::vector<const ColumnPredicate*>* delete_predicates, RowRanges* row_ranges,
const ColumnIteratorOptions& iter_opts) {
std::vector<uint32_t> page_indexes;
RETURN_IF_ERROR(_get_filtered_pages(col_predicates, delete_predicates, &page_indexes));
RETURN_IF_ERROR(_calculate_row_ranges(page_indexes, row_ranges));
RETURN_IF_ERROR(
_get_filtered_pages(col_predicates, delete_predicates, &page_indexes, iter_opts));
RETURN_IF_ERROR(_calculate_row_ranges(page_indexes, row_ranges, iter_opts));
return Status::OK();
}

Expand Down Expand Up @@ -514,8 +516,8 @@ bool ColumnReader::_zone_map_match_condition(const ZoneMapPB& zone_map,
Status ColumnReader::_get_filtered_pages(
const AndBlockColumnPredicate* col_predicates,
const std::vector<const ColumnPredicate*>* delete_predicates,
std::vector<uint32_t>* page_indexes) {
RETURN_IF_ERROR(_load_zone_map_index(_use_index_page_cache, _opts.kept_in_memory));
std::vector<uint32_t>* page_indexes, const ColumnIteratorOptions& iter_opts) {
RETURN_IF_ERROR(_load_zone_map_index(_use_index_page_cache, _opts.kept_in_memory, iter_opts));

FieldType type = _type_info->type();
const std::vector<ZoneMapPB>& zone_maps = _zone_map_index->page_zone_maps();
Expand Down Expand Up @@ -553,9 +555,10 @@ Status ColumnReader::_get_filtered_pages(
}

Status ColumnReader::_calculate_row_ranges(const std::vector<uint32_t>& page_indexes,
RowRanges* row_ranges) {
RowRanges* row_ranges,
const ColumnIteratorOptions& iter_opts) {
row_ranges->clear();
RETURN_IF_ERROR(_load_ordinal_index(_use_index_page_cache, _opts.kept_in_memory));
RETURN_IF_ERROR(_load_ordinal_index(_use_index_page_cache, _opts.kept_in_memory, iter_opts));
for (auto i : page_indexes) {
ordinal_t page_first_id = _ordinal_index->get_first_ordinal(i);
ordinal_t page_last_id = _ordinal_index->get_last_ordinal(i);
Expand All @@ -566,12 +569,14 @@ Status ColumnReader::_calculate_row_ranges(const std::vector<uint32_t>& page_ind
}

Status ColumnReader::get_row_ranges_by_bloom_filter(const AndBlockColumnPredicate* col_predicates,
RowRanges* row_ranges) {
RETURN_IF_ERROR(_load_ordinal_index(_use_index_page_cache, _opts.kept_in_memory));
RETURN_IF_ERROR(_load_bloom_filter_index(_use_index_page_cache, _opts.kept_in_memory));
RowRanges* row_ranges,
const ColumnIteratorOptions& iter_opts) {
RETURN_IF_ERROR(_load_ordinal_index(_use_index_page_cache, _opts.kept_in_memory, iter_opts));
RETURN_IF_ERROR(
_load_bloom_filter_index(_use_index_page_cache, _opts.kept_in_memory, iter_opts));
RowRanges bf_row_ranges;
std::unique_ptr<BloomFilterIndexIterator> bf_iter;
RETURN_IF_ERROR(_bloom_filter_index->new_iterator(&bf_iter));
RETURN_IF_ERROR(_bloom_filter_index->new_iterator(&bf_iter, iter_opts.stats));
size_t range_size = row_ranges->range_size();
// get covered page ids
std::set<uint32_t> page_ids;
Expand All @@ -598,16 +603,18 @@ Status ColumnReader::get_row_ranges_by_bloom_filter(const AndBlockColumnPredicat
return Status::OK();
}

Status ColumnReader::_load_ordinal_index(bool use_page_cache, bool kept_in_memory) {
Status ColumnReader::_load_ordinal_index(bool use_page_cache, bool kept_in_memory,
const ColumnIteratorOptions& iter_opts) {
if (!_ordinal_index) {
return Status::InternalError("ordinal_index not inited");
}
return _ordinal_index->load(use_page_cache, kept_in_memory);
return _ordinal_index->load(use_page_cache, kept_in_memory, iter_opts.stats);
}

Status ColumnReader::_load_zone_map_index(bool use_page_cache, bool kept_in_memory) {
Status ColumnReader::_load_zone_map_index(bool use_page_cache, bool kept_in_memory,
const ColumnIteratorOptions& iter_opts) {
if (_zone_map_index != nullptr) {
return _zone_map_index->load(use_page_cache, kept_in_memory);
return _zone_map_index->load(use_page_cache, kept_in_memory, iter_opts.stats);
}
return Status::OK();
}
Expand Down Expand Up @@ -681,24 +688,27 @@ bool ColumnReader::has_bloom_filter_index(bool ngram) const {
}
}

Status ColumnReader::_load_bloom_filter_index(bool use_page_cache, bool kept_in_memory) {
Status ColumnReader::_load_bloom_filter_index(bool use_page_cache, bool kept_in_memory,
const ColumnIteratorOptions& iter_opts) {
if (_bloom_filter_index != nullptr) {
return _bloom_filter_index->load(use_page_cache, kept_in_memory);
return _bloom_filter_index->load(use_page_cache, kept_in_memory, iter_opts.stats);
}
return Status::OK();
}

Status ColumnReader::seek_to_first(OrdinalPageIndexIterator* iter) {
RETURN_IF_ERROR(_load_ordinal_index(_use_index_page_cache, _opts.kept_in_memory));
Status ColumnReader::seek_to_first(OrdinalPageIndexIterator* iter,
const ColumnIteratorOptions& iter_opts) {
RETURN_IF_ERROR(_load_ordinal_index(_use_index_page_cache, _opts.kept_in_memory, iter_opts));
*iter = _ordinal_index->begin();
if (!iter->valid()) {
return Status::NotFound("Failed to seek to first rowid");
}
return Status::OK();
}

Status ColumnReader::seek_at_or_before(ordinal_t ordinal, OrdinalPageIndexIterator* iter) {
RETURN_IF_ERROR(_load_ordinal_index(_use_index_page_cache, _opts.kept_in_memory));
Status ColumnReader::seek_at_or_before(ordinal_t ordinal, OrdinalPageIndexIterator* iter,
const ColumnIteratorOptions& iter_opts) {
RETURN_IF_ERROR(_load_ordinal_index(_use_index_page_cache, _opts.kept_in_memory, iter_opts));
*iter = _ordinal_index->seek_at_or_before(ordinal);
if (!iter->valid()) {
return Status::NotFound("Failed to seek to ordinal {}, ", ordinal);
Expand Down Expand Up @@ -1172,7 +1182,7 @@ Status FileColumnIterator::init(const ColumnIteratorOptions& opts) {
FileColumnIterator::~FileColumnIterator() = default;

Status FileColumnIterator::seek_to_first() {
RETURN_IF_ERROR(_reader->seek_to_first(&_page_iter));
RETURN_IF_ERROR(_reader->seek_to_first(&_page_iter, _opts));
RETURN_IF_ERROR(_read_data_page(_page_iter));

_seek_to_pos_in_page(&_page, 0);
Expand All @@ -1183,7 +1193,7 @@ Status FileColumnIterator::seek_to_first() {
Status FileColumnIterator::seek_to_ordinal(ordinal_t ord) {
// if current page contains this row, we don't need to seek
if (!_page || !_page.contains(ord) || !_page_iter.valid()) {
RETURN_IF_ERROR(_reader->seek_at_or_before(ord, &_page_iter));
RETURN_IF_ERROR(_reader->seek_at_or_before(ord, &_page_iter, _opts));
RETURN_IF_ERROR(_read_data_page(_page_iter));
}
_seek_to_pos_in_page(&_page, ord - _page.first_ordinal);
Expand Down Expand Up @@ -1431,8 +1441,8 @@ Status FileColumnIterator::get_row_ranges_by_zone_map(
const AndBlockColumnPredicate* col_predicates,
const std::vector<const ColumnPredicate*>* delete_predicates, RowRanges* row_ranges) {
if (_reader->has_zone_map()) {
RETURN_IF_ERROR(
_reader->get_row_ranges_by_zone_map(col_predicates, delete_predicates, row_ranges));
RETURN_IF_ERROR(_reader->get_row_ranges_by_zone_map(col_predicates, delete_predicates,
row_ranges, _opts));
}
return Status::OK();
}
Expand All @@ -1441,7 +1451,7 @@ Status FileColumnIterator::get_row_ranges_by_bloom_filter(
const AndBlockColumnPredicate* col_predicates, RowRanges* row_ranges) {
if ((col_predicates->can_do_bloom_filter(false) && _reader->has_bloom_filter_index(false)) ||
(col_predicates->can_do_bloom_filter(true) && _reader->has_bloom_filter_index(true))) {
RETURN_IF_ERROR(_reader->get_row_ranges_by_bloom_filter(col_predicates, row_ranges));
RETURN_IF_ERROR(_reader->get_row_ranges_by_bloom_filter(col_predicates, row_ranges, _opts));
}
return Status::OK();
}
Expand Down
Loading
Loading