diff --git a/be/src/vec/common/pod_array.h b/be/src/vec/common/pod_array.h index 3ff676461577c4..9a00e295cb34fa 100644 --- a/be/src/vec/common/pod_array.h +++ b/be/src/vec/common/pod_array.h @@ -125,7 +125,19 @@ class PODArrayBase : private boost::noncopyable, char* c_end_of_storage = null; /// Does not include pad_right. /// The amount of memory occupied by the num_elements of the elements. - static size_t byte_size(size_t num_elements) { return num_elements * ELEMENT_SIZE; } + static size_t byte_size(size_t num_elements) { +#ifndef NDEBUG + size_t amount; + if (__builtin_mul_overflow(num_elements, ELEMENT_SIZE, &amount)) { + DCHECK(false) + << "Amount of memory requested to allocate is more than allowed, num_elements " + << num_elements << ", ELEMENT_SIZE " << ELEMENT_SIZE; + } + return amount; +#else + return num_elements * ELEMENT_SIZE; +#endif + } /// Minimum amount of memory to allocate for num_elements, including padding. static size_t minimum_memory_for_elements(size_t num_elements) { @@ -275,6 +287,19 @@ class PODArrayBase : private boost::noncopyable, #endif } + template + void assert_not_intersects(It1 from_begin [[maybe_unused]], It2 from_end [[maybe_unused]]) { +#ifndef NDEBUG + const char* ptr_begin = reinterpret_cast(&*from_begin); + const char* ptr_end = reinterpret_cast(&*from_end); + + /// Also it's safe if the range is empty. + assert(!((ptr_begin >= c_start && ptr_begin < c_end) || + (ptr_end > c_start && ptr_end <= c_end)) || + (ptr_begin == ptr_end)); +#endif + } + ~PODArrayBase() { dealloc(); } }; @@ -296,8 +321,8 @@ class PODArray : public PODArrayBase void insert_prepare(It1 from_begin, It2 from_end, TAllocatorParams&&... allocator_params) { + this->assert_not_intersects(from_begin, from_end); size_t required_capacity = this->size() + (from_end - from_begin); if (required_capacity > this->capacity()) this->reserve(round_up_to_power_of_two_or_zero(required_capacity), @@ -461,14 +487,17 @@ class PODArray : public PODArrayBase void insert(iterator it, It1 from_begin, It2 from_end) { - insert_prepare(from_begin, from_end); - size_t bytes_to_copy = this->byte_size(from_end - from_begin); - size_t bytes_to_move = (end() - it) * sizeof(T); + if (!bytes_to_copy) { + return; + } + size_t bytes_to_move = this->byte_size(end() - it); + insert_prepare(from_begin, from_end); - if (UNLIKELY(bytes_to_move)) - memcpy(this->c_end + bytes_to_copy - bytes_to_move, this->c_end - bytes_to_move, - bytes_to_move); + if (UNLIKELY(bytes_to_move)) { + memmove(this->c_end + bytes_to_copy - bytes_to_move, this->c_end - bytes_to_move, + bytes_to_move); + } memcpy(this->c_end - bytes_to_move, reinterpret_cast(&*from_begin), bytes_to_copy); @@ -477,6 +506,7 @@ class PODArray : public PODArrayBase void insert_assume_reserved(It1 from_begin, It2 from_end) { + this->assert_not_intersects(from_begin, from_end); size_t bytes_to_copy = this->byte_size(from_end - from_begin); memcpy(this->c_end, reinterpret_cast(&*from_begin), bytes_to_copy); this->c_end += bytes_to_copy; @@ -593,6 +623,7 @@ class PODArray : public PODArrayBase void assign(It1 from_begin, It2 from_end) { + this->assert_not_intersects(from_begin, from_end); size_t required_capacity = from_end - from_begin; if (required_capacity > this->capacity()) this->reserve(round_up_to_power_of_two_or_zero(required_capacity)); @@ -604,15 +635,13 @@ class PODArray : public PODArrayBase(first); auto last_no_const = const_cast(last); size_t items_to_move = end() - last; - while (items_to_move != 0) - { + while (items_to_move != 0) { *first_no_const = *last_no_const; ++first_no_const; @@ -621,13 +650,10 @@ class PODArray : public PODArrayBasec_end = reinterpret_cast(first_no_const); + this->c_end = reinterpret_cast(first_no_const); } - void erase(const_iterator pos) - { - this->erase(pos, pos + 1); - } + void erase(const_iterator pos) { this->erase(pos, pos + 1); } bool operator==(const PODArray& rhs) const { if (this->size() != rhs.size()) { diff --git a/be/src/vec/common/pod_array_fwd.h b/be/src/vec/common/pod_array_fwd.h index e1a428eda9dafb..bd0c7e272e4ea4 100644 --- a/be/src/vec/common/pod_array_fwd.h +++ b/be/src/vec/common/pod_array_fwd.h @@ -36,12 +36,12 @@ template class PODArray; -/** For columns. Padding is enough to read and write xmm-register at the address of the last element. - * TODO, pad_right is temporarily changed from 15 to 16, will waste 1 bytes, - * can rollback after fix wrong reinterpret_cast column and PODArray swap. +/** For columns. Padding is enough to read and write xmm-register at the address of the last element. + * TODO, Adapt internal data structures to 512-bit era https://github.com/ClickHouse/ClickHouse/pull/42564 + * Padding in internal data structures increased to 64 bytes., support AVX-512 simd. */ template > -using PaddedPODArray = PODArray; +using PaddedPODArray = PODArray; /** A helper for declaring PODArray that uses inline memory. * The initial size is set to use all the inline bytes, since using less would diff --git a/be/test/vec/common/pod_array_test.cpp b/be/test/vec/common/pod_array_test.cpp new file mode 100644 index 00000000000000..c8525d23b97d45 --- /dev/null +++ b/be/test/vec/common/pod_array_test.cpp @@ -0,0 +1,626 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "vec/common/pod_array.h" + +#include + +#include "vec/common/allocator_fwd.h" + +namespace doris { + +TEST(PODArrayTest, PODArrayBasicMove) { + static constexpr size_t initial_bytes = 32; + using Array = vectorized::PODArray, initial_bytes>>; + + { + Array arr; + Array arr2; + arr2 = std::move(arr); + } + + { + Array arr; + + arr.push_back(1); + arr.push_back(2); + arr.push_back(3); + + Array arr2; + + arr2 = std::move(arr); + + ASSERT_EQ(arr2.size(), 3); + ASSERT_EQ(arr2[0], 1); + ASSERT_EQ(arr2[1], 2); + ASSERT_EQ(arr2[2], 3); + + arr = std::move(arr2); + + ASSERT_EQ(arr.size(), 3); + ASSERT_EQ(arr[0], 1); + ASSERT_EQ(arr[1], 2); + ASSERT_EQ(arr[2], 3); + } + + { + Array arr; + + arr.push_back(1); + arr.push_back(2); + arr.push_back(3); + arr.push_back(4); + arr.push_back(5); + + Array arr2; + + arr2 = std::move(arr); + + ASSERT_EQ(arr2.size(), 5); + ASSERT_EQ(arr2[0], 1); + ASSERT_EQ(arr2[1], 2); + ASSERT_EQ(arr2[2], 3); + ASSERT_EQ(arr2[3], 4); + ASSERT_EQ(arr2[4], 5); + + arr = std::move(arr2); + + ASSERT_EQ(arr.size(), 5); + ASSERT_EQ(arr[0], 1); + ASSERT_EQ(arr[1], 2); + ASSERT_EQ(arr[2], 3); + ASSERT_EQ(arr[3], 4); + ASSERT_EQ(arr[4], 5); + } + + { + Array arr; + + arr.push_back(1); + arr.push_back(2); + arr.push_back(3); + + Array arr2; + + arr2.push_back(4); + arr2.push_back(5); + arr2.push_back(6); + arr2.push_back(7); + + arr2 = std::move(arr); + + ASSERT_EQ(arr2.size(), 3); + ASSERT_EQ(arr2[0], 1); + ASSERT_EQ(arr2[1], 2); + ASSERT_EQ(arr2[2], 3); + } + + { + Array arr; + + arr.push_back(1); + arr.push_back(2); + arr.push_back(3); + + Array arr2; + + arr2.push_back(4); + arr2.push_back(5); + arr2.push_back(6); + arr2.push_back(7); + arr2.push_back(8); + + arr = std::move(arr2); + + ASSERT_EQ(arr.size(), 5); + ASSERT_EQ(arr[0], 4); + ASSERT_EQ(arr[1], 5); + ASSERT_EQ(arr[2], 6); + ASSERT_EQ(arr[3], 7); + ASSERT_EQ(arr[4], 8); + } +} + +TEST(PODArrayTest, PODArrayBasicSwap) { + static constexpr size_t initial_bytes = 32; + using Array = vectorized::PODArray, initial_bytes>>; + + { + Array arr; + Array arr2; + arr.swap(arr2); + arr2.swap(arr); + } + + { + Array arr; + + Array arr2; + + arr2.push_back(1); + arr2.push_back(2); + arr2.push_back(3); + + arr.swap(arr2); + + ASSERT_TRUE(arr.size() == 3); + ASSERT_TRUE(arr[0] == 1); + ASSERT_TRUE(arr[1] == 2); + ASSERT_TRUE(arr[2] == 3); + + ASSERT_TRUE(arr2.empty()); + + arr.swap(arr2); + + ASSERT_TRUE(arr.empty()); + + ASSERT_TRUE(arr2.size() == 3); + ASSERT_TRUE(arr2[0] == 1); + ASSERT_TRUE(arr2[1] == 2); + ASSERT_TRUE(arr2[2] == 3); + } + + { + Array arr; + + Array arr2; + + arr2.push_back(1); + arr2.push_back(2); + arr2.push_back(3); + arr2.push_back(4); + arr2.push_back(5); + + arr.swap(arr2); + + ASSERT_TRUE(arr.size() == 5); + ASSERT_TRUE(arr[0] == 1); + ASSERT_TRUE(arr[1] == 2); + ASSERT_TRUE(arr[2] == 3); + ASSERT_TRUE(arr[3] == 4); + ASSERT_TRUE(arr[4] == 5); + + ASSERT_TRUE(arr2.empty()); + + arr.swap(arr2); + + ASSERT_TRUE(arr.empty()); + + ASSERT_TRUE(arr2.size() == 5); + ASSERT_TRUE(arr2[0] == 1); + ASSERT_TRUE(arr2[1] == 2); + ASSERT_TRUE(arr2[2] == 3); + ASSERT_TRUE(arr2[3] == 4); + ASSERT_TRUE(arr2[4] == 5); + } + + { + Array arr; + + arr.push_back(1); + arr.push_back(2); + arr.push_back(3); + + Array arr2; + + arr2.push_back(4); + arr2.push_back(5); + arr2.push_back(6); + + arr.swap(arr2); + + ASSERT_TRUE(arr.size() == 3); + ASSERT_TRUE(arr[0] == 4); + ASSERT_TRUE(arr[1] == 5); + ASSERT_TRUE(arr[2] == 6); + + ASSERT_TRUE(arr2.size() == 3); + ASSERT_TRUE(arr2[0] == 1); + ASSERT_TRUE(arr2[1] == 2); + ASSERT_TRUE(arr2[2] == 3); + + arr.swap(arr2); + + ASSERT_TRUE(arr.size() == 3); + ASSERT_TRUE(arr[0] == 1); + ASSERT_TRUE(arr[1] == 2); + ASSERT_TRUE(arr[2] == 3); + + ASSERT_TRUE(arr2.size() == 3); + ASSERT_TRUE(arr2[0] == 4); + ASSERT_TRUE(arr2[1] == 5); + ASSERT_TRUE(arr2[2] == 6); + } + + { + Array arr; + + arr.push_back(1); + arr.push_back(2); + + Array arr2; + + arr2.push_back(3); + arr2.push_back(4); + arr2.push_back(5); + + arr.swap(arr2); + + ASSERT_TRUE(arr.size() == 3); + ASSERT_TRUE(arr[0] == 3); + ASSERT_TRUE(arr[1] == 4); + ASSERT_TRUE(arr[2] == 5); + + ASSERT_TRUE(arr2.size() == 2); + ASSERT_TRUE(arr2[0] == 1); + ASSERT_TRUE(arr2[1] == 2); + + arr.swap(arr2); + + ASSERT_TRUE(arr.size() == 2); + ASSERT_TRUE(arr[0] == 1); + ASSERT_TRUE(arr[1] == 2); + + ASSERT_TRUE(arr2.size() == 3); + ASSERT_TRUE(arr2[0] == 3); + ASSERT_TRUE(arr2[1] == 4); + ASSERT_TRUE(arr2[2] == 5); + } + + { + Array arr; + + arr.push_back(1); + arr.push_back(2); + arr.push_back(3); + + Array arr2; + + arr2.push_back(4); + arr2.push_back(5); + arr2.push_back(6); + arr2.push_back(7); + arr2.push_back(8); + + arr.swap(arr2); + + ASSERT_TRUE(arr.size() == 5); + ASSERT_TRUE(arr[0] == 4); + ASSERT_TRUE(arr[1] == 5); + ASSERT_TRUE(arr[2] == 6); + ASSERT_TRUE(arr[3] == 7); + ASSERT_TRUE(arr[4] == 8); + + ASSERT_TRUE(arr2.size() == 3); + ASSERT_TRUE(arr2[0] == 1); + ASSERT_TRUE(arr2[1] == 2); + ASSERT_TRUE(arr2[2] == 3); + + arr.swap(arr2); + + ASSERT_TRUE(arr.size() == 3); + ASSERT_TRUE(arr[0] == 1); + ASSERT_TRUE(arr[1] == 2); + ASSERT_TRUE(arr[2] == 3); + + ASSERT_TRUE(arr2.size() == 5); + ASSERT_TRUE(arr2[0] == 4); + ASSERT_TRUE(arr2[1] == 5); + ASSERT_TRUE(arr2[2] == 6); + ASSERT_TRUE(arr2[3] == 7); + ASSERT_TRUE(arr2[4] == 8); + } + + { + Array arr; + + arr.push_back(1); + arr.push_back(2); + arr.push_back(3); + arr.push_back(4); + arr.push_back(5); + + Array arr2; + + arr2.push_back(6); + arr2.push_back(7); + arr2.push_back(8); + arr2.push_back(9); + arr2.push_back(10); + + arr.swap(arr2); + + ASSERT_TRUE(arr.size() == 5); + ASSERT_TRUE(arr[0] == 6); + ASSERT_TRUE(arr[1] == 7); + ASSERT_TRUE(arr[2] == 8); + ASSERT_TRUE(arr[3] == 9); + ASSERT_TRUE(arr[4] == 10); + + ASSERT_TRUE(arr2.size() == 5); + ASSERT_TRUE(arr2[0] == 1); + ASSERT_TRUE(arr2[1] == 2); + ASSERT_TRUE(arr2[2] == 3); + ASSERT_TRUE(arr2[3] == 4); + ASSERT_TRUE(arr2[4] == 5); + + arr.swap(arr2); + + ASSERT_TRUE(arr.size() == 5); + ASSERT_TRUE(arr[0] == 1); + ASSERT_TRUE(arr[1] == 2); + ASSERT_TRUE(arr[2] == 3); + ASSERT_TRUE(arr[3] == 4); + ASSERT_TRUE(arr[4] == 5); + + ASSERT_TRUE(arr2.size() == 5); + ASSERT_TRUE(arr2[0] == 6); + ASSERT_TRUE(arr2[1] == 7); + ASSERT_TRUE(arr2[2] == 8); + ASSERT_TRUE(arr2[3] == 9); + ASSERT_TRUE(arr2[4] == 10); + } +} + +TEST(PODArrayTest, PODArrayBasicSwapMoveConstructor) { + static constexpr size_t initial_bytes = 32; + using Array = vectorized::PODArray, initial_bytes>>; + + { + Array arr; + Array arr2 {std::move(arr)}; + } + + { + Array arr; + + arr.push_back(1); + arr.push_back(2); + arr.push_back(3); + + Array arr2 {std::move(arr)}; + + ASSERT_TRUE(arr.empty()); // NOLINT + + ASSERT_TRUE(arr2.size() == 3); + ASSERT_TRUE(arr2[0] == 1); + ASSERT_TRUE(arr2[1] == 2); + ASSERT_TRUE(arr2[2] == 3); + } + + { + Array arr; + + arr.push_back(1); + arr.push_back(2); + arr.push_back(3); + arr.push_back(4); + arr.push_back(5); + + Array arr2 {std::move(arr)}; + + ASSERT_TRUE(arr.empty()); // NOLINT + + ASSERT_TRUE(arr2.size() == 5); + ASSERT_TRUE(arr2[0] == 1); + ASSERT_TRUE(arr2[1] == 2); + ASSERT_TRUE(arr2[2] == 3); + ASSERT_TRUE(arr2[3] == 4); + ASSERT_TRUE(arr2[4] == 5); + } +} + +TEST(PODArrayTest, PODArrayInsert) { + { + std::string str = "test_string_abacaba"; + vectorized::PODArray chars; + chars.insert(chars.end(), str.begin(), str.end()); + EXPECT_EQ(str, std::string(chars.data(), chars.size())); + + std::string insert_in_the_middle = "insert_in_the_middle"; + auto pos = str.size() / 2; + str.insert(str.begin() + pos, insert_in_the_middle.begin(), insert_in_the_middle.end()); + chars.insert(chars.begin() + pos, insert_in_the_middle.begin(), insert_in_the_middle.end()); + EXPECT_EQ(str, std::string(chars.data(), chars.size())); + + std::string insert_with_resize; + insert_with_resize.reserve(chars.capacity() * 2); + char cur_char = 'a'; + while (insert_with_resize.size() < insert_with_resize.capacity()) { + insert_with_resize += cur_char; + if (cur_char == 'z') { + cur_char = 'a'; + } else { + ++cur_char; + } + } + str.insert(str.begin(), insert_with_resize.begin(), insert_with_resize.end()); + chars.insert(chars.begin(), insert_with_resize.begin(), insert_with_resize.end()); + EXPECT_EQ(str, std::string(chars.data(), chars.size())); + } + { + vectorized::PODArray values; + vectorized::PODArray values_to_insert; + + for (size_t i = 0; i < 120; ++i) { + values.emplace_back(i); + } + + values.insert(values.begin() + 1, values_to_insert.begin(), values_to_insert.end()); + ASSERT_EQ(values.size(), 120); + + values_to_insert.emplace_back(0); + values_to_insert.emplace_back(1); + + values.insert(values.begin() + 1, values_to_insert.begin(), values_to_insert.end()); + ASSERT_EQ(values.size(), 122); + + values_to_insert.clear(); + for (size_t i = 0; i < 240; ++i) { + values_to_insert.emplace_back(i); + } + + values.insert(values.begin() + 1, values_to_insert.begin(), values_to_insert.end()); + ASSERT_EQ(values.size(), 362); + } +} + +// TEST(PODArrayTest, PODArrayInsertFromItself) +// { +// { +// vectorized::PaddedPODArray array { 1 }; + +// for (size_t i = 0; i < 3; ++i) +// array.insertFromItself(array.begin(), array.end()); + +// vectorized::PaddedPODArray expected {1,1,1,1,1,1,1,1}; +// ASSERT_EQ(array,expected); +// } +// } + +TEST(PODArrayTest, PODArrayAssign) { + { + vectorized::PaddedPODArray array; + array.push_back(1); + array.push_back(2); + + array.assign({1, 2, 3}); + + ASSERT_EQ(array.size(), 3); + ASSERT_EQ(array, vectorized::PaddedPODArray({1, 2, 3})); + } + { + vectorized::PaddedPODArray array; + array.push_back(1); + array.push_back(2); + + array.assign({}); + + ASSERT_TRUE(array.empty()); + } + { + vectorized::PaddedPODArray array; + array.assign({}); + + ASSERT_TRUE(array.empty()); + } +} + +TEST(PODArrayTest, PODNoOverallocation) { + /// Check that PaddedPODArray allocates for smaller number of elements than the power of two due to padding. + /// NOTE: It's Ok to change these numbers if you will modify initial size or padding. + + vectorized::PaddedPODArray chars; + std::vector capacities; + + size_t prev_capacity = 0; + for (size_t i = 0; i < 1000000; ++i) { + chars.emplace_back(); + if (chars.capacity() != prev_capacity) { + prev_capacity = chars.capacity(); + capacities.emplace_back(prev_capacity); + } + } + + EXPECT_EQ(capacities, (std::vector {4064, 8160, 16352, 32736, 65504, 131040, 262112, + 524256, 1048544})); +} + +template +struct ItemWithSize { + char v[size] {}; +}; + +TEST(PODArrayTest, PODInsertElementSizeNotMultipleOfLeftPadding) { + using ItemWith24Size = ItemWithSize<24>; + vectorized::PaddedPODArray arr1_initially_empty; + + size_t items_to_insert_size = 120000; + + for (size_t test = 0; test < items_to_insert_size; ++test) { + arr1_initially_empty.emplace_back(); + } + + EXPECT_EQ(arr1_initially_empty.size(), items_to_insert_size); + + vectorized::PaddedPODArray arr2_initially_nonempty; + + for (size_t test = 0; test < items_to_insert_size; ++test) { + arr2_initially_nonempty.emplace_back(); + } + + EXPECT_EQ(arr1_initially_empty.size(), items_to_insert_size); +} + +TEST(PODArrayTest, PODErase) { + { + vectorized::PaddedPODArray items {0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; + vectorized::PaddedPODArray expected; + expected = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; + + items.erase(items.begin(), items.begin()); + EXPECT_EQ(items, expected); + + items.erase(items.end(), items.end()); + EXPECT_EQ(items, expected); + } + { + vectorized::PaddedPODArray actual {0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; + vectorized::PaddedPODArray expected; + + expected = {0, 1, 4, 5, 6, 7, 8, 9}; + actual.erase(actual.begin() + 2, actual.begin() + 4); + EXPECT_EQ(actual, expected); + + expected = {0, 1, 4}; + actual.erase(actual.begin() + 3, actual.end()); + EXPECT_EQ(actual, expected); + + expected = {}; + actual.erase(actual.begin(), actual.end()); + EXPECT_EQ(actual, expected); + + for (size_t i = 0; i < 10; ++i) { + actual.emplace_back(static_cast(i)); + } + + expected = {0, 1, 4, 5, 6, 7, 8, 9}; + actual.erase(actual.begin() + 2, actual.begin() + 4); + EXPECT_EQ(actual, expected); + + expected = {0, 1, 4}; + actual.erase(actual.begin() + 3, actual.end()); + EXPECT_EQ(actual, expected); + + expected = {}; + actual.erase(actual.begin(), actual.end()); + EXPECT_EQ(actual, expected); + } + { + vectorized::PaddedPODArray actual {0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; + vectorized::PaddedPODArray expected; + + expected = {1, 2, 3, 4, 5, 6, 7, 8, 9}; + actual.erase(actual.begin()); + EXPECT_EQ(actual, expected); + } +} + +} // end namespace doris