[fix](string64) fix coredump caused by ColumnArray<ColumnStr<uint64_t>>::insert_indices_from

be/src/vec/columns/column_array.cpp

@@ -389,7 +389,8 @@ void ColumnArray::insert_from(const IColumn& src_, size_t n) {
 
     if (!get_data().is_nullable() && src.get_data().is_nullable()) {
         // Note: we can't process the case of 'Array(Nullable(nest))'
-        DCHECK(false);
+        throw Exception(ErrorCode::INTERNAL_ERROR, "insert '{}' into '{}'", src.get_name(),
+                        get_name());
     } else if (get_data().is_nullable() && !src.get_data().is_nullable()) {
         // Note: here we should process the case of 'Array(NotNullable(nest))'
         reinterpret_cast<ColumnNullable*>(&get_data())

be/src/vec/columns/column_string.cpp

@@ -22,7 +22,6 @@
 
 #include <algorithm>
 #include <boost/iterator/iterator_facade.hpp>
-#include <ostream>
 
 #include "util/memcpy_inlined.h"
 #include "util/simd/bits.h"
@@ -94,6 +93,13 @@ MutableColumnPtr ColumnStr<T>::get_shrinked_column() {
     return shrinked_column;
 }
 
+// This method is only called by MutableBlock::merge_ignore_overflow
+// by hash join operator to collect build data to avoid
+// the total string length of a ColumnStr<uint32_t> column exceeds the 4G limit.
+//
+// After finishing collecting build data, a ColumnStr<uint32_t> column
+// will be converted to ColumnStr<uint64_t> if the total string length
+// exceeds the 4G limit by calling Block::replace_if_overflow.
 template <typename T>
 void ColumnStr<T>::insert_range_from_ignore_overflow(const doris::vectorized::IColumn& src,
                                                      size_t start, size_t length) {
@@ -123,6 +129,8 @@ void ColumnStr<T>::insert_range_from_ignore_overflow(const doris::vectorized::IC
         offsets.resize(old_size + length);
 
         for (size_t i = 0; i < length; ++i) {
+            // unsinged integer overflow is well defined in C++,
+            // so we don't need to check the overflow here.
             offsets[old_size + i] =
                     src_concrete.offsets[start + i] - nested_offset + prev_max_offset;
         }
@@ -134,34 +142,41 @@ void ColumnStr<T>::insert_range_from(const IColumn& src, size_t start, size_t le
     if (length == 0) {
         return;
     }
+    auto do_insert = [&](const auto& src_concrete) {
+        const auto& src_offsets = src_concrete.get_offsets();
+        const auto& src_chars = src_concrete.get_chars();
+        if (start + length > src_offsets.size()) {
+            throw doris::Exception(
+                    doris::ErrorCode::INTERNAL_ERROR,
+                    "Parameter out of bound in IColumnStr<T>::insert_range_from method.");
+        }
+        size_t nested_offset = src_offsets[static_cast<ssize_t>(start) - 1];
+        size_t nested_length = src_offsets[start + length - 1] - nested_offset;
 
-    const auto& src_concrete = assert_cast<const ColumnStr<T>&>(src);
-
-    if (start + length > src_concrete.offsets.size()) {
-        throw doris::Exception(
-                doris::ErrorCode::INTERNAL_ERROR,
-                "Parameter out of bound in IColumnStr<T>::insert_range_from method.");
-    }
-
-    size_t nested_offset = src_concrete.offset_at(start);
-    size_t nested_length = src_concrete.offsets[start + length - 1] - nested_offset;
-
-    size_t old_chars_size = chars.size();
-    check_chars_length(old_chars_size + nested_length, offsets.size() + length);
-    chars.resize(old_chars_size + nested_length);
-    memcpy(&chars[old_chars_size], &src_concrete.chars[nested_offset], nested_length);
+        size_t old_chars_size = chars.size();
+        check_chars_length(old_chars_size + nested_length, offsets.size() + length);
+        chars.resize(old_chars_size + nested_length);
+        memcpy(&chars[old_chars_size], &src_chars[nested_offset], nested_length);
 
-    if (start == 0 && offsets.empty()) {
-        offsets.assign(src_concrete.offsets.begin(), src_concrete.offsets.begin() + length);
-    } else {
-        size_t old_size = offsets.size();
-        size_t prev_max_offset = offsets.back(); /// -1th index is Ok, see PaddedPODArray
-        offsets.resize(old_size + length);
+        using OffsetsType = std::decay_t<decltype(src_offsets)>;
+        if (std::is_same_v<T, typename OffsetsType::value_type> && start == 0 && offsets.empty()) {
+            offsets.assign(src_offsets.begin(), src_offsets.begin() + length);
+        } else {
+            size_t old_size = offsets.size();
+            size_t prev_max_offset = offsets.back(); /// -1th index is Ok, see PaddedPODArray
+            offsets.resize(old_size + length);
 
-        for (size_t i = 0; i < length; ++i) {
-            offsets[old_size + i] =
-                    src_concrete.offsets[start + i] - nested_offset + prev_max_offset;
+            for (size_t i = 0; i < length; ++i) {
+                offsets[old_size + i] = src_offsets[start + i] - nested_offset + prev_max_offset;
+            }
         }
+    };
+    // insert_range_from maybe called by ColumnArray::insert_indices_from(which is used by hash join operator),
+    // so we need to support both ColumnStr<uint32_t> and ColumnStr<uint64_t>
+    if (src.is_column_string64()) {
+        do_insert(assert_cast<const ColumnStr<uint64_t>&>(src));
+    } else {
+        do_insert(assert_cast<const ColumnStr<uint32_t>&>(src));
     }
 }
 

be/test/vec/core/column_array_test.cpp

@@ -108,6 +108,53 @@ TEST(ColumnArrayTest, StringArrayTest) {
     }
 }
 
+TEST(ColumnArrayTest, String64ArrayTest) {
+    auto off_column = ColumnVector<ColumnArray::Offset64>::create();
+    auto str64_column = ColumnString64::create();
+    // init column array with [["abc","d"],["ef"],[], [""]];
+    std::vector<ColumnArray::Offset64> offs = {0, 2, 3, 3, 4};
+    std::vector<std::string> vals = {"abc", "d", "ef", ""};
+    for (size_t i = 1; i < offs.size(); ++i) {
+        off_column->insert_data((const char*)(&offs[i]), 0);
+    }
+    for (auto& v : vals) {
+        str64_column->insert_data(v.data(), v.size());
+    }
+
+    ColumnArray str64_array_column(std::move(str64_column), std::move(off_column));
+    EXPECT_EQ(str64_array_column.size(), offs.size() - 1);
+    for (size_t i = 0; i < str64_array_column.size(); ++i) {
+        auto v = get<Array>(str64_array_column[i]);
+        EXPECT_EQ(v.size(), offs[i + 1] - offs[i]);
+        for (size_t j = 0; j < v.size(); ++j) {
+            EXPECT_EQ(vals[offs[i] + j], get<std::string>(v[j]));
+        }
+    }
+    // test insert ColumnArray<ColumnStr<uint64_t>> into ColumnArray<ColumnStr<uint32_t>>
+    auto str32_column = ColumnString::create();
+    ColumnArray str32_array_column(std::move(str32_column));
+    std::vector<uint32_t> indices;
+    indices.push_back(0);
+    indices.push_back(1);
+    indices.push_back(3);
+    str32_array_column.insert_indices_from(str64_array_column, indices.data(),
+                                           indices.data() + indices.size());
+    EXPECT_EQ(str32_array_column.size(), 3);
+
+    auto v = get<Array>(str32_array_column[0]);
+    EXPECT_EQ(v.size(), 2);
+    EXPECT_EQ(get<std::string>(v[0]), vals[0]);
+    EXPECT_EQ(get<std::string>(v[1]), vals[1]);
+
+    v = get<Array>(str32_array_column[1]);
+    EXPECT_EQ(v.size(), 1);
+    EXPECT_EQ(get<std::string>(v[0]), vals[2]);
+
+    v = get<Array>(str32_array_column[2]);
+    EXPECT_EQ(v.size(), 1);
+    EXPECT_EQ(get<std::string>(v[0]), vals[3]);
+}
+
 TEST(ColumnArrayTest, IntArrayPermuteTest) {
     auto off_column = ColumnVector<ColumnArray::Offset64>::create();
     auto data_column = ColumnVector<int32_t>::create();

be/test/vec/core/column_map_test.cpp

@@ -0,0 +1,155 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "vec/columns/column_map.h"
+
+#include <gtest/gtest-message.h>
+#include <gtest/gtest-test-part.h>
+
+#include "gtest/gtest_pred_impl.h"
+#include "vec/columns/column.h"
+#include "vec/columns/column_string.h"
+#include "vec/columns/column_vector.h"
+#include "vec/core/field.h"
+
+namespace doris::vectorized {
+TEST(ColumnMapTest, StringKeyTest) {
+    auto col_map_str64 = ColumnMap(ColumnString64::create(), ColumnInt64::create(),
+                                   ColumnArray::ColumnOffsets::create());
+    Array k1 = {"a", "b", "c"};
+    Array v1 = {1, 2, 3};
+    {
+        Map map;
+        map.push_back(k1);
+        map.push_back(v1);
+        col_map_str64.insert(map);
+    }
+    Array k2 = {"aa", "bb", "cc"};
+    Array v2 = {11, 22, 33};
+    {
+        Map map;
+        map.push_back(k2);
+        map.push_back(v2);
+        col_map_str64.insert(map);
+    }
+    Array k3 = {"aaa", "bbb", "ccc"};
+    Array v3 = {111, 222, 333};
+    {
+        Map map;
+        map.push_back(k3);
+        map.push_back(v3);
+        col_map_str64.insert(map);
+    }
+
+    // test insert ColumnMap<ColumnStr<uint64_t>, Column> into ColumnMap<ColumnStr<uint32_t>, Column>
+    auto col_map_str32 = ColumnMap(ColumnString::create(), ColumnInt64::create(),
+                                   ColumnArray::ColumnOffsets::create());
+    std::vector<uint32_t> indices;
+    indices.push_back(0);
+    indices.push_back(2);
+    col_map_str32.insert_indices_from(col_map_str64, indices.data(),
+                                      indices.data() + indices.size());
+    EXPECT_EQ(col_map_str32.size(), 2);
+
+    auto map = get<Map>(col_map_str32[0]);
+    auto k = get<Array>(map[0]);
+    auto v = get<Array>(map[1]);
+    EXPECT_EQ(k.size(), 3);
+    for (size_t i = 0; i < k.size(); ++i) {
+        EXPECT_EQ(k[i], k1[i]);
+    }
+    EXPECT_EQ(v.size(), 3);
+    for (size_t i = 0; i < v.size(); ++i) {
+        EXPECT_EQ(v[i], v1[i]);
+    }
+
+    map = get<Map>(col_map_str32[1]);
+    k = get<Array>(map[0]);
+    v = get<Array>(map[1]);
+    EXPECT_EQ(k.size(), 3);
+    for (size_t i = 0; i < k.size(); ++i) {
+        EXPECT_EQ(k[i], k3[i]);
+    }
+    EXPECT_EQ(v.size(), 3);
+    for (size_t i = 0; i < v.size(); ++i) {
+        EXPECT_EQ(v[i], v3[i]);
+    }
+};
+
+TEST(ColumnMapTest, StringValueTest) {
+    auto col_map_str64 = ColumnMap(ColumnInt64::create(), ColumnString64::create(),
+                                   ColumnArray::ColumnOffsets::create());
+    Array k1 = {1, 2, 3};
+    Array v1 = {"a", "b", "c"};
+    {
+        Map map;
+        map.push_back(k1);
+        map.push_back(v1);
+        col_map_str64.insert(map);
+    }
+    Array k2 = {11, 22, 33};
+    Array v2 = {"aa", "bb", "cc"};
+    {
+        Map map;
+        map.push_back(k2);
+        map.push_back(v2);
+        col_map_str64.insert(map);
+    }
+    Array k3 = {111, 222, 333};
+    Array v3 = {"aaa", "bbb", "ccc"};
+    {
+        Map map;
+        map.push_back(k3);
+        map.push_back(v3);
+        col_map_str64.insert(map);
+    }
+
+    // test insert ColumnMap<ColumnStr<uint64_t>, Column> into ColumnMap<ColumnStr<uint32_t>, Column>
+    auto col_map_str32 = ColumnMap(ColumnInt64::create(), ColumnString::create(),
+                                   ColumnArray::ColumnOffsets::create());
+    std::vector<uint32_t> indices;
+    indices.push_back(0);
+    indices.push_back(2);
+    col_map_str32.insert_indices_from(col_map_str64, indices.data(),
+                                      indices.data() + indices.size());
+    EXPECT_EQ(col_map_str32.size(), 2);
+
+    auto map = get<Map>(col_map_str32[0]);
+    auto k = get<Array>(map[0]);
+    auto v = get<Array>(map[1]);
+    EXPECT_EQ(k.size(), 3);
+    for (size_t i = 0; i < k.size(); ++i) {
+        EXPECT_EQ(k[i], k1[i]);
+    }
+    EXPECT_EQ(v.size(), 3);
+    for (size_t i = 0; i < v.size(); ++i) {
+        EXPECT_EQ(v[i], v1[i]);
+    }
+
+    map = get<Map>(col_map_str32[1]);
+    k = get<Array>(map[0]);
+    v = get<Array>(map[1]);
+    EXPECT_EQ(k.size(), 3);
+    for (size_t i = 0; i < k.size(); ++i) {
+        EXPECT_EQ(k[i], k3[i]);
+    }
+    EXPECT_EQ(v.size(), 3);
+    for (size_t i = 0; i < v.size(); ++i) {
+        EXPECT_EQ(v[i], v3[i]);
+    }
+};
+} // namespace doris::vectorized

be/test/vec/core/column_string_test.cpp

@@ -19,6 +19,7 @@
 
 #include <gtest/gtest.h>
 
+#include "vec/columns/column.h"
 #include "vec/core/block.h"
 #include "vec/data_types/data_type_string.h"
 #include "vec/functions/function_string.h"
@@ -64,4 +65,41 @@ TEST(ColumnStringTest, TestConcat) {
     auto actual_res_col_str = assert_cast<const ColumnString*>(actual_res_col.get());
     actual_res_col_str->sanity_check();
 }
+
+TEST(ColumnStringTest, TestStringInsert) {
+    {
+        auto str32_column = ColumnString::create();
+        std::vector<std::string> vals_tmp = {"x", "yy", "zzz", ""};
+        auto str32_column_tmp = ColumnString::create();
+        for (auto& v : vals_tmp) {
+            str32_column_tmp->insert_data(v.data(), v.size());
+        }
+        str32_column->insert_range_from(*str32_column_tmp, 0, vals_tmp.size());
+        str32_column->insert_range_from(*str32_column_tmp, 0, vals_tmp.size());
+        auto row_count = str32_column->size();
+        EXPECT_EQ(row_count, vals_tmp.size() * 2);
+        for (size_t i = 0; i < row_count; ++i) {
+            auto row_data = str32_column->get_data_at(i);
+            EXPECT_EQ(row_data.to_string(), vals_tmp[i % vals_tmp.size()]);
+        }
+    }
+
+    {
+        // test insert ColumnString64 to ColumnString
+        auto str32_column = ColumnString::create();
+        std::vector<std::string> vals_tmp = {"x", "yy", "zzz", ""};
+        auto str64_column_tmp = ColumnString64::create();
+        for (auto& v : vals_tmp) {
+            str64_column_tmp->insert_data(v.data(), v.size());
+        }
+        str32_column->insert_range_from(*str64_column_tmp, 0, vals_tmp.size());
+        str32_column->insert_range_from(*str64_column_tmp, 0, vals_tmp.size());
+        auto row_count = str32_column->size();
+        EXPECT_EQ(row_count, vals_tmp.size() * 2);
+        for (size_t i = 0; i < row_count; ++i) {
+            auto row_data = str32_column->get_data_at(i);
+            EXPECT_EQ(row_data.to_string(), vals_tmp[i % vals_tmp.size()]);
+        }
+    }
+}
 } // namespace doris::vectorized