diff --git a/be/src/olap/rowset_builder.cpp b/be/src/olap/rowset_builder.cpp index 1e1e467e743660..2b214d57dbea60 100644 --- a/be/src/olap/rowset_builder.cpp +++ b/be/src/olap/rowset_builder.cpp @@ -415,7 +415,19 @@ void BaseRowsetBuilder::_build_current_tablet_schema(int64_t index_id, indexes[i], ori_tablet_schema); } if (_tablet_schema->schema_version() > ori_tablet_schema.schema_version()) { - _tablet->update_max_version_schema(_tablet_schema); + // After schema change, should include extracted column + // For example: a table has two columns, k and v + // After adding a column v2, the schema version increases, max_version_schema needs to be updated. + // _tablet_schema includes k, v, and v2 + // if v is a variant, need to add the columns decomposed from the v to the _tablet_schema. + if (_tablet_schema->num_variant_columns() > 0) { + TabletSchemaSPtr max_version_schema = std::make_shared(); + max_version_schema->copy_from(*_tablet_schema); + max_version_schema->copy_extracted_columns(ori_tablet_schema); + _tablet->update_max_version_schema(max_version_schema); + } else { + _tablet->update_max_version_schema(_tablet_schema); + } } _tablet_schema->set_table_id(table_schema_param->table_id()); diff --git a/be/src/olap/tablet_schema.cpp b/be/src/olap/tablet_schema.cpp index d13606d1adb3ca..7b6b5f313c144e 100644 --- a/be/src/olap/tablet_schema.cpp +++ b/be/src/olap/tablet_schema.cpp @@ -1222,6 +1222,23 @@ bool TabletSchema::is_dropped_column(const TabletColumn& col) const { return it == _field_name_to_index.end() || _cols[it->second]->unique_id() != col.unique_id(); } +void TabletSchema::copy_extracted_columns(const TabletSchema& src_schema) { + std::unordered_set variant_columns; + for (const auto& col : columns()) { + if (col->is_variant_type()) { + variant_columns.insert(col->unique_id()); + } + } + for (const TabletColumnPtr& col : src_schema.columns()) { + if (col->is_extracted_column() && variant_columns.contains(col->parent_unique_id())) { + ColumnPB col_pb; + col->to_schema_pb(&col_pb); + TabletColumn new_col(col_pb); + append_column(new_col, ColumnType::VARIANT); + } + } +} + void TabletSchema::reserve_extracted_columns() { for (auto it = _cols.begin(); it != _cols.end();) { if (!(*it)->is_extracted_column()) { diff --git a/be/src/olap/tablet_schema.h b/be/src/olap/tablet_schema.h index 1e3eeac04b88f3..3dfe055fbf4a89 100644 --- a/be/src/olap/tablet_schema.h +++ b/be/src/olap/tablet_schema.h @@ -462,6 +462,9 @@ class TabletSchema : public MetadataAdder { bool is_dropped_column(const TabletColumn& col) const; + // copy extracted columns from src_schema + void copy_extracted_columns(const TabletSchema& src_schema); + // only reserve extracted columns void reserve_extracted_columns(); diff --git a/regression-test/data/variant_p0/desc.out b/regression-test/data/variant_p0/desc.out index 9a483e642b6c35..ffd87b9d05bc37 100644 --- a/regression-test/data/variant_p0/desc.out +++ b/regression-test/data/variant_p0/desc.out @@ -3,21 +3,14 @@ k bigint Yes true \N v variant Yes false \N NONE v.a smallint Yes false \N NONE -v.b json Yes false \N NONE -v.c.c smallint Yes false \N NONE -v.c.e double Yes false \N NONE v.xxxx text Yes false \N NONE -- !sql_2 -- k bigint Yes true \N v variant Yes false \N NONE v.a smallint Yes false \N NONE -v.b json Yes false \N NONE -v.c.c smallint Yes false \N NONE -v.c.e double Yes false \N NONE v.ddd.aaa tinyint Yes false \N NONE v.ddd.mxmxm json Yes false \N NONE -v.oooo.xxxx.xxx tinyint Yes false \N NONE v.xxxx text Yes false \N NONE -- !sql_3 -- @@ -33,21 +26,14 @@ v.xxxx text Yes false \N NONE k bigint Yes true \N v variant Yes false \N NONE v.a smallint Yes false \N NONE -v.b json Yes false \N NONE -v.c.c smallint Yes false \N NONE -v.c.e double Yes false \N NONE v.ddd.aaa tinyint Yes false \N NONE v.ddd.mxmxm json Yes false \N NONE -v.oooo.xxxx.xxx tinyint Yes false \N NONE v.xxxx text Yes false \N NONE -- !sql_6_2 -- k bigint Yes true \N v variant Yes false \N NONE v.a smallint Yes false \N NONE -v.b json Yes false \N NONE -v.c.c smallint Yes false \N NONE -v.c.e double Yes false \N NONE v.xxxx text Yes false \N NONE -- !sql_6_3 -- @@ -67,7 +53,6 @@ v.c.c smallint Yes false \N NONE v.c.e double Yes false \N NONE v.ddd.aaa tinyint Yes false \N NONE v.ddd.mxmxm json Yes false \N NONE -v.oooo.xxxx.xxx tinyint Yes false \N NONE v.xxxx text Yes false \N NONE -- !sql_7 -- @@ -164,11 +149,21 @@ v2.oooo.xxxx.xxx tinyint Yes false \N NONE -- !sql_10_2 -- k bigint Yes true \N v variant Yes false \N NONE +v.a smallint Yes false \N NONE +v.b json Yes false \N NONE +v.c.c smallint Yes false \N NONE +v.c.e double Yes false \N NONE v.k1 tinyint Yes false \N NONE v.k2 text Yes false \N NONE v.k3 array Yes false \N NONE v.k4 double Yes false \N NONE v.k5 json Yes false \N NONE +v.oooo.xxxx.xxx tinyint Yes false \N NONE +v2.a smallint Yes false \N NONE +v2.b json Yes false \N NONE +v2.c.c smallint Yes false \N NONE +v2.c.e double Yes false \N NONE +v2.oooo.xxxx.xxx tinyint Yes false \N NONE -- !sql_10_3 -- k bigint Yes true \N @@ -178,6 +173,11 @@ v.a smallint Yes false \N NONE v.b json Yes false \N NONE v.c.c smallint Yes false \N NONE v.c.e double Yes false \N NONE +v.k1 tinyint Yes false \N NONE +v.k2 text Yes false \N NONE +v.k3 array Yes false \N NONE +v.k4 double Yes false \N NONE +v.k5 json Yes false \N NONE v.oooo.xxxx.xxx tinyint Yes false \N NONE v3.a smallint Yes false \N NONE v3.b json Yes false \N NONE diff --git a/regression-test/suites/variant_p0/desc.groovy b/regression-test/suites/variant_p0/desc.groovy index dea1a9be724540..5efcda3a04308e 100644 --- a/regression-test/suites/variant_p0/desc.groovy +++ b/regression-test/suites/variant_p0/desc.groovy @@ -97,7 +97,7 @@ suite("regression_test_variant_desc", "nonConcurrent"){ // sparse columns def table_name = "sparse_columns" create_table table_name - set_be_config.call("variant_ratio_of_defaults_as_sparse_column", "1") + set_be_config.call("variant_ratio_of_defaults_as_sparse_column", "0.95") sql """set describe_extend_variant_column = true""" sql """insert into sparse_columns select 0, '{"a": 11245, "b" : [123, {"xx" : 1}], "c" : {"c" : 456, "d" : null, "e" : 7.111}}' as json_str union all select 0, '{"a": 1123}' as json_str union all select 0, '{"a" : 1234, "xxxx" : "kaana"}' as json_str from numbers("number" = "4096") limit 4096 ;""" @@ -126,7 +126,7 @@ suite("regression_test_variant_desc", "nonConcurrent"){ table_name = "partition_data" create_table_partition.call(table_name, "4") sql "set enable_two_phase_read_opt = false;" - set_be_config.call("variant_ratio_of_defaults_as_sparse_column", "1") + set_be_config.call("variant_ratio_of_defaults_as_sparse_column", "0.95") sql """insert into ${table_name} select 2500, '{"a": 1123, "b" : [123, {"xx" : 1}], "c" : {"c" : 456, "d" : null, "e" : 7.111}, "zzz" : null, "oooo" : {"akakaka" : null, "xxxx" : {"xxx" : 123}}}' as json_str union all select 2500, '{"a" : 1234, "xxxx" : "kaana", "ddd" : {"aaa" : 123, "mxmxm" : [456, "789"]}}' as json_str from numbers("number" = "4096") limit 4096 ;""" sql """insert into ${table_name} select 45000, '{"a": 11245, "b" : [123, {"xx" : 1}], "c" : {"c" : 456, "d" : null, "e" : 7.111}}' as json_str @@ -207,11 +207,9 @@ suite("regression_test_variant_desc", "nonConcurrent"){ sql """ insert into ${table_name} values (0, '{"a": 1123, "b" : [123, {"xx" : 1}], "c" : {"c" : 456, "d" : null, "e" : 7.111}, "zzz" : null, "oooo" : {"akakaka" : null, "xxxx" : {"xxx" : 123}}}', '{"a": 1123, "b" : [123, {"xx" : 1}], "c" : {"c" : 456, "d" : null, "e" : 7.111}, "zzz" : null, "oooo" : {"akakaka" : null, "xxxx" : {"xxx" : 123}}}')""" sql "select * from ${table_name} limit 1" - sql """INSERT INTO ${table_name} (k, v) values(0, '{"k1":1, "k2": "hello world", "k3" : [1234], "k4" : 1.10000, "k5" : [[123]]}')""" qt_sql_10_1 """desc ${table_name}""" // drop cloumn sql "alter table ${table_name} drop column v2" - sql """INSERT INTO ${table_name} (k, v) values(0, '{"k1":1, "k2": "hello world", "k3" : [1234], "k4" : 1.10000, "k5" : [[123]]}')""" qt_sql_10_2 """desc ${table_name}""" // add column sql "alter table ${table_name} add column v3 variant default null" @@ -276,6 +274,6 @@ suite("regression_test_variant_desc", "nonConcurrent"){ sql "desc large_tablets" } finally { // reset flags - set_be_config.call("variant_ratio_of_defaults_as_sparse_column", "1") + set_be_config.call("variant_ratio_of_defaults_as_sparse_column", "0.95") } }