Skip to content

Commit

Permalink
revert
Browse files Browse the repository at this point in the history
  • Loading branch information
@eldenmoon
eldenmoon committed Dec 20, 2024
1 parent 38b5a6a commit 2ef1bea
Show file tree
Hide file tree
Showing 5 changed files with 54 additions and 23 deletions.
19 changes: 16 additions & 3 deletions be/src/olap/rowset_builder.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -399,7 +399,6 @@ Status BaseRowsetBuilder::cancel() {
void BaseRowsetBuilder::_build_current_tablet_schema(int64_t index_id,
const OlapTableSchemaParam* table_schema_param,
const TabletSchema& ori_tablet_schema) {
_tablet_schema->copy_from(ori_tablet_schema);
// find the right index id
int i = 0;
auto indexes = table_schema_param->indexes();
Expand All @@ -408,14 +407,28 @@ void BaseRowsetBuilder::_build_current_tablet_schema(int64_t index_id,
break;
}
}

if (!indexes.empty() && !indexes[i]->columns.empty() &&
indexes[i]->columns[0]->unique_id() >= 0) {
_tablet_schema->shawdow_copy_without_columns(ori_tablet_schema);
_tablet_schema->build_current_tablet_schema(index_id, table_schema_param->version(),
indexes[i], ori_tablet_schema);
} else {
_tablet_schema->copy_from(ori_tablet_schema);
}
if (_tablet_schema->schema_version() > ori_tablet_schema.schema_version()) {
_tablet->update_max_version_schema(_tablet_schema);
// After schema change, should include extracted column
// For example: a table has two columns, k and v
// After adding a column v2, the schema version increases, max_version_schema needs to be updated.
// _tablet_schema includes k, v, and v2
// if v is a variant, need to add the columns decomposed from the v to the _tablet_schema.
if (_tablet_schema->num_variant_columns() > 0) {
TabletSchemaSPtr max_version_schema = std::make_shared<TabletSchema>();
max_version_schema->copy_from(*_tablet_schema);
max_version_schema->copy_extracted_columns(ori_tablet_schema);
_tablet->update_max_version_schema(max_version_schema);
} else {
_tablet->update_max_version_schema(_tablet_schema);
}
}

_tablet_schema->set_table_id(table_schema_param->table_id());
Expand Down
17 changes: 17 additions & 0 deletions be/src/olap/tablet_schema.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1222,6 +1222,23 @@ bool TabletSchema::is_dropped_column(const TabletColumn& col) const {
return it == _field_name_to_index.end() || _cols[it->second]->unique_id() != col.unique_id();
}

void TabletSchema::copy_extracted_columns(const TabletSchema& src_schema) {
std::unordered_set<int32_t> variant_columns;
for (const auto& col : columns()) {
if (col->is_variant_type()) {
variant_columns.insert(col->unique_id());
}
}
for (const TabletColumnPtr& col : src_schema.columns()) {
if (col->is_extracted_column() && variant_columns.contains(col->parent_unique_id())) {
ColumnPB col_pb;
col->to_schema_pb(&col_pb);
TabletColumn new_col(col_pb);
append_column(new_col, ColumnType::VARIANT);
}
}
}

void TabletSchema::reserve_extracted_columns() {
for (auto it = _cols.begin(); it != _cols.end();) {
if (!(*it)->is_extracted_column()) {
Expand Down
3 changes: 3 additions & 0 deletions be/src/olap/tablet_schema.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -462,6 +462,9 @@ class TabletSchema : public MetadataAdder<TabletSchema> {

bool is_dropped_column(const TabletColumn& col) const;

// copy extracted columns from src_schema
void copy_extracted_columns(const TabletSchema& src_schema);

// only reserve extracted columns
void reserve_extracted_columns();

Expand Down
30 changes: 15 additions & 15 deletions regression-test/data/variant_p0/desc.out
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,21 +3,14 @@
k bigint Yes true \N
v variant Yes false \N NONE
v.a smallint Yes false \N NONE
v.b json Yes false \N NONE
v.c.c smallint Yes false \N NONE
v.c.e double Yes false \N NONE
v.xxxx text Yes false \N NONE

-- !sql_2 --
k bigint Yes true \N
v variant Yes false \N NONE
v.a smallint Yes false \N NONE
v.b json Yes false \N NONE
v.c.c smallint Yes false \N NONE
v.c.e double Yes false \N NONE
v.ddd.aaa tinyint Yes false \N NONE
v.ddd.mxmxm json Yes false \N NONE
v.oooo.xxxx.xxx tinyint Yes false \N NONE
v.xxxx text Yes false \N NONE

-- !sql_3 --
Expand All @@ -33,21 +26,14 @@ v.xxxx text Yes false \N NONE
k bigint Yes true \N
v variant Yes false \N NONE
v.a smallint Yes false \N NONE
v.b json Yes false \N NONE
v.c.c smallint Yes false \N NONE
v.c.e double Yes false \N NONE
v.ddd.aaa tinyint Yes false \N NONE
v.ddd.mxmxm json Yes false \N NONE
v.oooo.xxxx.xxx tinyint Yes false \N NONE
v.xxxx text Yes false \N NONE

-- !sql_6_2 --
k bigint Yes true \N
v variant Yes false \N NONE
v.a smallint Yes false \N NONE
v.b json Yes false \N NONE
v.c.c smallint Yes false \N NONE
v.c.e double Yes false \N NONE
v.xxxx text Yes false \N NONE

-- !sql_6_3 --
Expand All @@ -67,7 +53,6 @@ v.c.c smallint Yes false \N NONE
v.c.e double Yes false \N NONE
v.ddd.aaa tinyint Yes false \N NONE
v.ddd.mxmxm json Yes false \N NONE
v.oooo.xxxx.xxx tinyint Yes false \N NONE
v.xxxx text Yes false \N NONE

-- !sql_7 --
Expand Down Expand Up @@ -164,11 +149,21 @@ v2.oooo.xxxx.xxx tinyint Yes false \N NONE
-- !sql_10_2 --
k bigint Yes true \N
v variant Yes false \N NONE
v.a smallint Yes false \N NONE
v.b json Yes false \N NONE
v.c.c smallint Yes false \N NONE
v.c.e double Yes false \N NONE
v.k1 tinyint Yes false \N NONE
v.k2 text Yes false \N NONE
v.k3 array<smallint> Yes false \N NONE
v.k4 double Yes false \N NONE
v.k5 json Yes false \N NONE
v.oooo.xxxx.xxx tinyint Yes false \N NONE
v2.a smallint Yes false \N NONE
v2.b json Yes false \N NONE
v2.c.c smallint Yes false \N NONE
v2.c.e double Yes false \N NONE
v2.oooo.xxxx.xxx tinyint Yes false \N NONE

-- !sql_10_3 --
k bigint Yes true \N
Expand All @@ -178,6 +173,11 @@ v.a smallint Yes false \N NONE
v.b json Yes false \N NONE
v.c.c smallint Yes false \N NONE
v.c.e double Yes false \N NONE
v.k1 tinyint Yes false \N NONE
v.k2 text Yes false \N NONE
v.k3 array<smallint> Yes false \N NONE
v.k4 double Yes false \N NONE
v.k5 json Yes false \N NONE
v.oooo.xxxx.xxx tinyint Yes false \N NONE
v3.a smallint Yes false \N NONE
v3.b json Yes false \N NONE
Expand Down
8 changes: 3 additions & 5 deletions regression-test/suites/variant_p0/desc.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,7 @@ suite("regression_test_variant_desc", "nonConcurrent"){
// sparse columns
def table_name = "sparse_columns"
create_table table_name
set_be_config.call("variant_ratio_of_defaults_as_sparse_column", "1")
set_be_config.call("variant_ratio_of_defaults_as_sparse_column", "0.95")
sql """set describe_extend_variant_column = true"""
sql """insert into sparse_columns select 0, '{"a": 11245, "b" : [123, {"xx" : 1}], "c" : {"c" : 456, "d" : null, "e" : 7.111}}' as json_str
union all select 0, '{"a": 1123}' as json_str union all select 0, '{"a" : 1234, "xxxx" : "kaana"}' as json_str from numbers("number" = "4096") limit 4096 ;"""
Expand Down Expand Up @@ -126,7 +126,7 @@ suite("regression_test_variant_desc", "nonConcurrent"){
table_name = "partition_data"
create_table_partition.call(table_name, "4")
sql "set enable_two_phase_read_opt = false;"
set_be_config.call("variant_ratio_of_defaults_as_sparse_column", "1")
set_be_config.call("variant_ratio_of_defaults_as_sparse_column", "0.95")
sql """insert into ${table_name} select 2500, '{"a": 1123, "b" : [123, {"xx" : 1}], "c" : {"c" : 456, "d" : null, "e" : 7.111}, "zzz" : null, "oooo" : {"akakaka" : null, "xxxx" : {"xxx" : 123}}}' as json_str
union all select 2500, '{"a" : 1234, "xxxx" : "kaana", "ddd" : {"aaa" : 123, "mxmxm" : [456, "789"]}}' as json_str from numbers("number" = "4096") limit 4096 ;"""
sql """insert into ${table_name} select 45000, '{"a": 11245, "b" : [123, {"xx" : 1}], "c" : {"c" : 456, "d" : null, "e" : 7.111}}' as json_str
Expand Down Expand Up @@ -207,11 +207,9 @@ suite("regression_test_variant_desc", "nonConcurrent"){
sql """ insert into ${table_name} values (0, '{"a": 1123, "b" : [123, {"xx" : 1}], "c" : {"c" : 456, "d" : null, "e" : 7.111}, "zzz" : null, "oooo" : {"akakaka" : null, "xxxx" : {"xxx" : 123}}}',
'{"a": 1123, "b" : [123, {"xx" : 1}], "c" : {"c" : 456, "d" : null, "e" : 7.111}, "zzz" : null, "oooo" : {"akakaka" : null, "xxxx" : {"xxx" : 123}}}')"""
sql "select * from ${table_name} limit 1"
sql """INSERT INTO ${table_name} (k, v) values(0, '{"k1":1, "k2": "hello world", "k3" : [1234], "k4" : 1.10000, "k5" : [[123]]}')"""
qt_sql_10_1 """desc ${table_name}"""
// drop cloumn
sql "alter table ${table_name} drop column v2"
sql """INSERT INTO ${table_name} (k, v) values(0, '{"k1":1, "k2": "hello world", "k3" : [1234], "k4" : 1.10000, "k5" : [[123]]}')"""
qt_sql_10_2 """desc ${table_name}"""
// add column
sql "alter table ${table_name} add column v3 variant default null"
Expand Down Expand Up @@ -276,6 +274,6 @@ suite("regression_test_variant_desc", "nonConcurrent"){
sql "desc large_tablets"
} finally {
// reset flags
set_be_config.call("variant_ratio_of_defaults_as_sparse_column", "1")
set_be_config.call("variant_ratio_of_defaults_as_sparse_column", "0.95")
}
}

0 comments on commit 2ef1bea

Please sign in to comment.