diff --git a/be/src/vec/exec/format/json/new_json_reader.cpp b/be/src/vec/exec/format/json/new_json_reader.cpp index b658e15e467e4db..7d97aabe39ce3bf 100644 --- a/be/src/vec/exec/format/json/new_json_reader.cpp +++ b/be/src/vec/exec/format/json/new_json_reader.cpp @@ -1348,6 +1348,9 @@ Status NewJsonReader::_simdjson_set_column_value(simdjson::ondemand::object* val // This key is not exist in slot desc, just ignore continue; } + if (_seen_columns[column_index]) { + continue; + } simdjson::ondemand::value val = field.value(); auto* column_ptr = block.get_by_position(column_index).column->assume_mutable().get(); RETURN_IF_ERROR( diff --git a/regression-test/data/load_p0/stream_load/test_json_load.out b/regression-test/data/load_p0/stream_load/test_json_load.out index 9f064572f137b17..502d4d95621d2e2 100644 --- a/regression-test/data/load_p0/stream_load/test_json_load.out +++ b/regression-test/data/load_p0/stream_load/test_json_load.out @@ -205,29 +205,6 @@ John 30 New York {"email":"john@example.com","phone":"+1-123-456-7890"} -- !select22 -- 11324 1321313082437 1678834024274 20230315 {"base_mac_value_null":24,"base_1_value_respiratoryrate":11,"base_3_value_heartrate":51,"base_3_status_onoroutofbed":3,"base_null_count_circulation":84,"base_1_status_onoroutofbed":3,"base_1_value_heartrate":51,"base_3_value_respiratoryrate":11,"base_3_value_bodyactivityenergy":43652,"base_2_value_respiratoryrate":11,"base_2_value_bodyactivityenergy":28831,"base_2_status_onoroutofbed":3,"base_1_value_bodyactivityenergy":56758,"base_2_value_heartrate":51,"tsltype":"properties","datatimestamp":1678834024274,"command":"0105","macaddress":"405EE1805029"} --- !select -- -2 shanghai 2345672 -3 guangzhou 2345673 -4 shenzhen 2345674 -5 hangzhou 2345675 -6 nanjing 2345676 -7 wuhan 2345677 -8 chengdu 2345678 -9 xian 2345679 -200 changsha 3456789 - --- !select -- -2 shanghai 2345672 -3 guangzhou 2345673 -4 shenzhen 2345674 -200 changsha 3456789 - --- !select -- -2 shanghai 2345672 -3 guangzhou 2345673 -4 shenzhen 2345674 -200 changsha 3456789 - -- !select26 -- 10 1454547 20 1244264 @@ -248,3 +225,6 @@ android \N \N \N \N \N -- !select28 -- test k2_value +-- !select28 -- +10 \N + diff --git a/regression-test/suites/load_p0/stream_load/test_json_load.groovy b/regression-test/suites/load_p0/stream_load/test_json_load.groovy index 4cc737820367af3..3ad808484f0c72e 100644 --- a/regression-test/suites/load_p0/stream_load/test_json_load.groovy +++ b/regression-test/suites/load_p0/stream_load/test_json_load.groovy @@ -853,4 +853,29 @@ suite("test_json_load", "p0,nonConcurrent") { } finally { try_sql("DROP TABLE IF EXISTS ${testTable}") } + + // add duplicate json entry case + try { + sql "DROP TABLE IF EXISTS ${testTable}" + sql """CREATE TABLE IF NOT EXISTS ${testTable} + ( + `k1` varchar(1024) NULL, + `k2` varchar(1024) NULL + ) + DUPLICATE KEY(`k1`) + COMMENT '' + DISTRIBUTED BY RANDOM BUCKETS 1 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1" + );""" + + load_json_data.call("${testTable}", "${testTable}_case29", 'false', 'true', 'json', '', '', + '', '', '', 'test_duplicate_json_keys.json', false, 1) + + sql "sync" + qt_select28 "select * from ${testTable}" + + } finally { + try_sql("DROP TABLE IF EXISTS ${testTable}") + } }