[Fix](JsonReader) fix json with duplicate key entry may result out of…

… bound exception
eldenmoon · Jul 19, 2024 · 96f97c9 · 96f97c9
1 parent 5670e1a
commit 96f97c9
Show file tree

Hide file tree

Showing 3 changed files with 31 additions and 23 deletions.
diff --git a/be/src/vec/exec/format/json/new_json_reader.cpp b/be/src/vec/exec/format/json/new_json_reader.cpp
@@ -1348,6 +1348,9 @@ Status NewJsonReader::_simdjson_set_column_value(simdjson::ondemand::object* val
             // This key is not exist in slot desc, just ignore
             continue;
         }
+        if (_seen_columns[column_index]) {
+            continue;
+        }
         simdjson::ondemand::value val = field.value();
         auto* column_ptr = block.get_by_position(column_index).column->assume_mutable().get();
         RETURN_IF_ERROR(

diff --git a/regression-test/data/load_p0/stream_load/test_json_load.out b/regression-test/data/load_p0/stream_load/test_json_load.out
@@ -205,29 +205,6 @@ John	30	New York	{"email":"[email protected]","phone":"+1-123-456-7890"}
 -- !select22 --
 11324	1321313082437	1678834024274	20230315	{"base_mac_value_null":24,"base_1_value_respiratoryrate":11,"base_3_value_heartrate":51,"base_3_status_onoroutofbed":3,"base_null_count_circulation":84,"base_1_status_onoroutofbed":3,"base_1_value_heartrate":51,"base_3_value_respiratoryrate":11,"base_3_value_bodyactivityenergy":43652,"base_2_value_respiratoryrate":11,"base_2_value_bodyactivityenergy":28831,"base_2_status_onoroutofbed":3,"base_1_value_bodyactivityenergy":56758,"base_2_value_heartrate":51,"tsltype":"properties","datatimestamp":1678834024274,"command":"0105","macaddress":"405EE1805029"}
 
--- !select --
-2	shanghai	2345672
-3	guangzhou	2345673
-4	shenzhen	2345674
-5	hangzhou	2345675
-6	nanjing	2345676
-7	wuhan	2345677
-8	chengdu	2345678
-9	xian	2345679
-200	changsha	3456789
-
--- !select --
-2	shanghai	2345672
-3	guangzhou	2345673
-4	shenzhen	2345674
-200	changsha	3456789
-
--- !select --
-2	shanghai	2345672
-3	guangzhou	2345673
-4	shenzhen	2345674
-200	changsha	3456789
-
 -- !select26 --
 10	1454547
 20	1244264
@@ -248,3 +225,6 @@ android	\N	\N	\N	\N	\N
 -- !select28 --
 test	k2_value
 
+-- !select28 --
+10	\N
+
diff --git a/regression-test/suites/load_p0/stream_load/test_json_load.groovy b/regression-test/suites/load_p0/stream_load/test_json_load.groovy
@@ -853,4 +853,29 @@ suite("test_json_load", "p0,nonConcurrent") {
     } finally {
         try_sql("DROP TABLE IF EXISTS ${testTable}")
     }
+
+    // add duplicate json entry case
+    try {
+        sql "DROP TABLE IF EXISTS ${testTable}"
+        sql """CREATE TABLE IF NOT EXISTS ${testTable} 
+            (
+                `k1` varchar(1024) NULL,
+                `k2` varchar(1024) NULL
+            )
+            DUPLICATE KEY(`k1`)
+            COMMENT ''
+            DISTRIBUTED BY RANDOM BUCKETS 1
+            PROPERTIES (
+            "replication_allocation" = "tag.location.default: 1"
+            );"""
+
+        load_json_data.call("${testTable}", "${testTable}_case29", 'false', 'true', 'json', '', '',
+                             '', '', '', 'test_duplicate_json_keys.json', false, 1)
+
+        sql "sync"
+        qt_select28 "select * from ${testTable}"
+
+    } finally {
+        try_sql("DROP TABLE IF EXISTS ${testTable}")
+    }
 }