[Fix](JsonReader) fix json with duplicate key entry may result out of…

… bound exception
eldenmoon · Jul 19, 2024 · c2e7c60 · c2e7c60
1 parent d4d1688
commit c2e7c60
Show file tree

Hide file tree

Showing 4 changed files with 31 additions and 0 deletions.
diff --git a/be/src/vec/exec/format/json/new_json_reader.cpp b/be/src/vec/exec/format/json/new_json_reader.cpp
@@ -1349,6 +1349,9 @@ Status NewJsonReader::_simdjson_set_column_value(simdjson::ondemand::object* val
             // This key is not exist in slot desc, just ignore
             continue;
         }
+        if (_seen_columns[column_index]) {
+            continue;
+        }
         simdjson::ondemand::value val = field.value();
         auto* column_ptr = block.get_by_position(column_index).column->assume_mutable().get();
         RETURN_IF_ERROR(

diff --git a/regression-test/data/load_p0/stream_load/test_duplicate_json_keys.json b/regression-test/data/load_p0/stream_load/test_duplicate_json_keys.json
@@ -0,0 +1 @@
+{"k1" : 10, "k1" : 100}
diff --git a/regression-test/data/load_p0/stream_load/test_json_load.out b/regression-test/data/load_p0/stream_load/test_json_load.out
@@ -248,3 +248,5 @@ android	\N	\N	\N	\N	\N
 -- !select28 --
 test	k2_value
 
+-- !select29 --
+10	\N
diff --git a/regression-test/suites/load_p0/stream_load/test_json_load.groovy b/regression-test/suites/load_p0/stream_load/test_json_load.groovy
@@ -853,4 +853,29 @@ suite("test_json_load", "p0") {
     } finally {
         try_sql("DROP TABLE IF EXISTS ${testTable}")
     }
+
+    // add duplicate json entry case
+    try {
+        sql "DROP TABLE IF EXISTS ${testTable}"
+        sql """CREATE TABLE IF NOT EXISTS ${testTable} 
+            (
+                `k1` varchar(1024) NULL,
+                `k2` varchar(1024) NULL
+            )
+            DUPLICATE KEY(`k1`)
+            COMMENT ''
+            DISTRIBUTED BY RANDOM BUCKETS 1
+            PROPERTIES (
+            "replication_allocation" = "tag.location.default: 1"
+            );"""
+
+        load_json_data.call("${testTable}", "${testTable}_case29", 'false', 'true', 'json', '', '',
+                             '', '', '', 'test_duplicate_json_keys.json', false, 1)
+
+        sql "sync"
+        qt_select29 "select * from ${testTable}"
+
+    } finally {
+        try_sql("DROP TABLE IF EXISTS ${testTable}")
+    }
 }