From c2e7c60bc6f795485c303cbf62aa9d181ecd4a65 Mon Sep 17 00:00:00 2001
From: eldenmoon <15605149486@163.com>
Date: Fri, 19 Jul 2024 17:30:54 +0800
Subject: [PATCH] [Fix](JsonReader) fix json with duplicate key entry may
 result out of bound exception

---
 .../vec/exec/format/json/new_json_reader.cpp  |  3 +++
 .../stream_load/test_duplicate_json_keys.json |  1 +
 .../load_p0/stream_load/test_json_load.out    |  2 ++
 .../load_p0/stream_load/test_json_load.groovy | 25 +++++++++++++++++++
 4 files changed, 31 insertions(+)
 create mode 100644 regression-test/data/load_p0/stream_load/test_duplicate_json_keys.json

diff --git a/be/src/vec/exec/format/json/new_json_reader.cpp b/be/src/vec/exec/format/json/new_json_reader.cpp
index 5636e2427618cf..fabb699d6f3f76 100644
--- a/be/src/vec/exec/format/json/new_json_reader.cpp
+++ b/be/src/vec/exec/format/json/new_json_reader.cpp
@@ -1349,6 +1349,9 @@ Status NewJsonReader::_simdjson_set_column_value(simdjson::ondemand::object* val
             // This key is not exist in slot desc, just ignore
             continue;
         }
+        if (_seen_columns[column_index]) {
+            continue;
+        }
         simdjson::ondemand::value val = field.value();
         auto* column_ptr = block.get_by_position(column_index).column->assume_mutable().get();
         RETURN_IF_ERROR(
diff --git a/regression-test/data/load_p0/stream_load/test_duplicate_json_keys.json b/regression-test/data/load_p0/stream_load/test_duplicate_json_keys.json
new file mode 100644
index 00000000000000..4bf1079cf22ffc
--- /dev/null
+++ b/regression-test/data/load_p0/stream_load/test_duplicate_json_keys.json
@@ -0,0 +1 @@
+{"k1" : 10, "k1" : 100}
\ No newline at end of file
diff --git a/regression-test/data/load_p0/stream_load/test_json_load.out b/regression-test/data/load_p0/stream_load/test_json_load.out
index 9f064572f137b1..588b6edb00463a 100644
--- a/regression-test/data/load_p0/stream_load/test_json_load.out
+++ b/regression-test/data/load_p0/stream_load/test_json_load.out
@@ -248,3 +248,5 @@ android	\N	\N	\N	\N	\N
 -- !select28 --
 test	k2_value
 
+-- !select29 --
+10	\N
diff --git a/regression-test/suites/load_p0/stream_load/test_json_load.groovy b/regression-test/suites/load_p0/stream_load/test_json_load.groovy
index da6bea1afcd48a..7f182a44b74204 100644
--- a/regression-test/suites/load_p0/stream_load/test_json_load.groovy
+++ b/regression-test/suites/load_p0/stream_load/test_json_load.groovy
@@ -853,4 +853,29 @@ suite("test_json_load", "p0") {
     } finally {
         try_sql("DROP TABLE IF EXISTS ${testTable}")
     }
+
+    // add duplicate json entry case
+    try {
+        sql "DROP TABLE IF EXISTS ${testTable}"
+        sql """CREATE TABLE IF NOT EXISTS ${testTable} 
+            (
+                `k1` varchar(1024) NULL,
+                `k2` varchar(1024) NULL
+            )
+            DUPLICATE KEY(`k1`)
+            COMMENT ''
+            DISTRIBUTED BY RANDOM BUCKETS 1
+            PROPERTIES (
+            "replication_allocation" = "tag.location.default: 1"
+            );"""
+
+        load_json_data.call("${testTable}", "${testTable}_case29", 'false', 'true', 'json', '', '',
+                             '', '', '', 'test_duplicate_json_keys.json', false, 1)
+        
+        sql "sync"
+        qt_select29 "select * from ${testTable}"
+
+    } finally {
+        try_sql("DROP TABLE IF EXISTS ${testTable}")
+    }
 }