From 09befdb7a421ee429fb966ac21dcaa885923dfe9 Mon Sep 17 00:00:00 2001
From: daidai <changyuwei@selectdb.com>
Date: Fri, 10 Jan 2025 18:18:27 +0800
Subject: [PATCH] [fix](hive)fix select count(*) hive full acid tb opt error.
 (#46732)

Problem Summary:
before pr : #44038
In the previous PR, the generation method of split in the count( * )
scenario was optimized.
However, there were some problems with the hive acid table. This PR
mainly fixes this and adds tests.
In the count( * ) scenario, reading the hive full acid table cannot be
optimized, and the file still needs to be split (merge on read is
required), and the hive insert only acid table does not need to be
split.
---
 .../format/table/transactional_hive_reader.cpp    |  1 +
 .../datasource/hive/source/HiveScanNode.java      |  4 ++--
 .../hive/test_transactional_hive.out              | 15 +++++++++++++++
 .../hive/test_hive_translation_insert_only.out    | 10 ++++++++++
 .../hive/test_transactional_hive.groovy           | 12 ++++++++++++
 .../hive/test_hive_translation_insert_only.groovy |  5 +++++
 6 files changed, 45 insertions(+), 2 deletions(-)

diff --git a/be/src/vec/exec/format/table/transactional_hive_reader.cpp b/be/src/vec/exec/format/table/transactional_hive_reader.cpp
index bc4262b7451d53..f840b0af2522fc 100644
--- a/be/src/vec/exec/format/table/transactional_hive_reader.cpp
+++ b/be/src/vec/exec/format/table/transactional_hive_reader.cpp
@@ -180,6 +180,7 @@ Status TransactionalHiveReader::init_row_filters(const TFileRangeDesc& range,
         ++num_delete_files;
     }
     if (num_delete_rows > 0) {
+        orc_reader->set_push_down_agg_type(TPushAggOp::NONE);
         orc_reader->set_delete_rows(&_delete_rows);
         COUNTER_UPDATE(_transactional_orc_profile.num_delete_files, num_delete_files);
         COUNTER_UPDATE(_transactional_orc_profile.num_delete_rows, num_delete_rows);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/source/HiveScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/source/HiveScanNode.java
index 3780a80faa2280..8f965ddf022d45 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/source/HiveScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/source/HiveScanNode.java
@@ -292,12 +292,12 @@ private void getFileSplitByPartitions(HiveMetaStoreCache cache, List<HivePartiti
          * we don't need to split the file because for parquet/orc format, only metadata is read.
          * If we split the file, we will read metadata of a file multiple times, which is not efficient.
          *
-         * - Hive Transactional Table may need merge on read, so do not apply this optimization.
+         * - Hive Full Acid Transactional Table may need merge on read, so do not apply this optimization.
          * - If the file format is not parquet/orc, eg, text, we need to split the file to increase the parallelism.
          */
         boolean needSplit = true;
         if (getPushDownAggNoGroupingOp() == TPushAggOp.COUNT
-                && hiveTransaction != null) {
+                && !(hmsTable.isHiveTransactionalTable() && hmsTable.isFullAcidTable())) {
             int totalFileNum = 0;
             for (FileCacheValue fileCacheValue : fileCaches) {
                 if (fileCacheValue.getFiles() != null) {
diff --git a/regression-test/data/external_table_p0/hive/test_transactional_hive.out b/regression-test/data/external_table_p0/hive/test_transactional_hive.out
index 060fa8c048e5a0..94e32a43db7f3b 100644
--- a/regression-test/data/external_table_p0/hive/test_transactional_hive.out
+++ b/regression-test/data/external_table_p0/hive/test_transactional_hive.out
@@ -122,3 +122,18 @@ F
 -- !16 --
 4	DD
 
+-- !count_1 --
+3
+
+-- !count_2 --
+6
+
+-- !count_3 --
+4
+
+-- !count_4 --
+3
+
+-- !count_5 --
+3
+
diff --git a/regression-test/data/external_table_p2/hive/test_hive_translation_insert_only.out b/regression-test/data/external_table_p2/hive/test_hive_translation_insert_only.out
index 04fccc9d4c0a2d..f43a630f4a387c 100644
--- a/regression-test/data/external_table_p2/hive/test_hive_translation_insert_only.out
+++ b/regression-test/data/external_table_p2/hive/test_hive_translation_insert_only.out
@@ -18,3 +18,13 @@
 3	C
 4	D
 5	E
+
+-- !count_1 --
+4
+
+-- !count_2 --
+5
+
+-- !count_3 --
+5
+
diff --git a/regression-test/suites/external_table_p0/hive/test_transactional_hive.groovy b/regression-test/suites/external_table_p0/hive/test_transactional_hive.groovy
index 81f2358e9da8ff..8c092d40b66498 100644
--- a/regression-test/suites/external_table_p0/hive/test_transactional_hive.groovy
+++ b/regression-test/suites/external_table_p0/hive/test_transactional_hive.groovy
@@ -115,6 +115,14 @@ suite("test_transactional_hive", "p0,external,hive,external_docker,external_dock
 
         }
     }
+    
+    def test_acid_count = {
+        qt_count_1 """ select count(*) from orc_full_acid; """ // 3 
+        qt_count_2 """ select count(*) from orc_full_acid_par; """  // 6
+        qt_count_3 """ select count(*) from orc_to_acid_compacted_tb; """ //4
+        qt_count_4 """ select count(*) from orc_acid_minor; """ //3
+        qt_count_5 """ select count(*) from orc_acid_major; """ //3
+    }
 
 
     String enabled = context.config.otherConfigs.get("enableHiveTest")
@@ -149,6 +157,10 @@ suite("test_transactional_hive", "p0,external,hive,external_docker,external_dock
             test_acid()
             test_acid_write()
 
+
+            test_acid_count()
+            
+            
             sql """drop catalog if exists ${catalog_name}"""
         } finally {
         }
diff --git a/regression-test/suites/external_table_p2/hive/test_hive_translation_insert_only.groovy b/regression-test/suites/external_table_p2/hive/test_hive_translation_insert_only.groovy
index 9b021e1dc81372..900a031db7b49c 100644
--- a/regression-test/suites/external_table_p2/hive/test_hive_translation_insert_only.groovy
+++ b/regression-test/suites/external_table_p2/hive/test_hive_translation_insert_only.groovy
@@ -45,6 +45,11 @@ suite("test_hive_translation_insert_only", "p2,external,hive,external_remote,ext
     qt_2 """ select * from parquet_insert_only_major order by id """ 
     qt_3 """ select * from orc_insert_only_minor order by id """ 
 
+    qt_count_1 """ select count(*) from text_insert_only """ //4 
+    qt_count_2 """ select count(*) from parquet_insert_only_major """ //5 
+    qt_count_3 """ select count(*) from orc_insert_only_minor """ //5
+
+
     sql """drop catalog ${hms_catalog_name};"""
 }