[feature][insert]add hive table sink thrift (apache#32274)

add hive table sink thrift add partition type add new compress type issue: apache#31442
codegit10001 · Mar 15, 2024 · 84572d7 · 84572d7
1 parent f359124
commit 84572d7
Show file tree

Hide file tree

Showing 4 changed files with 34 additions and 15 deletions.
diff --git a/gensrc/thrift/DataSinks.thrift b/gensrc/thrift/DataSinks.thrift
@@ -38,6 +38,7 @@ enum TDataSinkType {
     MULTI_CAST_DATA_STREAM_SINK,
     GROUP_COMMIT_OLAP_TABLE_SINK, // deprecated
     GROUP_COMMIT_BLOCK_SINK,
+    HIVE_TABLE_SINK,
 }
 
 enum TResultSinkType {
@@ -101,7 +102,7 @@ enum TParquetRepetitionType {
 struct TParquetSchema {
     1: optional TParquetRepetitionType schema_repetition_type
     2: optional TParquetDataType schema_data_type
-    3: optional string schema_column_name    
+    3: optional string schema_column_name
     4: optional TParquetDataLogicalType schema_data_logical_type
 }
 
@@ -282,6 +283,7 @@ struct TOlapTableSink {
 struct THiveLocationParams {
   1: optional string write_path
   2: optional string target_path
+  3: optional Types.TFileType file_type
 }
 
 struct TSortedColumn {
@@ -300,11 +302,16 @@ struct THiveBucket {
     4: optional list<TSortedColumn> sorted_by
 }
 
-enum THiveCompressionType {
-    SNAPPY = 3,
-    LZ4 = 4,
-    ZLIB = 6,
-    ZSTD = 7,
+enum THiveColumnType {
+    PARTITION_KEY = 0,
+    REGULAR = 1,
+    SYNTHESIZED = 2
+}
+
+struct THiveColumn {
+  1: optional string name
+  2: optional Types.TTypeDesc data_type
+  3: optional THiveColumnType column_type
 }
 
 struct THivePartition {
@@ -316,13 +323,14 @@ struct THivePartition {
 struct THiveTableSink {
     1: optional string db_name
     2: optional string table_name
-    3: optional list<string> data_column_names
-    4: optional list<string> partition_column_names
-    5: optional list<THivePartition> partitions
-    6: optional list<THiveBucket> buckets
-    7: optional PlanNodes.TFileFormatType file_format
-    8: optional THiveCompressionType compression_type
-    9: optional THiveLocationParams location
+    3: optional list<THiveColumn> columns
+    4: optional list<THivePartition> partitions
+    5: optional THiveBucket bucket_info
+    6: optional PlanNodes.TFileFormatType file_format
+    7: optional PlanNodes.TFileCompressType compression_type
+    8: optional THiveLocationParams location
+    9: optional map<string, string> hadoop_config
+    10: optional bool overwrite
 }
 
 enum TUpdateMode {

diff --git a/gensrc/thrift/FrontendService.thrift b/gensrc/thrift/FrontendService.thrift
@@ -29,6 +29,7 @@ include "Exprs.thrift"
 include "RuntimeProfile.thrift"
 include "MasterService.thrift"
 include "AgentService.thrift"
+include "DataSinks.thrift"
 
 // These are supporting structs for JniFrontend.java, which serves as the glue
 // between our C++ execution environment and the Java frontend.
@@ -481,6 +482,8 @@ struct TReportExecStatusParams {
   24: optional TQueryStatistics query_statistics // deprecated
 
   25: optional TReportWorkloadRuntimeStatusParams report_workload_runtime_status
+
+  26: optional list<DataSinks.THivePartitionUpdate> hive_partition_updates
 }
 
 struct TFeResult {

diff --git a/gensrc/thrift/Partitions.thrift b/gensrc/thrift/Partitions.thrift
@@ -43,7 +43,13 @@ enum TPartitionType {
   BUCKET_SHFFULE_HASH_PARTITIONED,
 
   // used for shuffle data by parititon and tablet
-  TABLET_SINK_SHUFFLE_PARTITIONED
+  TABLET_SINK_SHUFFLE_PARTITIONED,
+
+  // used for shuffle data by hive parititon
+  TABLE_SINK_HASH_PARTITIONED,
+
+  // used for hive unparititoned table
+  TABLE_SINK_RANDOM_PARTITIONED
 }
 
 enum TDistributionType {

diff --git a/gensrc/thrift/PlanNodes.thrift b/gensrc/thrift/PlanNodes.thrift
@@ -138,7 +138,9 @@ enum TFileCompressType {
     DEFLATE,
     LZOP,
     LZ4BLOCK,
-    SNAPPYBLOCK
+    SNAPPYBLOCK,
+    ZLIB,
+    ZSTD
 }
 
 struct THdfsConf {