From 46e0cb174d6e842fdcdf6f80f3daf44834850ef6 Mon Sep 17 00:00:00 2001 From: xuyu <11161569@vivo.com> Date: Thu, 22 Aug 2024 20:47:43 +0800 Subject: [PATCH 1/5] add pk to hms --- .../java/org/apache/paimon/hive/HiveCatalog.java | 11 +++++++++++ .../apache/paimon/hive/HiveCatalogITCaseBase.java | 15 ++++++++++++++- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java index 0cd834f8ee82..039ba629415b 100644 --- a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java +++ b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java @@ -521,6 +521,17 @@ private Table createHiveTable(Identifier identifier, TableSchema tableSchema) { tblProperties = convertToPropertiesPrefixKey(tableSchema.options(), HIVE_PREFIX); } + // add primary-key, partition-key, bucket-id to tblproperties + if (!tableSchema.primaryKeys().isEmpty()) { + tblProperties.put("primary-key", String.join(",", tableSchema.primaryKeys())); + } + if (!tableSchema.partitionKeys().isEmpty()) { + tblProperties.put("partition-key", String.join(",", tableSchema.partitionKeys())); + } + if (!tableSchema.bucketKeys().isEmpty()) { + tblProperties.put("bucket-id", String.join(",", tableSchema.bucketKeys())); + } + Table table = newHmsTable(identifier, tblProperties); updateHmsTable(table, identifier, tableSchema); return table; diff --git a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveCatalogITCaseBase.java b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveCatalogITCaseBase.java index 0e8caab0d3a7..aae8f918cd01 100644 --- a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveCatalogITCaseBase.java +++ b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveCatalogITCaseBase.java @@ -378,7 +378,8 @@ public void testCreateCatalogWithSyncTblProperties() throws Exception { .await(); tEnv.executeSql("USE CATALOG paimon_catalog_sync").await(); tEnv.executeSql("USE test_db").await(); - tEnv.executeSql("CREATE TABLE t01 ( aa INT, bb STRING ) WITH ( 'file.format' = 'avro' )") + tEnv.executeSql( + "CREATE TABLE t01 ( aa INT, bb STRING, cc STRING, PRIMARY KEY (cc, aa)) PARTITIONED BY (cc) WITH ('file.format' = 'avro')") .await(); // assert contain properties assertThat( @@ -387,6 +388,18 @@ public void testCreateCatalogWithSyncTblProperties() throws Exception { .contains("\tfile.format \tavro ")) .isTrue(); + assertThat( + hiveShell + .executeQuery("DESC FORMATTED t01") + .contains("\tfile.format \tavro ")) + .isTrue(); + + assertThat( + hiveShell + .executeQuery("DESC FORMATTED t01") + .contains("\tfile.format \tavro ")) + .isTrue(); + tEnv.executeSql("ALTER TABLE t01 SET ( 'file.format' = 'parquet' )").await(); assertThat( hiveShell From abd009d39d7118fcd9bd9299346fc6b4b5b1a901 Mon Sep 17 00:00:00 2001 From: xuzifu666 <1206332514@qq.com> Date: Thu, 22 Aug 2024 21:05:06 +0800 Subject: [PATCH 2/5] add ut --- .../org/apache/paimon/hive/HiveCatalog.java | 22 ++++++------- .../paimon/hive/HiveCatalogITCaseBase.java | 33 ++++++++++++++++--- 2 files changed, 40 insertions(+), 15 deletions(-) diff --git a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java index 039ba629415b..4f793d8958c1 100644 --- a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java +++ b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java @@ -517,21 +517,21 @@ private Table createHiveTable(Identifier identifier, TableSchema tableSchema) { Map tblProperties; if (syncAllProperties()) { tblProperties = new HashMap<>(tableSchema.options()); + + // add primary-key, partition-key, bucket-id to tblproperties + if (!tableSchema.primaryKeys().isEmpty()) { + tblProperties.put("primary-key", String.join(",", tableSchema.primaryKeys())); + } + if (!tableSchema.partitionKeys().isEmpty()) { + tblProperties.put("partition-key", String.join(",", tableSchema.partitionKeys())); + } + if (!tableSchema.bucketKeys().isEmpty()) { + tblProperties.put("bucket-id", String.join(",", tableSchema.bucketKeys())); + } } else { tblProperties = convertToPropertiesPrefixKey(tableSchema.options(), HIVE_PREFIX); } - // add primary-key, partition-key, bucket-id to tblproperties - if (!tableSchema.primaryKeys().isEmpty()) { - tblProperties.put("primary-key", String.join(",", tableSchema.primaryKeys())); - } - if (!tableSchema.partitionKeys().isEmpty()) { - tblProperties.put("partition-key", String.join(",", tableSchema.partitionKeys())); - } - if (!tableSchema.bucketKeys().isEmpty()) { - tblProperties.put("bucket-id", String.join(",", tableSchema.bucketKeys())); - } - Table table = newHmsTable(identifier, tblProperties); updateHmsTable(table, identifier, tableSchema); return table; diff --git a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveCatalogITCaseBase.java b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveCatalogITCaseBase.java index aae8f918cd01..129053f02c1c 100644 --- a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveCatalogITCaseBase.java +++ b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveCatalogITCaseBase.java @@ -379,7 +379,7 @@ public void testCreateCatalogWithSyncTblProperties() throws Exception { tEnv.executeSql("USE CATALOG paimon_catalog_sync").await(); tEnv.executeSql("USE test_db").await(); tEnv.executeSql( - "CREATE TABLE t01 ( aa INT, bb STRING, cc STRING, PRIMARY KEY (cc, aa)) PARTITIONED BY (cc) WITH ('file.format' = 'avro')") + "CREATE TABLE t01 ( aa INT, bb STRING, cc STRING, PRIMARY KEY (cc, aa) NOT ENFORCED) PARTITIONED BY (cc) WITH ('file.format' = 'avro')") .await(); // assert contain properties assertThat( @@ -391,13 +391,19 @@ public void testCreateCatalogWithSyncTblProperties() throws Exception { assertThat( hiveShell .executeQuery("DESC FORMATTED t01") - .contains("\tfile.format \tavro ")) + .contains("\tprimary-key \tcc,aa ")) .isTrue(); assertThat( hiveShell .executeQuery("DESC FORMATTED t01") - .contains("\tfile.format \tavro ")) + .contains("\tpartition-key \tcc ")) + .isTrue(); + + assertThat( + hiveShell + .executeQuery("DESC FORMATTED t01") + .contains("\tbucket-id \taa ")) .isTrue(); tEnv.executeSql("ALTER TABLE t01 SET ( 'file.format' = 'parquet' )").await(); @@ -429,7 +435,8 @@ public void testCreateCatalogWithSyncTblProperties() throws Exception { .await(); tEnv.executeSql("USE CATALOG paimon_catalog_sync01").await(); tEnv.executeSql("USE test_db").await(); - tEnv.executeSql("CREATE TABLE t02 ( aa INT, bb STRING ) WITH ( 'file.format' = 'avro' )") + tEnv.executeSql( + "CREATE TABLE t02 ( aa INT, bb STRING, cc STRING, PRIMARY KEY (cc, aa) NOT ENFORCED) PARTITIONED BY (cc) WITH ('file.format' = 'avro')") .await(); // assert not contain properties @@ -439,6 +446,24 @@ public void testCreateCatalogWithSyncTblProperties() throws Exception { .contains("\tfile.format \tavro ")) .isFalse(); + assertThat( + hiveShell + .executeQuery("DESC FORMATTED t02") + .contains("\tprimary-key \tcc,aa ")) + .isFalse(); + + assertThat( + hiveShell + .executeQuery("DESC FORMATTED t02") + .contains("\tpartition-key \tcc ")) + .isFalse(); + + assertThat( + hiveShell + .executeQuery("DESC FORMATTED t02") + .contains("\tbucket-id \taa ")) + .isFalse(); + tEnv.executeSql("ALTER TABLE t02 SET ( 'file.format' = 'parquet' )").await(); assertThat( hiveShell From 231fa16d2a82b3ac2b29c1a9ed031faf985ee081 Mon Sep 17 00:00:00 2001 From: xuzifu666 <1206332514@qq.com> Date: Thu, 22 Aug 2024 22:50:53 +0800 Subject: [PATCH 3/5] add bucket --- .../org/apache/paimon/hive/HiveCatalog.java | 1 + .../paimon/hive/HiveCatalogITCaseBase.java | 18 ++++++++++++++++-- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java index 4f793d8958c1..26592ddea1e9 100644 --- a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java +++ b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java @@ -528,6 +528,7 @@ private Table createHiveTable(Identifier identifier, TableSchema tableSchema) { if (!tableSchema.bucketKeys().isEmpty()) { tblProperties.put("bucket-id", String.join(",", tableSchema.bucketKeys())); } + tblProperties.put("bucket", String.valueOf(tableSchema.numBuckets())); } else { tblProperties = convertToPropertiesPrefixKey(tableSchema.options(), HIVE_PREFIX); } diff --git a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveCatalogITCaseBase.java b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveCatalogITCaseBase.java index 129053f02c1c..5e21357ea64e 100644 --- a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveCatalogITCaseBase.java +++ b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveCatalogITCaseBase.java @@ -379,9 +379,10 @@ public void testCreateCatalogWithSyncTblProperties() throws Exception { tEnv.executeSql("USE CATALOG paimon_catalog_sync").await(); tEnv.executeSql("USE test_db").await(); tEnv.executeSql( - "CREATE TABLE t01 ( aa INT, bb STRING, cc STRING, PRIMARY KEY (cc, aa) NOT ENFORCED) PARTITIONED BY (cc) WITH ('file.format' = 'avro')") + "CREATE TABLE t01 ( aa INT, bb STRING, cc STRING, PRIMARY KEY (cc, aa) NOT ENFORCED) PARTITIONED BY (cc) WITH ('file.format' = 'avro', 'bucket' = '3')") .await(); // assert contain properties + List descFormattedT01 = hiveShell.executeQuery("DESC FORMATTED t01"); assertThat( hiveShell .executeQuery("DESC FORMATTED t01") @@ -406,6 +407,12 @@ public void testCreateCatalogWithSyncTblProperties() throws Exception { .contains("\tbucket-id \taa ")) .isTrue(); + assertThat( + hiveShell + .executeQuery("DESC FORMATTED t01") + .contains("\tbucket \t3 ")) + .isTrue(); + tEnv.executeSql("ALTER TABLE t01 SET ( 'file.format' = 'parquet' )").await(); assertThat( hiveShell @@ -436,10 +443,11 @@ public void testCreateCatalogWithSyncTblProperties() throws Exception { tEnv.executeSql("USE CATALOG paimon_catalog_sync01").await(); tEnv.executeSql("USE test_db").await(); tEnv.executeSql( - "CREATE TABLE t02 ( aa INT, bb STRING, cc STRING, PRIMARY KEY (cc, aa) NOT ENFORCED) PARTITIONED BY (cc) WITH ('file.format' = 'avro')") + "CREATE TABLE t02 ( aa INT, bb STRING, cc STRING, PRIMARY KEY (cc, aa) NOT ENFORCED) PARTITIONED BY (cc) WITH ('file.format' = 'avro', 'bucket' = '3')") .await(); // assert not contain properties + List descFormattedT02 = hiveShell.executeQuery("DESC FORMATTED t02"); assertThat( hiveShell .executeQuery("DESC FORMATTED t02") @@ -464,6 +472,12 @@ public void testCreateCatalogWithSyncTblProperties() throws Exception { .contains("\tbucket-id \taa ")) .isFalse(); + assertThat( + hiveShell + .executeQuery("DESC FORMATTED t02") + .contains("\tbucket \t3 ")) + .isFalse(); + tEnv.executeSql("ALTER TABLE t02 SET ( 'file.format' = 'parquet' )").await(); assertThat( hiveShell From 5437ee68eeaf311647f120ebc08215bac585fca7 Mon Sep 17 00:00:00 2001 From: xuzifu666 <1206332514@qq.com> Date: Fri, 23 Aug 2024 12:12:37 +0800 Subject: [PATCH 4/5] change key --- .../src/main/java/org/apache/paimon/hive/HiveCatalog.java | 8 ++++---- .../org/apache/paimon/hive/HiveCatalogITCaseBase.java | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java index 26592ddea1e9..bf784330e71f 100644 --- a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java +++ b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java @@ -520,15 +520,15 @@ private Table createHiveTable(Identifier identifier, TableSchema tableSchema) { // add primary-key, partition-key, bucket-id to tblproperties if (!tableSchema.primaryKeys().isEmpty()) { - tblProperties.put("primary-key", String.join(",", tableSchema.primaryKeys())); + tblProperties.put(CoreOptions.PRIMARY_KEY.key(), String.join(",", tableSchema.primaryKeys())); } if (!tableSchema.partitionKeys().isEmpty()) { - tblProperties.put("partition-key", String.join(",", tableSchema.partitionKeys())); + tblProperties.put(CoreOptions.PARTITION.key(), String.join(",", tableSchema.partitionKeys())); } if (!tableSchema.bucketKeys().isEmpty()) { - tblProperties.put("bucket-id", String.join(",", tableSchema.bucketKeys())); + tblProperties.put(CoreOptions.BUCKET_KEY.key(), String.join(",", tableSchema.bucketKeys())); } - tblProperties.put("bucket", String.valueOf(tableSchema.numBuckets())); + tblProperties.put(CoreOptions.BUCKET.key(), String.valueOf(tableSchema.numBuckets())); } else { tblProperties = convertToPropertiesPrefixKey(tableSchema.options(), HIVE_PREFIX); } diff --git a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveCatalogITCaseBase.java b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveCatalogITCaseBase.java index 5e21357ea64e..d9c38a670988 100644 --- a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveCatalogITCaseBase.java +++ b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveCatalogITCaseBase.java @@ -398,13 +398,13 @@ public void testCreateCatalogWithSyncTblProperties() throws Exception { assertThat( hiveShell .executeQuery("DESC FORMATTED t01") - .contains("\tpartition-key \tcc ")) + .contains("\tpartition \tcc ")) .isTrue(); assertThat( hiveShell .executeQuery("DESC FORMATTED t01") - .contains("\tbucket-id \taa ")) + .contains("\tbucket-key \taa ")) .isTrue(); assertThat( @@ -463,13 +463,13 @@ public void testCreateCatalogWithSyncTblProperties() throws Exception { assertThat( hiveShell .executeQuery("DESC FORMATTED t02") - .contains("\tpartition-key \tcc ")) + .contains("\tpartition \tcc ")) .isFalse(); assertThat( hiveShell .executeQuery("DESC FORMATTED t02") - .contains("\tbucket-id \taa ")) + .contains("\tbucket-key \taa ")) .isFalse(); assertThat( From a37db2534359c326030713843c1bee7f1484974d Mon Sep 17 00:00:00 2001 From: xuzifu666 <1206332514@qq.com> Date: Fri, 23 Aug 2024 12:17:45 +0800 Subject: [PATCH 5/5] fix styles --- .../main/java/org/apache/paimon/hive/HiveCatalog.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java index bf784330e71f..31c3a28a823d 100644 --- a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java +++ b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java @@ -520,13 +520,16 @@ private Table createHiveTable(Identifier identifier, TableSchema tableSchema) { // add primary-key, partition-key, bucket-id to tblproperties if (!tableSchema.primaryKeys().isEmpty()) { - tblProperties.put(CoreOptions.PRIMARY_KEY.key(), String.join(",", tableSchema.primaryKeys())); + tblProperties.put( + CoreOptions.PRIMARY_KEY.key(), String.join(",", tableSchema.primaryKeys())); } if (!tableSchema.partitionKeys().isEmpty()) { - tblProperties.put(CoreOptions.PARTITION.key(), String.join(",", tableSchema.partitionKeys())); + tblProperties.put( + CoreOptions.PARTITION.key(), String.join(",", tableSchema.partitionKeys())); } if (!tableSchema.bucketKeys().isEmpty()) { - tblProperties.put(CoreOptions.BUCKET_KEY.key(), String.join(",", tableSchema.bucketKeys())); + tblProperties.put( + CoreOptions.BUCKET_KEY.key(), String.join(",", tableSchema.bucketKeys())); } tblProperties.put(CoreOptions.BUCKET.key(), String.valueOf(tableSchema.numBuckets())); } else {