From 1c10fd00927ca820633c22c0f0b58c8a4f292fca Mon Sep 17 00:00:00 2001 From: zouxxyy Date: Tue, 17 Dec 2024 16:25:17 +0800 Subject: [PATCH] 1 --- .../generated/catalog_configuration.html | 2 +- .../apache/paimon/options/CatalogOptions.java | 2 +- .../apache/paimon/catalog/AbstractCatalog.java | 6 ++++++ .../apache/paimon/catalog/FileSystemCatalog.java | 7 ------- .../java/org/apache/paimon/rest/RESTCatalog.java | 2 +- .../java/org/apache/paimon/hive/HiveCatalog.java | 6 ------ .../org/apache/paimon/hive/HiveCatalogTest.java | 16 +++++++++++++--- .../paimon/hive/HiveCatalogITCaseBase.java | 16 +++++++++++++--- .../org/apache/paimon/spark/SparkCatalog.java | 16 +++------------- .../apache/paimon/spark/SparkGenericCatalog.java | 13 +------------ .../spark/SparkGenericCatalogWithHiveTest.java | 8 +++----- 11 files changed, 42 insertions(+), 52 deletions(-) diff --git a/docs/layouts/shortcodes/generated/catalog_configuration.html b/docs/layouts/shortcodes/generated/catalog_configuration.html index 6355c9558653..a5a7007f198e 100644 --- a/docs/layouts/shortcodes/generated/catalog_configuration.html +++ b/docs/layouts/shortcodes/generated/catalog_configuration.html @@ -70,7 +70,7 @@
case-sensitive
- (none) + true Boolean Indicates whether this catalog is case-sensitive. diff --git a/paimon-common/src/main/java/org/apache/paimon/options/CatalogOptions.java b/paimon-common/src/main/java/org/apache/paimon/options/CatalogOptions.java index b22274e011fb..2011bf65e2a3 100644 --- a/paimon-common/src/main/java/org/apache/paimon/options/CatalogOptions.java +++ b/paimon-common/src/main/java/org/apache/paimon/options/CatalogOptions.java @@ -131,7 +131,7 @@ public class CatalogOptions { public static final ConfigOption CASE_SENSITIVE = ConfigOptions.key("case-sensitive") .booleanType() - .noDefaultValue() + .defaultValue(true) .withFallbackKeys("allow-upper-case") .withDescription("Indicates whether this catalog is case-sensitive."); diff --git a/paimon-core/src/main/java/org/apache/paimon/catalog/AbstractCatalog.java b/paimon-core/src/main/java/org/apache/paimon/catalog/AbstractCatalog.java index 51cb346d4bd7..8773233925c4 100644 --- a/paimon-core/src/main/java/org/apache/paimon/catalog/AbstractCatalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/catalog/AbstractCatalog.java @@ -60,6 +60,7 @@ import static org.apache.paimon.CoreOptions.TYPE; import static org.apache.paimon.CoreOptions.createCommitUser; +import static org.apache.paimon.options.CatalogOptions.CASE_SENSITIVE; import static org.apache.paimon.options.CatalogOptions.LOCK_ENABLED; import static org.apache.paimon.options.CatalogOptions.LOCK_TYPE; import static org.apache.paimon.utils.BranchManager.DEFAULT_MAIN_BRANCH; @@ -114,6 +115,11 @@ public Optional defaultLockFactory() { return Optional.empty(); } + @Override + public boolean caseSensitive() { + return catalogOptions.get(CASE_SENSITIVE); + } + public Optional lockContext() { return Optional.of(CatalogLockContext.fromOptions(catalogOptions)); } diff --git a/paimon-core/src/main/java/org/apache/paimon/catalog/FileSystemCatalog.java b/paimon-core/src/main/java/org/apache/paimon/catalog/FileSystemCatalog.java index cb0c358259f8..545cd2ae3e52 100644 --- a/paimon-core/src/main/java/org/apache/paimon/catalog/FileSystemCatalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/catalog/FileSystemCatalog.java @@ -34,8 +34,6 @@ import java.util.Map; import java.util.concurrent.Callable; -import static org.apache.paimon.options.CatalogOptions.CASE_SENSITIVE; - /** A catalog implementation for {@link FileIO}. */ public class FileSystemCatalog extends AbstractCatalog { @@ -161,9 +159,4 @@ public void close() throws Exception {} public String warehouse() { return warehouse.toString(); } - - @Override - public boolean caseSensitive() { - return catalogOptions.getOptional(CASE_SENSITIVE).orElse(true); - } } diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java index 8b53bef8486b..9266d8ed67b3 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java @@ -258,7 +258,7 @@ public List listPartitions(Identifier identifier) @Override public boolean caseSensitive() { - return options.getOptional(CASE_SENSITIVE).orElse(true); + return options.get(CASE_SENSITIVE); } @Override diff --git a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java index a590ede6d2e6..2867d548ddb3 100644 --- a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java +++ b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java @@ -106,7 +106,6 @@ import static org.apache.paimon.hive.HiveCatalogOptions.IDENTIFIER; import static org.apache.paimon.hive.HiveCatalogOptions.LOCATION_IN_PROPERTIES; import static org.apache.paimon.hive.HiveTableUtils.convertToFormatTable; -import static org.apache.paimon.options.CatalogOptions.CASE_SENSITIVE; import static org.apache.paimon.options.CatalogOptions.FORMAT_TABLE_ENABLED; import static org.apache.paimon.options.CatalogOptions.SYNC_ALL_PROPERTIES; import static org.apache.paimon.options.CatalogOptions.TABLE_TYPE; @@ -899,11 +898,6 @@ private void alterTableToHms(Table table, Identifier identifier, TableSchema new clients.execute(client -> HiveAlterTableUtils.alterTable(client, identifier, table)); } - @Override - public boolean caseSensitive() { - return catalogOptions.getOptional(CASE_SENSITIVE).orElse(false); - } - @Override protected boolean allowCustomTablePath() { return true; diff --git a/paimon-hive/paimon-hive-catalog/src/test/java/org/apache/paimon/hive/HiveCatalogTest.java b/paimon-hive/paimon-hive-catalog/src/test/java/org/apache/paimon/hive/HiveCatalogTest.java index e3b48f02a696..3e96067d5dac 100644 --- a/paimon-hive/paimon-hive-catalog/src/test/java/org/apache/paimon/hive/HiveCatalogTest.java +++ b/paimon-hive/paimon-hive-catalog/src/test/java/org/apache/paimon/hive/HiveCatalogTest.java @@ -84,10 +84,19 @@ public void testListDatabasesWhenNoDatabases() { @Test public void testCheckIdentifierUpperCase() throws Exception { - catalog.createDatabase("test_db", false); + HiveConf hiveConf = new HiveConf(); + String jdoConnectionURL = "jdbc:derby:memory:" + UUID.randomUUID(); + hiveConf.setVar(METASTORECONNECTURLKEY, jdoConnectionURL + ";create=true"); + String metastoreClientClass = "org.apache.hadoop.hive.metastore.HiveMetaStoreClient"; + Options catalogOptions = new Options(); + catalogOptions.set(CatalogOptions.CASE_SENSITIVE, false); + HiveCatalog hiveCatalog = + new HiveCatalog(fileIO, hiveConf, metastoreClientClass, catalogOptions, warehouse); + + hiveCatalog.createDatabase("test_db", false); assertThatThrownBy( () -> - catalog.createTable( + hiveCatalog.createTable( Identifier.create("TEST_DB", "new_table"), DEFAULT_TABLE_SCHEMA, false)) @@ -96,12 +105,13 @@ public void testCheckIdentifierUpperCase() throws Exception { assertThatThrownBy( () -> - catalog.createTable( + hiveCatalog.createTable( Identifier.create("test_db", "NEW_TABLE"), DEFAULT_TABLE_SCHEMA, false)) .isInstanceOf(IllegalArgumentException.class) .hasMessage("Table name [NEW_TABLE] cannot contain upper case in the catalog."); + hiveCatalog.close(); } private static final String HADOOP_CONF_DIR = diff --git a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveCatalogITCaseBase.java b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveCatalogITCaseBase.java index 2266a8484d9d..72c0701614f9 100644 --- a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveCatalogITCaseBase.java +++ b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveCatalogITCaseBase.java @@ -991,18 +991,23 @@ public void testCreateTableAs() throws Exception { @Test public void testRenameTable() throws Exception { + Map props = new HashMap<>(); + props.put("case-sensitive", "false"); + registerHiveCatalog("case_insensitive_hive", props); + tEnv.executeSql("USE CATALOG case_insensitive_hive"); + tEnv.executeSql("USE test_db"); tEnv.executeSql("CREATE TABLE t1 (a INT)").await(); tEnv.executeSql("CREATE TABLE t2 (a INT)").await(); tEnv.executeSql("INSERT INTO t1 SELECT 1").await(); // the source table do not exist. assertThatThrownBy(() -> tEnv.executeSql("ALTER TABLE t3 RENAME TO t4")) .hasMessage( - "Table `my_hive`.`test_db`.`t3` doesn't exist or is a temporary table."); + "Table `case_insensitive_hive`.`test_db`.`t3` doesn't exist or is a temporary table."); // the target table has existed. assertThatThrownBy(() -> tEnv.executeSql("ALTER TABLE t1 RENAME TO t2")) .hasMessage( - "Could not execute ALTER TABLE my_hive.test_db.t1 RENAME TO my_hive.test_db.t2"); + "Could not execute ALTER TABLE case_insensitive_hive.test_db.t1 RENAME TO my_hive.test_db.t2"); // the target table name has upper case. assertThatThrownBy(() -> tEnv.executeSql("ALTER TABLE t1 RENAME TO T1")) @@ -1159,7 +1164,12 @@ public void testHiveLock() throws InterruptedException, Catalog.TableNotExistExc } @Test - public void testUpperCase() { + public void testUpperCase() throws Exception { + Map props = new HashMap<>(); + props.put("case-sensitive", "false"); + registerHiveCatalog("case_insensitive_hive", props); + tEnv.executeSql("USE CATALOG case_insensitive_hive"); + assertThatThrownBy( () -> tEnv.executeSql( diff --git a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkCatalog.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkCatalog.java index 12023cb84779..f32b87603f44 100644 --- a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkCatalog.java +++ b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkCatalog.java @@ -52,7 +52,6 @@ import org.apache.spark.sql.execution.datasources.v2.csv.CSVTable; import org.apache.spark.sql.execution.datasources.v2.orc.OrcTable; import org.apache.spark.sql.execution.datasources.v2.parquet.ParquetTable; -import org.apache.spark.sql.internal.SessionState; import org.apache.spark.sql.types.StructField; import org.apache.spark.sql.types.StructType; import org.apache.spark.sql.util.CaseInsensitiveStringMap; @@ -70,7 +69,6 @@ import static org.apache.paimon.CoreOptions.FILE_FORMAT; import static org.apache.paimon.CoreOptions.TYPE; import static org.apache.paimon.TableType.FORMAT_TABLE; -import static org.apache.paimon.options.CatalogOptions.CASE_SENSITIVE; import static org.apache.paimon.spark.SparkCatalogOptions.DEFAULT_DATABASE; import static org.apache.paimon.spark.SparkTypeUtils.toPaimonType; import static org.apache.paimon.spark.util.OptionUtils.copyWithSQLConf; @@ -91,18 +89,10 @@ public class SparkCatalog extends SparkBaseCatalog implements SupportFunction, S @Override public void initialize(String name, CaseInsensitiveStringMap options) { this.catalogName = name; - Map newOptions = new HashMap<>(options.asCaseSensitiveMap()); - SessionState sessionState = SparkSession.active().sessionState(); - CatalogContext catalogContext = - CatalogContext.create(Options.fromMap(options), sessionState.newHadoopConf()); - - // if spark is case-insensitive, set case-sensitive to catalog - if (!sessionState.conf().caseSensitiveAnalysis()) { - newOptions.put(CASE_SENSITIVE.key(), "true"); - } - options = new CaseInsensitiveStringMap(newOptions); - + CatalogContext.create( + Options.fromMap(options), + SparkSession.active().sessionState().newHadoopConf()); this.catalog = CatalogFactory.createCatalog(catalogContext); this.defaultDatabase = options.getOrDefault(DEFAULT_DATABASE.key(), DEFAULT_DATABASE.defaultValue()); diff --git a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkGenericCatalog.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkGenericCatalog.java index b57228fa44f0..ac1543f2fe17 100644 --- a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkGenericCatalog.java +++ b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkGenericCatalog.java @@ -62,7 +62,6 @@ import java.util.Map; import java.util.concurrent.Callable; -import static org.apache.paimon.options.CatalogOptions.CASE_SENSITIVE; import static org.apache.paimon.options.CatalogOptions.METASTORE; import static org.apache.paimon.options.CatalogOptions.WAREHOUSE; import static org.apache.paimon.spark.SparkCatalogOptions.CREATE_UNDERLYING_SESSION_CATALOG; @@ -242,7 +241,6 @@ public final void initialize(String name, CaseInsensitiveStringMap options) { SparkSession sparkSession = SparkSession.active(); SessionState sessionState = sparkSession.sessionState(); Configuration hadoopConf = sessionState.newHadoopConf(); - SparkConf sparkConf = new SparkConf(); if (options.containsKey(METASTORE.key()) && options.get(METASTORE.key()).equalsIgnoreCase("hive")) { String uri = options.get(CatalogOptions.URI.key()); @@ -257,11 +255,6 @@ public final void initialize(String name, CaseInsensitiveStringMap options) { } } } - if ("in-memory" - .equals(sparkSession.conf().get(StaticSQLConf.CATALOG_IMPLEMENTATION().key()))) { - LOG.warn("InMemoryCatalog here may cause bad effect."); - } - this.catalogName = name; this.sparkCatalog = new SparkCatalog(); @@ -273,6 +266,7 @@ public final void initialize(String name, CaseInsensitiveStringMap options) { CREATE_UNDERLYING_SESSION_CATALOG.key(), CREATE_UNDERLYING_SESSION_CATALOG.defaultValue())) { this.underlyingSessionCatalogEnabled = true; + SparkConf sparkConf = new SparkConf(); for (Map.Entry entry : options.entrySet()) { sparkConf.set("spark.hadoop." + entry.getKey(), entry.getValue()); hadoopConf.set(entry.getKey(), entry.getValue()); @@ -330,11 +324,6 @@ private void fillCommonConfigurations(Map options, SQLConf sqlCo } else { options.put(DEFAULT_DATABASE.key(), sessionCatalogDefaultDatabase); } - - // if spark is case-insensitive, set case-sensitive to catalog - if (!sqlConf.caseSensitiveAnalysis()) { - options.put(CASE_SENSITIVE.key(), "true"); - } } @Override diff --git a/paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/SparkGenericCatalogWithHiveTest.java b/paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/SparkGenericCatalogWithHiveTest.java index 84ea1ab5cba2..2596b9289750 100644 --- a/paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/SparkGenericCatalogWithHiveTest.java +++ b/paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/SparkGenericCatalogWithHiveTest.java @@ -50,18 +50,17 @@ public static void closeMetastore() throws Exception { @Test public void testCreateTableCaseSensitive(@TempDir java.nio.file.Path tempDir) { - // firstly, we use hive metastore to creata table, and check the result. + // firstly, we use hive metastore to create table, and check the result. Path warehousePath = new Path("file:" + tempDir.toString()); SparkSession spark = SparkSession.builder() .config("spark.sql.warehouse.dir", warehousePath.toString()) - // with case-sensitive false - .config("spark.sql.caseSensitive", "false") // with hive metastore .config("spark.sql.catalogImplementation", "hive") .config( "spark.sql.catalog.spark_catalog", SparkGenericCatalog.class.getName()) + .config("spark.sql.catalog.spark_catalog.case-sensitive", "true") .master("local[2]") .getOrCreate(); @@ -81,13 +80,12 @@ public void testCreateTableCaseSensitive(@TempDir java.nio.file.Path tempDir) { SparkSession spark1 = SparkSession.builder() .config("spark.sql.warehouse.dir", warehousePath.toString()) - // with case-sensitive true - .config("spark.sql.caseSensitive", "true") // with hive metastore .config("spark.sql.catalogImplementation", "hive") .config( "spark.sql.catalog.spark_catalog", SparkGenericCatalog.class.getName()) + .config("spark.sql.catalog.spark_catalog.case-sensitive", "false") .master("local[2]") .getOrCreate();