diff --git a/paimon-core/src/main/java/org/apache/paimon/catalog/AbstractCatalog.java b/paimon-core/src/main/java/org/apache/paimon/catalog/AbstractCatalog.java index 51cb346d4bd7..d7447c37dd79 100644 --- a/paimon-core/src/main/java/org/apache/paimon/catalog/AbstractCatalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/catalog/AbstractCatalog.java @@ -48,7 +48,6 @@ import java.io.IOException; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -282,8 +281,6 @@ public void createTable(Identifier identifier, Schema schema, boolean ignoreIfEx throws TableAlreadyExistException, DatabaseNotExistException { checkNotBranch(identifier, "createTable"); checkNotSystemTable(identifier, "createTable"); - validateIdentifierNameCaseInsensitive(identifier); - validateFieldNameCaseInsensitive(schema.rowType().getFieldNames()); validateAutoCreateClose(schema.options()); validateCustomTablePath(schema.options()); @@ -339,7 +336,6 @@ public void renameTable(Identifier fromTable, Identifier toTable, boolean ignore checkNotBranch(toTable, "renameTable"); checkNotSystemTable(fromTable, "renameTable"); checkNotSystemTable(toTable, "renameTable"); - validateIdentifierNameCaseInsensitive(toTable); try { getTable(fromTable); @@ -366,8 +362,6 @@ public void alterTable( Identifier identifier, List changes, boolean ignoreIfNotExists) throws TableNotExistException, ColumnAlreadyExistException, ColumnNotExistException { checkNotSystemTable(identifier, "alterTable"); - validateIdentifierNameCaseInsensitive(identifier); - validateFieldNameCaseInsensitiveInSchemaChange(changes); try { getTable(identifier); @@ -571,30 +565,6 @@ protected void checkNotSystemDatabase(String database) { } } - protected void validateIdentifierNameCaseInsensitive(Identifier identifier) { - CatalogUtils.validateCaseInsensitive( - caseSensitive(), "Database", identifier.getDatabaseName()); - CatalogUtils.validateCaseInsensitive(caseSensitive(), "Table", identifier.getObjectName()); - } - - private void validateFieldNameCaseInsensitiveInSchemaChange(List changes) { - List fieldNames = new ArrayList<>(); - for (SchemaChange change : changes) { - if (change instanceof SchemaChange.AddColumn) { - SchemaChange.AddColumn addColumn = (SchemaChange.AddColumn) change; - fieldNames.addAll(Arrays.asList(addColumn.fieldNames())); - } else if (change instanceof SchemaChange.RenameColumn) { - SchemaChange.RenameColumn rename = (SchemaChange.RenameColumn) change; - fieldNames.add(rename.newName()); - } - } - validateFieldNameCaseInsensitive(fieldNames); - } - - protected void validateFieldNameCaseInsensitive(List fieldNames) { - CatalogUtils.validateCaseInsensitive(caseSensitive(), "Field", fieldNames); - } - private void validateAutoCreateClose(Map options) { checkArgument( !Boolean.parseBoolean( diff --git a/paimon-core/src/main/java/org/apache/paimon/catalog/CatalogUtils.java b/paimon-core/src/main/java/org/apache/paimon/catalog/CatalogUtils.java index bae23c627607..043da0504d7f 100644 --- a/paimon-core/src/main/java/org/apache/paimon/catalog/CatalogUtils.java +++ b/paimon-core/src/main/java/org/apache/paimon/catalog/CatalogUtils.java @@ -21,14 +21,10 @@ import org.apache.paimon.fs.Path; import org.apache.paimon.schema.SchemaManager; -import java.util.Arrays; -import java.util.List; import java.util.Map; -import java.util.stream.Collectors; import static org.apache.paimon.catalog.Catalog.TABLE_DEFAULT_OPTION_PREFIX; import static org.apache.paimon.options.OptionsUtils.convertToPropertiesPrefixKey; -import static org.apache.paimon.utils.Preconditions.checkArgument; /** Utils for {@link Catalog}. */ public class CatalogUtils { @@ -64,25 +60,4 @@ public static String table(String path) { public static Map tableDefaultOptions(Map options) { return convertToPropertiesPrefixKey(options, TABLE_DEFAULT_OPTION_PREFIX); } - - /** Validate database, table and field names must be lowercase when not case-sensitive. */ - public static void validateCaseInsensitive( - boolean caseSensitive, String type, String... names) { - validateCaseInsensitive(caseSensitive, type, Arrays.asList(names)); - } - - /** Validate database, table and field names must be lowercase when not case-sensitive. */ - public static void validateCaseInsensitive( - boolean caseSensitive, String type, List names) { - if (caseSensitive) { - return; - } - List illegalNames = - names.stream().filter(f -> !f.equals(f.toLowerCase())).collect(Collectors.toList()); - checkArgument( - illegalNames.isEmpty(), - String.format( - "%s name %s cannot contain upper case in the catalog.", - type, illegalNames)); - } } diff --git a/paimon-core/src/test/java/org/apache/paimon/catalog/FileSystemCatalogTest.java b/paimon-core/src/test/java/org/apache/paimon/catalog/FileSystemCatalogTest.java index 7045daca8e86..dcd27a91eda5 100644 --- a/paimon-core/src/test/java/org/apache/paimon/catalog/FileSystemCatalogTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/catalog/FileSystemCatalogTest.java @@ -19,7 +19,6 @@ package org.apache.paimon.catalog; import org.apache.paimon.fs.Path; -import org.apache.paimon.options.CatalogOptions; import org.apache.paimon.options.Options; import org.apache.paimon.schema.Schema; import org.apache.paimon.types.DataTypes; @@ -29,7 +28,6 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import static org.assertj.core.api.Assertions.assertThatExceptionOfType; import static org.assertj.core.api.Assertions.assertThatThrownBy; /** Tests for {@link FileSystemCatalog}. */ @@ -39,14 +37,13 @@ public class FileSystemCatalogTest extends CatalogTestBase { public void setUp() throws Exception { super.setUp(); Options catalogOptions = new Options(); - catalogOptions.set(CatalogOptions.CASE_SENSITIVE, false); catalog = new FileSystemCatalog(fileIO, new Path(warehouse), catalogOptions); } @Test public void testCreateTableCaseSensitive() throws Exception { catalog.createDatabase("test_db", false); - Identifier identifier = Identifier.create("test_db", "new_table"); + Identifier identifier = Identifier.create("test_db", "new_TABLE"); Schema schema = Schema.newBuilder() .column("Pk1", DataTypes.INT()) @@ -64,11 +61,7 @@ public void testCreateTableCaseSensitive() throws Exception { .partitionKeys("Pk1", "pk2") .primaryKey("Pk1", "pk2", "pk3") .build(); - - // Create table throws Exception if using uppercase when 'allow-upper-case' is false - assertThatExceptionOfType(IllegalArgumentException.class) - .isThrownBy(() -> catalog.createTable(identifier, schema, false)) - .withMessage("Field name [Pk1, Col1] cannot contain upper case in the catalog."); + catalog.createTable(identifier, schema, false); } @Test diff --git a/paimon-core/src/test/java/org/apache/paimon/jdbc/JdbcCatalogTest.java b/paimon-core/src/test/java/org/apache/paimon/jdbc/JdbcCatalogTest.java index f01a46fd6bb4..51e2bf5c779d 100644 --- a/paimon-core/src/test/java/org/apache/paimon/jdbc/JdbcCatalogTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/jdbc/JdbcCatalogTest.java @@ -18,6 +18,7 @@ package org.apache.paimon.jdbc; +import org.apache.paimon.catalog.Catalog; import org.apache.paimon.catalog.CatalogTestBase; import org.apache.paimon.catalog.Identifier; import org.apache.paimon.options.CatalogOptions; @@ -87,7 +88,7 @@ public void testCleanTimeoutLockAndAcquireLock() throws SQLException, Interrupte } @Test - public void testCheckIdentifierUpperCase() throws Exception { + public void testUpperCase() throws Exception { catalog.createDatabase("test_db", false); assertThatThrownBy( () -> @@ -95,17 +96,10 @@ public void testCheckIdentifierUpperCase() throws Exception { Identifier.create("TEST_DB", "new_table"), DEFAULT_TABLE_SCHEMA, false)) - .isInstanceOf(IllegalArgumentException.class) - .hasMessage("Database name [TEST_DB] cannot contain upper case in the catalog."); + .isInstanceOf(Catalog.DatabaseNotExistException.class) + .hasMessage("Database TEST_DB does not exist."); - assertThatThrownBy( - () -> - catalog.createTable( - Identifier.create("test_db", "NEW_TABLE"), - DEFAULT_TABLE_SCHEMA, - false)) - .isInstanceOf(IllegalArgumentException.class) - .hasMessage("Table name [NEW_TABLE] cannot contain upper case in the catalog."); + catalog.createTable(Identifier.create("test_db", "new_TABLE"), DEFAULT_TABLE_SCHEMA, false); } @Test diff --git a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/SyncDatabaseActionBase.java b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/SyncDatabaseActionBase.java index 56334c1e7bff..0c4d0182fbf8 100644 --- a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/SyncDatabaseActionBase.java +++ b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/SyncDatabaseActionBase.java @@ -19,7 +19,6 @@ package org.apache.paimon.flink.action.cdc; import org.apache.paimon.catalog.Catalog; -import org.apache.paimon.catalog.CatalogUtils; import org.apache.paimon.flink.action.Action; import org.apache.paimon.flink.action.MultiTablesSinkMode; import org.apache.paimon.flink.sink.cdc.EventParser; @@ -154,13 +153,6 @@ public SyncDatabaseActionBase withPrimaryKeys(String... primaryKeys) { return this; } - @Override - protected void validateCaseSensitivity() { - CatalogUtils.validateCaseInsensitive(caseSensitive, "Database", database); - CatalogUtils.validateCaseInsensitive(caseSensitive, "Table prefix", tablePrefix); - CatalogUtils.validateCaseInsensitive(caseSensitive, "Table suffix", tableSuffix); - } - @Override protected FlatMapFunction recordParse() { return syncJobHandler.provideRecordParser( diff --git a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/SyncTableActionBase.java b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/SyncTableActionBase.java index 6fcdbd44bca2..c74cbff24ec8 100644 --- a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/SyncTableActionBase.java +++ b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/SyncTableActionBase.java @@ -20,7 +20,6 @@ import org.apache.paimon.annotation.VisibleForTesting; import org.apache.paimon.catalog.Catalog; -import org.apache.paimon.catalog.CatalogUtils; import org.apache.paimon.catalog.Identifier; import org.apache.paimon.flink.FlinkConnectorOptions; import org.apache.paimon.flink.action.Action; @@ -113,12 +112,6 @@ protected Schema buildPaimonSchema(Schema retrievedSchema) { true); } - @Override - protected void validateCaseSensitivity() { - CatalogUtils.validateCaseInsensitive(caseSensitive, "Database", database); - CatalogUtils.validateCaseInsensitive(caseSensitive, "Table", table); - } - @Override protected void beforeBuildingSourceSink() throws Exception { Identifier identifier = new Identifier(database, table); diff --git a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/SynchronizationActionBase.java b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/SynchronizationActionBase.java index d755b200a957..74c051396307 100644 --- a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/SynchronizationActionBase.java +++ b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/SynchronizationActionBase.java @@ -114,8 +114,6 @@ public void build() throws Exception { catalog.createDatabase(database, true); - validateCaseSensitivity(); - beforeBuildingSourceSink(); DataStream input = @@ -126,8 +124,6 @@ public void build() throws Exception { buildSink(input, parserFactory); } - protected abstract void validateCaseSensitivity(); - protected void beforeBuildingSourceSink() throws Exception {} protected Source buildSource() { diff --git a/paimon-hive/paimon-hive-catalog/src/test/java/org/apache/paimon/hive/HiveCatalogTest.java b/paimon-hive/paimon-hive-catalog/src/test/java/org/apache/paimon/hive/HiveCatalogTest.java index e3b48f02a696..bf6eb02f3e55 100644 --- a/paimon-hive/paimon-hive-catalog/src/test/java/org/apache/paimon/hive/HiveCatalogTest.java +++ b/paimon-hive/paimon-hive-catalog/src/test/java/org/apache/paimon/hive/HiveCatalogTest.java @@ -85,23 +85,18 @@ public void testListDatabasesWhenNoDatabases() { @Test public void testCheckIdentifierUpperCase() throws Exception { catalog.createDatabase("test_db", false); - assertThatThrownBy( - () -> - catalog.createTable( - Identifier.create("TEST_DB", "new_table"), - DEFAULT_TABLE_SCHEMA, - false)) - .isInstanceOf(IllegalArgumentException.class) - .hasMessage("Database name [TEST_DB] cannot contain upper case in the catalog."); - + assertThatThrownBy(() -> catalog.createDatabase("TEST_DB", false)) + .isInstanceOf(Catalog.DatabaseAlreadyExistException.class) + .hasMessage("Database TEST_DB already exists."); + catalog.createTable(Identifier.create("TEST_DB", "new_table"), DEFAULT_TABLE_SCHEMA, false); assertThatThrownBy( () -> catalog.createTable( Identifier.create("test_db", "NEW_TABLE"), DEFAULT_TABLE_SCHEMA, false)) - .isInstanceOf(IllegalArgumentException.class) - .hasMessage("Table name [NEW_TABLE] cannot contain upper case in the catalog."); + .isInstanceOf(Catalog.TableAlreadyExistException.class) + .hasMessage("Table test_db.NEW_TABLE already exists."); } private static final String HADOOP_CONF_DIR = diff --git a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveCatalogITCaseBase.java b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveCatalogITCaseBase.java index 2266a8484d9d..c39b85cb3d22 100644 --- a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveCatalogITCaseBase.java +++ b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveCatalogITCaseBase.java @@ -577,8 +577,7 @@ public void testCreateInsensitiveTable() throws Exception { " 'uri' = '',", " 'warehouse' = '" + path + "',", " 'lock.enabled' = 'true',", - " 'table.type' = 'EXTERNAL',", - " 'allow-upper-case' = 'true'", + " 'table.type' = 'EXTERNAL'", ")")) .await(); tEnv.executeSql("USE CATALOG paimon_catalog_01").await(); @@ -593,30 +592,6 @@ public void testCreateInsensitiveTable() throws Exception { tEnv.executeSql("DROP TABLE t").await(); Path tablePath = new Path(path, "test_db.db/t"); assertThat(tablePath.getFileSystem().exists(tablePath)).isTrue(); - - tEnv.executeSql( - String.join( - "\n", - "CREATE CATALOG paimon_catalog_02 WITH (", - " 'type' = 'paimon',", - " 'metastore' = 'hive',", - " 'uri' = '',", - " 'warehouse' = '" + path + "',", - " 'lock.enabled' = 'true',", - " 'table.type' = 'EXTERNAL',", - " 'allow-upper-case' = 'false'", - ")")) - .await(); - tEnv.executeSql("USE CATALOG paimon_catalog_02").await(); - tEnv.executeSql("USE test_db").await(); - - // set case-sensitive = false would throw exception out - assertThatThrownBy( - () -> - tEnv.executeSql( - "CREATE TABLE t1 ( aa INT, Bb STRING ) WITH ( 'file.format' = 'avro' )") - .await()) - .isInstanceOf(RuntimeException.class); } @Test @@ -1006,7 +981,8 @@ public void testRenameTable() throws Exception { // the target table name has upper case. assertThatThrownBy(() -> tEnv.executeSql("ALTER TABLE t1 RENAME TO T1")) - .hasMessage("Table name [T1] cannot contain upper case in the catalog."); + .hasMessage( + "Could not execute ALTER TABLE my_hive.test_db.t1 RENAME TO my_hive.test_db.T1"); tEnv.executeSql("ALTER TABLE t1 RENAME TO t3").await(); @@ -1160,24 +1136,16 @@ public void testHiveLock() throws InterruptedException, Catalog.TableNotExistExc @Test public void testUpperCase() { + tEnv.executeSql("CREATE TABLE T (a INT, b STRING ) WITH ( 'file.format' = 'avro' )"); + tEnv.executeSql( + "CREATE TABLE tT (A INT, b STRING, C STRING) WITH ( 'file.format' = 'avro')"); assertThatThrownBy( () -> tEnv.executeSql( - "CREATE TABLE T ( a INT, b STRING ) WITH ( 'file.format' = 'avro' )") - .await()) - .hasRootCauseMessage( - String.format( - "Table name [%s] cannot contain upper case in the catalog.", "T")); - - assertThatThrownBy( - () -> - tEnv.executeSql( - "CREATE TABLE t (A INT, b STRING, C STRING) WITH ( 'file.format' = 'avro')") + "CREATE TABLE tt ( A INT, b STRING, C STRING) WITH ( 'file.format' = 'avro' )") .await()) .hasRootCauseMessage( - String.format( - "Field name %s cannot contain upper case in the catalog.", - "[A, C]")); + "Table (or view) test_db.tt already exists in Catalog my_hive."); } @Test diff --git a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkCatalog.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkCatalog.java index 12023cb84779..f32b87603f44 100644 --- a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkCatalog.java +++ b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkCatalog.java @@ -52,7 +52,6 @@ import org.apache.spark.sql.execution.datasources.v2.csv.CSVTable; import org.apache.spark.sql.execution.datasources.v2.orc.OrcTable; import org.apache.spark.sql.execution.datasources.v2.parquet.ParquetTable; -import org.apache.spark.sql.internal.SessionState; import org.apache.spark.sql.types.StructField; import org.apache.spark.sql.types.StructType; import org.apache.spark.sql.util.CaseInsensitiveStringMap; @@ -70,7 +69,6 @@ import static org.apache.paimon.CoreOptions.FILE_FORMAT; import static org.apache.paimon.CoreOptions.TYPE; import static org.apache.paimon.TableType.FORMAT_TABLE; -import static org.apache.paimon.options.CatalogOptions.CASE_SENSITIVE; import static org.apache.paimon.spark.SparkCatalogOptions.DEFAULT_DATABASE; import static org.apache.paimon.spark.SparkTypeUtils.toPaimonType; import static org.apache.paimon.spark.util.OptionUtils.copyWithSQLConf; @@ -91,18 +89,10 @@ public class SparkCatalog extends SparkBaseCatalog implements SupportFunction, S @Override public void initialize(String name, CaseInsensitiveStringMap options) { this.catalogName = name; - Map newOptions = new HashMap<>(options.asCaseSensitiveMap()); - SessionState sessionState = SparkSession.active().sessionState(); - CatalogContext catalogContext = - CatalogContext.create(Options.fromMap(options), sessionState.newHadoopConf()); - - // if spark is case-insensitive, set case-sensitive to catalog - if (!sessionState.conf().caseSensitiveAnalysis()) { - newOptions.put(CASE_SENSITIVE.key(), "true"); - } - options = new CaseInsensitiveStringMap(newOptions); - + CatalogContext.create( + Options.fromMap(options), + SparkSession.active().sessionState().newHadoopConf()); this.catalog = CatalogFactory.createCatalog(catalogContext); this.defaultDatabase = options.getOrDefault(DEFAULT_DATABASE.key(), DEFAULT_DATABASE.defaultValue()); diff --git a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkGenericCatalog.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkGenericCatalog.java index b57228fa44f0..ac1543f2fe17 100644 --- a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkGenericCatalog.java +++ b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkGenericCatalog.java @@ -62,7 +62,6 @@ import java.util.Map; import java.util.concurrent.Callable; -import static org.apache.paimon.options.CatalogOptions.CASE_SENSITIVE; import static org.apache.paimon.options.CatalogOptions.METASTORE; import static org.apache.paimon.options.CatalogOptions.WAREHOUSE; import static org.apache.paimon.spark.SparkCatalogOptions.CREATE_UNDERLYING_SESSION_CATALOG; @@ -242,7 +241,6 @@ public final void initialize(String name, CaseInsensitiveStringMap options) { SparkSession sparkSession = SparkSession.active(); SessionState sessionState = sparkSession.sessionState(); Configuration hadoopConf = sessionState.newHadoopConf(); - SparkConf sparkConf = new SparkConf(); if (options.containsKey(METASTORE.key()) && options.get(METASTORE.key()).equalsIgnoreCase("hive")) { String uri = options.get(CatalogOptions.URI.key()); @@ -257,11 +255,6 @@ public final void initialize(String name, CaseInsensitiveStringMap options) { } } } - if ("in-memory" - .equals(sparkSession.conf().get(StaticSQLConf.CATALOG_IMPLEMENTATION().key()))) { - LOG.warn("InMemoryCatalog here may cause bad effect."); - } - this.catalogName = name; this.sparkCatalog = new SparkCatalog(); @@ -273,6 +266,7 @@ public final void initialize(String name, CaseInsensitiveStringMap options) { CREATE_UNDERLYING_SESSION_CATALOG.key(), CREATE_UNDERLYING_SESSION_CATALOG.defaultValue())) { this.underlyingSessionCatalogEnabled = true; + SparkConf sparkConf = new SparkConf(); for (Map.Entry entry : options.entrySet()) { sparkConf.set("spark.hadoop." + entry.getKey(), entry.getValue()); hadoopConf.set(entry.getKey(), entry.getValue()); @@ -330,11 +324,6 @@ private void fillCommonConfigurations(Map options, SQLConf sqlCo } else { options.put(DEFAULT_DATABASE.key(), sessionCatalogDefaultDatabase); } - - // if spark is case-insensitive, set case-sensitive to catalog - if (!sqlConf.caseSensitiveAnalysis()) { - options.put(CASE_SENSITIVE.key(), "true"); - } } @Override diff --git a/paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/SparkGenericCatalogWithHiveTest.java b/paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/SparkGenericCatalogWithHiveTest.java index 84ea1ab5cba2..f3d4ba878905 100644 --- a/paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/SparkGenericCatalogWithHiveTest.java +++ b/paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/SparkGenericCatalogWithHiveTest.java @@ -31,7 +31,6 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; -import static org.junit.jupiter.api.Assertions.assertThrows; /** Base tests for spark read. */ public class SparkGenericCatalogWithHiveTest { @@ -48,59 +47,6 @@ public static void closeMetastore() throws Exception { testHiveMetastore.stop(); } - @Test - public void testCreateTableCaseSensitive(@TempDir java.nio.file.Path tempDir) { - // firstly, we use hive metastore to creata table, and check the result. - Path warehousePath = new Path("file:" + tempDir.toString()); - SparkSession spark = - SparkSession.builder() - .config("spark.sql.warehouse.dir", warehousePath.toString()) - // with case-sensitive false - .config("spark.sql.caseSensitive", "false") - // with hive metastore - .config("spark.sql.catalogImplementation", "hive") - .config( - "spark.sql.catalog.spark_catalog", - SparkGenericCatalog.class.getName()) - .master("local[2]") - .getOrCreate(); - - spark.sql("CREATE DATABASE IF NOT EXISTS my_db1"); - spark.sql("USE my_db1"); - spark.sql( - "CREATE TABLE IF NOT EXISTS t2 (a INT, Bb INT, c STRING) USING paimon TBLPROPERTIES" - + " ('file.format'='avro')"); - - assertThat( - spark.sql("SHOW TABLES").collectAsList().stream() - .map(s -> s.get(1)) - .map(Object::toString)) - .containsExactlyInAnyOrder("t2"); - spark.close(); - - SparkSession spark1 = - SparkSession.builder() - .config("spark.sql.warehouse.dir", warehousePath.toString()) - // with case-sensitive true - .config("spark.sql.caseSensitive", "true") - // with hive metastore - .config("spark.sql.catalogImplementation", "hive") - .config( - "spark.sql.catalog.spark_catalog", - SparkGenericCatalog.class.getName()) - .master("local[2]") - .getOrCreate(); - - spark1.sql("USE my_db1"); - assertThrows( - RuntimeException.class, - () -> - spark1.sql( - "CREATE TABLE IF NOT EXISTS t3 (a INT, Bb INT, c STRING) USING paimon TBLPROPERTIES" - + " ('file.format'='avro')")); - spark1.close(); - } - @Test public void testBuildWithHive(@TempDir java.nio.file.Path tempDir) { // firstly, we use hive metastore to create table, and check the result. @@ -123,7 +69,7 @@ public void testBuildWithHive(@TempDir java.nio.file.Path tempDir) { + " ('file.format'='avro')"); assertThat(spark.sql("SHOW NAMESPACES").collectAsList().stream().map(Object::toString)) - .containsExactlyInAnyOrder("[default]", "[my_db]", "[my_db1]"); + .containsExactlyInAnyOrder("[default]", "[my_db]"); assertThat( spark.sql("SHOW TABLES").collectAsList().stream() diff --git a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/DDLWithHiveCatalogTestBase.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/DDLWithHiveCatalogTestBase.scala index 526e24250751..022f7f07dd76 100644 --- a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/DDLWithHiveCatalogTestBase.scala +++ b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/DDLWithHiveCatalogTestBase.scala @@ -531,6 +531,53 @@ abstract class DDLWithHiveCatalogTestBase extends PaimonHiveTestBase { } } + test("Paimon DDL with hive catalog: case sensitive") { + Seq(sparkCatalogName, paimonHiveCatalogName).foreach { + catalogName => + Seq(false, true).foreach { + caseSensitive => + withSparkSQLConf("spark.sql.caseSensitive" -> caseSensitive.toString) { + spark.sql(s"USE $catalogName") + withDatabase("paimon_case_sensitive_DB") { + spark.sql(s"CREATE DATABASE paimon_case_sensitive_DB") + + // check create db + // note: db name is always lower case in hive + intercept[Exception](spark.sql("CREATE DATABASE paimon_case_sensitive_db")) + + spark.sql(s"USE paimon_case_sensitive_DB") + withTable("tT", "tt") { + spark.sql("CREATE TABLE tT (aA INT) USING paimon") + spark.sql("INSERT INTO tT VALUES 1") + + // check select + checkAnswer(spark.sql("SELECT aA FROM tT"), Row(1)) + if (caseSensitive) { + intercept[Exception](spark.sql(s"SELECT aa FROM tT")) + } else { + checkAnswer(spark.sql("SELECT aa FROM tT"), Row(1)) + } + + // check alter table rename + // note: table name is always lower case in hive + intercept[Exception](spark.sql(s"ALTER TABLE tT RENAME TO tt")) + + // check alter table rename column + // note: col name can be upper case in hive + if (caseSensitive) { + spark.sql("ALTER TABLE tT RENAME COLUMN aA TO aa") + checkAnswer(spark.sql("SELECT aa FROM tT"), Row(1)) + intercept[Exception](spark.sql(s"SELECT aA FROM tT")) + } else { + intercept[Exception](spark.sql("ALTER TABLE tT RENAME COLUMN aA TO aa")) + } + } + } + } + } + } + } + def getDatabaseProp(dbName: String, propertyName: String): String = { spark .sql(s"DESC DATABASE EXTENDED $dbName")