Skip to content

Commit

Permalink
1
Browse files Browse the repository at this point in the history
  • Loading branch information
@Zouxxyy
Zouxxyy committed Dec 17, 2024
1 parent cdd5bb7 commit 0661f92
Show file tree
Hide file tree
Showing 7 changed files with 11 additions and 32 deletions.
2 changes: 1 addition & 1 deletion paimon-common/src/main/java/org/apache/paimon/options/CatalogOptions.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -131,7 +131,7 @@ public class CatalogOptions {
public static final ConfigOption<Boolean> CASE_SENSITIVE =
ConfigOptions.key("case-sensitive")
.booleanType()
.noDefaultValue()
.defaultValue(true)
.withFallbackKeys("allow-upper-case")
.withDescription("Indicates whether this catalog is case-sensitive.");

Expand Down
2 changes: 1 addition & 1 deletion paimon-core/src/main/java/org/apache/paimon/catalog/FileSystemCatalog.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -159,6 +159,6 @@ public String warehouse() {

@Override
public boolean caseSensitive() {
return catalogOptions.getOptional(CASE_SENSITIVE).orElse(true);
return catalogOptions.get(CASE_SENSITIVE);
}
}
2 changes: 1 addition & 1 deletion paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -225,7 +225,7 @@ public List<PartitionEntry> listPartitions(Identifier identifier)

@Override
public boolean caseSensitive() {
return options.getOptional(CASE_SENSITIVE).orElse(true);
return options.get(CASE_SENSITIVE);
}

@Override
Expand Down
2 changes: 1 addition & 1 deletion paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -873,7 +873,7 @@ private void alterTableToHms(Table table, Identifier identifier, TableSchema new

@Override
public boolean caseSensitive() {
return catalogOptions.getOptional(CASE_SENSITIVE).orElse(false);
return catalogOptions.get(CASE_SENSITIVE);
}

@Override
Expand Down
16 changes: 3 additions & 13 deletions paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkCatalog.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,6 @@
import org.apache.spark.sql.execution.datasources.v2.csv.CSVTable;
import org.apache.spark.sql.execution.datasources.v2.orc.OrcTable;
import org.apache.spark.sql.execution.datasources.v2.parquet.ParquetTable;
import org.apache.spark.sql.internal.SessionState;
import org.apache.spark.sql.types.StructField;
import org.apache.spark.sql.types.StructType;
import org.apache.spark.sql.util.CaseInsensitiveStringMap;
Expand All @@ -69,7 +68,6 @@
import static org.apache.paimon.CoreOptions.FILE_FORMAT;
import static org.apache.paimon.CoreOptions.TYPE;
import static org.apache.paimon.TableType.FORMAT_TABLE;
import static org.apache.paimon.options.CatalogOptions.CASE_SENSITIVE;
import static org.apache.paimon.spark.SparkCatalogOptions.DEFAULT_DATABASE;
import static org.apache.paimon.spark.SparkTypeUtils.toPaimonType;
import static org.apache.paimon.spark.util.OptionUtils.copyWithSQLConf;
Expand All @@ -90,18 +88,10 @@ public class SparkCatalog extends SparkBaseCatalog implements SupportFunction, S
@Override
public void initialize(String name, CaseInsensitiveStringMap options) {
this.catalogName = name;
Map<String, String> newOptions = new HashMap<>(options.asCaseSensitiveMap());
SessionState sessionState = SparkSession.active().sessionState();

CatalogContext catalogContext =
CatalogContext.create(Options.fromMap(options), sessionState.newHadoopConf());

// if spark is case-insensitive, set case-sensitive to catalog
if (!sessionState.conf().caseSensitiveAnalysis()) {
newOptions.put(CASE_SENSITIVE.key(), "true");
}
options = new CaseInsensitiveStringMap(newOptions);

CatalogContext.create(
Options.fromMap(options),
SparkSession.active().sessionState().newHadoopConf());
this.catalog = CatalogFactory.createCatalog(catalogContext);
this.defaultDatabase =
options.getOrDefault(DEFAULT_DATABASE.key(), DEFAULT_DATABASE.defaultValue());
Expand Down
13 changes: 1 addition & 12 deletions ...-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkGenericCatalog.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,6 @@
import java.util.Map;
import java.util.concurrent.Callable;

import static org.apache.paimon.options.CatalogOptions.CASE_SENSITIVE;
import static org.apache.paimon.options.CatalogOptions.METASTORE;
import static org.apache.paimon.options.CatalogOptions.WAREHOUSE;
import static org.apache.paimon.spark.SparkCatalogOptions.CREATE_UNDERLYING_SESSION_CATALOG;
Expand Down Expand Up @@ -242,7 +241,6 @@ public final void initialize(String name, CaseInsensitiveStringMap options) {
SparkSession sparkSession = SparkSession.active();
SessionState sessionState = sparkSession.sessionState();
Configuration hadoopConf = sessionState.newHadoopConf();
SparkConf sparkConf = new SparkConf();
if (options.containsKey(METASTORE.key())
&& options.get(METASTORE.key()).equalsIgnoreCase("hive")) {
String uri = options.get(CatalogOptions.URI.key());
Expand All @@ -257,11 +255,6 @@ public final void initialize(String name, CaseInsensitiveStringMap options) {
}
}
}
if ("in-memory"
.equals(sparkSession.conf().get(StaticSQLConf.CATALOG_IMPLEMENTATION().key()))) {
LOG.warn("InMemoryCatalog here may cause bad effect.");
}

this.catalogName = name;
this.sparkCatalog = new SparkCatalog();

Expand All @@ -273,6 +266,7 @@ public final void initialize(String name, CaseInsensitiveStringMap options) {
CREATE_UNDERLYING_SESSION_CATALOG.key(),
CREATE_UNDERLYING_SESSION_CATALOG.defaultValue())) {
this.underlyingSessionCatalogEnabled = true;
SparkConf sparkConf = new SparkConf();
for (Map.Entry<String, String> entry : options.entrySet()) {
sparkConf.set("spark.hadoop." + entry.getKey(), entry.getValue());
hadoopConf.set(entry.getKey(), entry.getValue());
Expand Down Expand Up @@ -330,11 +324,6 @@ private void fillCommonConfigurations(Map<String, String> options, SQLConf sqlCo
} else {
options.put(DEFAULT_DATABASE.key(), sessionCatalogDefaultDatabase);
}

// if spark is case-insensitive, set case-sensitive to catalog
if (!sqlConf.caseSensitiveAnalysis()) {
options.put(CASE_SENSITIVE.key(), "true");
}
}

@Override
Expand Down
6 changes: 3 additions & 3 deletions ...aimon-spark-ut/src/test/java/org/apache/paimon/spark/SparkGenericCatalogWithHiveTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ public static void closeMetastore() throws Exception {

@Test
public void testCreateTableCaseSensitive(@TempDir java.nio.file.Path tempDir) {
// firstly, we use hive metastore to creata table, and check the result.
// firstly, we use hive metastore to create table, and check the result.
Path warehousePath = new Path("file:" + tempDir.toString());
SparkSession spark =
SparkSession.builder()
Expand All @@ -62,6 +62,7 @@ public void testCreateTableCaseSensitive(@TempDir java.nio.file.Path tempDir) {
.config(
"spark.sql.catalog.spark_catalog",
SparkGenericCatalog.class.getName())
.config("spark.sql.catalog.spark_catalog.case-sensitive", "true")
.master("local[2]")
.getOrCreate();

Expand All @@ -81,13 +82,12 @@ public void testCreateTableCaseSensitive(@TempDir java.nio.file.Path tempDir) {
SparkSession spark1 =
SparkSession.builder()
.config("spark.sql.warehouse.dir", warehousePath.toString())
// with case-sensitive true
.config("spark.sql.caseSensitive", "true")
// with hive metastore
.config("spark.sql.catalogImplementation", "hive")
.config(
"spark.sql.catalog.spark_catalog",
SparkGenericCatalog.class.getName())
.config("spark.sql.catalog.spark_catalog.case-sensitive", "false")
.master("local[2]")
.getOrCreate();

Expand Down

0 comments on commit 0661f92

Please sign in to comment.