From 19ec6f2031f53cea6a305dac6381bf3459090ea8 Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Sat, 18 Mar 2023 00:20:34 +0800 Subject: [PATCH] [core] Rename paimon: tablestore to paimon --- README.md | 2 +- docs/content/engines/flink.md | 4 +- docs/content/engines/hive.md | 12 +- docs/content/engines/spark2.md | 6 +- docs/content/engines/spark3.md | 16 +- docs/content/filesystems/oss.md | 22 +- docs/content/filesystems/s3.md | 22 +- docs/content/how-to/creating-catalogs.md | 24 +-- docs/content/how-to/creating-tables.md | 4 +- docs/content/how-to/lookup-joins.md | 2 +- docs/content/maintenance/configurations.md | 6 +- .../generated/catalog_configuration.html | 2 +- .../generated/core_configuration.html | 4 +- .../flink_connector_configuration.html | 2 +- .../src/main/resources/bin/metric_client.sh | 2 +- .../src/main/resources/bin/run_benchmark.sh | 2 +- .../paimon/benchmark/TableBenchmark.java | 3 +- .../apache/paimon/catalog/CatalogContext.java | 2 +- .../java/org/apache/paimon/fs/FileIO.java | 3 +- .../org/apache/paimon/fs/PluginFileIO.java | 3 +- .../apache/paimon/fs/hadoop/HadoopFileIO.java | 3 +- .../apache/paimon/fs/local/LocalFileIO.java | 3 +- .../apache/paimon/options/CatalogOptions.java | 3 +- .../org/apache/paimon/types/DataType.java | 2 +- .../org/apache/paimon/fs/FileIOFinder.java | 3 +- .../java/org/apache/paimon/CoreOptions.java | 4 +- .../main/java/org/apache/paimon/Snapshot.java | 32 ++- .../java/org/apache/paimon/WriteMode.java | 4 +- .../org/apache/paimon/catalog/Catalog.java | 2 +- .../apache/paimon/catalog/CatalogFactory.java | 4 +- .../catalog/FileSystemCatalogFactory.java | 2 - .../paimon/sort/BinaryInMemorySortBuffer.java | 2 +- .../paimon/table/AbstractFileStoreTable.java | 2 +- .../paimon/table/FileStoreTableFactory.java | 3 +- .../apache/paimon/utils/JsonSerdeUtil.java | 2 +- ... org.apache.paimon.catalog.CatalogFactory} | 0 .../paimon/catalog/CatalogFactoryTest.java | 2 - .../apache/paimon/utils/TraceableFileIO.java | 3 +- paimon-e2e-tests/pom.xml | 2 +- .../paimon/tests/FileStoreBatchE2eTest.java | 4 +- .../paimon/tests/FileStoreStreamE2eTest.java | 4 +- .../paimon/tests/FlinkActionsE2eTest.java | 10 +- .../org/apache/paimon/tests/HiveE2eTest.java | 2 +- .../apache/paimon/tests/LogStoreE2eTest.java | 4 +- .../org/apache/paimon/tests/SparkE2eTest.java | 8 +- .../java/org/apache/paimon/oss/OSSFileIO.java | 3 +- .../java/org/apache/paimon/oss/OSSLoader.java | 3 +- .../java/org/apache/paimon/s3/S3FileIO.java | 3 +- .../java/org/apache/paimon/s3/S3Loader.java | 3 +- .../connector/AbstractTableStoreFactory.java | 4 +- .../apache/paimon/connector/FlinkCatalog.java | 4 +- .../paimon/connector/FlinkCatalogFactory.java | 3 +- .../connector/FlinkConnectorOptions.java | 2 +- .../apache/paimon/connector/FlinkFileIO.java | 2 +- .../apache/paimon/connector/FlinkUtils.java | 2 +- .../connector/TableStoreConnectorFactory.java | 2 +- .../paimon/connector/action/ActionBase.java | 2 +- .../paimon/connector/sink/FileStoreSink.java | 2 +- .../paimon/connector/sink/FlinkSink.java | 2 +- .../paimon/connector/sink/StoreSinkWrite.java | 2 +- .../paimon/connector/sink/TableStoreSink.java | 2 +- .../paimon/connector/source/FlinkSource.java | 2 +- .../connector/source/FlinkTableSource.java | 2 +- ...ositePkAndMultiPartitionedTableITCase.java | 2 +- ...ComputedColumnAndWatermarkTableITCase.java | 2 +- .../paimon/connector/CreateTableITCase.java | 2 +- .../paimon/connector/DropTableITCase.java | 2 +- .../connector/FileSystemCatalogITCase.java | 3 +- .../paimon/connector/FlinkCatalogTest.java | 4 +- .../paimon/connector/LogSystemITCase.java | 3 +- .../connector/ReadWriteTableITCase.java | 2 +- .../connector/StreamingWarehouseITCase.java | 2 +- .../paimon/connector/TableStoreTestBase.java | 2 +- .../connector/action/ActionITCaseBase.java | 3 +- paimon-format/pom.xml | 32 ++- .../format/parquet/ParquetInputFile.java | 2 +- .../format/parquet/ParquetInputStream.java | 2 +- paimon-hive/paimon-hive-catalog/pom.xml | 16 +- .../org/apache/paimon/hive/HiveCatalog.java | 9 +- .../paimon/hive/HiveCatalogFactory.java | 5 +- ... org.apache.paimon.catalog.CatalogFactory} | 0 .../apache/paimon/hive/HiveCatalogITCase.java | 8 +- .../org/apache/paimon/hive/HiveSchema.java | 9 +- .../paimon/hive/TableStoreHiveMetaHook.java | 2 +- .../hive/TableStoreHiveStorageHandler.java | 2 +- .../apache/paimon/hive/TableStoreJobConf.java | 4 +- .../apache/paimon/hive/TableStoreSerDe.java | 2 +- .../hive/mapred/TableStoreInputFormat.java | 5 +- .../hive/mapred/TableStoreInputSplit.java | 3 +- .../hive/mapred/TableStoreOutputFormat.java | 2 +- .../hive/mapred/TableStoreRecordReader.java | 2 +- .../paimon/hive/HiveTableSchemaTest.java | 12 +- .../paimon/spark/SparkDataSourceReader.java | 2 +- .../paimon/spark/SparkInputPartition.java | 2 +- .../org/apache/paimon/spark/SparkSource.java | 7 +- .../apache/paimon/spark/SparkReadITCase.java | 5 +- .../org/apache/paimon/spark/SparkCatalog.java | 5 +- .../paimon/spark/SparkInputPartition.java | 2 +- .../paimon/spark/SparkReaderFactory.java | 2 +- .../org/apache/paimon/spark/SparkScan.java | 4 +- .../apache/paimon/spark/SparkScanBuilder.java | 2 +- .../org/apache/paimon/spark/SparkSource.java | 7 +- .../org/apache/paimon/spark/SparkTable.java | 2 +- .../apache/paimon/spark/SparkReadITCase.java | 77 ++++--- .../paimon/spark/SparkReadTestBase.java | 12 +- .../apache/paimon/spark/SparkS3ITCase.java | 10 +- .../spark/SparkSchemaEvolutionITCase.java | 193 ++++++++---------- .../apache/paimon/spark/SparkWriteITCase.java | 8 +- 108 files changed, 369 insertions(+), 413 deletions(-) rename paimon-core/src/main/resources/META-INF/services/{org.apache.paimon.file.org.apache.paimon.catalog.CatalogFactory => org.apache.paimon.catalog.CatalogFactory} (100%) rename paimon-hive/paimon-hive-catalog/src/main/resources/META-INF/services/{org.apache.paimon.file.org.apache.paimon.catalog.CatalogFactory => org.apache.paimon.catalog.CatalogFactory} (100%) diff --git a/README.md b/README.md index 9d40869d7354..b11440d7b293 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ Flink Table Store is developed under the umbrella of [Apache Flink](https://flin Please check out the full [documentation](https://nightlies.apache.org/flink/paimon-docs-master/), hosted by the [ASF](https://www.apache.org/), for detailed information and user guides. -Check our [quick-start](https://nightlies.apache.org/flink/paimon-docs-master/docs/try-paimon/quick-start/) guide for simple setup instructions to get you started with the table store. +Check our [quick-start](https://nightlies.apache.org/flink/paimon-docs-master/docs/try-paimon/quick-start/) guide for simple setup instructions to get you started with the paimon. ## Building diff --git a/docs/content/engines/flink.md b/docs/content/engines/flink.md index 65afd124fcfd..ec7ddce69f82 100644 --- a/docs/content/engines/flink.md +++ b/docs/content/engines/flink.md @@ -71,7 +71,7 @@ tar -xzf flink-*.tgz **Step 2: Copy Table Store Bundled Jar** -Copy table store bundled jar to the `lib` directory of your Flink home. +Copy paimon bundled jar to the `lib` directory of your Flink home. ```bash cp paimon-flink-*.jar /lib/ @@ -138,7 +138,7 @@ CREATE TEMPORARY TABLE word_table ( 'fields.word.length' = '1' ); --- table store requires checkpoint interval in streaming mode +-- paimon requires checkpoint interval in streaming mode SET 'execution.checkpointing.interval' = '10 s'; -- write streaming data to dynamic table diff --git a/docs/content/engines/hive.md b/docs/content/engines/hive.md index 465a66333328..ac74887403c1 100644 --- a/docs/content/engines/hive.md +++ b/docs/content/engines/hive.md @@ -70,13 +70,13 @@ You can find Hive connector jar in `./paimon-hive/paimon-hive-connector-}}.jar` into `auxlib`. -* You can also copy this jar to a path accessible by Hive, then use `add jar /path/to/paimon-hive-connector-{{< version >}}.jar` to enable table store support in Hive. Note that this method is not recommended. If you're using the MR execution engine and running a join statement, you may be faced with the exception `org.apache.hive.com.esotericsoftware.kryo.kryoexception: unable to find class`. +* You can also copy this jar to a path accessible by Hive, then use `add jar /path/to/paimon-hive-connector-{{< version >}}.jar` to enable paimon support in Hive. Note that this method is not recommended. If you're using the MR execution engine and running a join statement, you may be faced with the exception `org.apache.hive.com.esotericsoftware.kryo.kryoexception: unable to find class`. NOTE: If you are using HDFS, make sure that the environment variable `HADOOP_HOME` or `HADOOP_CONF_DIR` is set. ## Quick Start with Table Store Hive Catalog -By using table store Hive catalog, you can create, drop and insert into table store tables from Flink. These operations directly affect the corresponding Hive metastore. Tables created in this way can also be accessed directly from Hive. +By using paimon Hive catalog, you can create, drop and insert into paimon tables from Flink. These operations directly affect the corresponding Hive metastore. Tables created in this way can also be accessed directly from Hive. **Step 1: Prepare Table Store Hive Catalog Jar File for Flink** @@ -88,7 +88,7 @@ Execute the following Flink SQL script in Flink SQL client to define a Table Sto ```sql -- Flink SQL CLI --- Define table store Hive catalog +-- Define paimon Hive catalog CREATE CATALOG my_hive WITH ( 'type' = 'paimon', @@ -97,11 +97,11 @@ CREATE CATALOG my_hive WITH ( 'warehouse' = '/path/to/table/store/warehouse' ); --- Use table store Hive catalog +-- Use paimon Hive catalog USE CATALOG my_hive; --- Create a table in table store Hive catalog (use "default" database by default) +-- Create a table in paimon Hive catalog (use "default" database by default) CREATE TABLE test_table ( a int, @@ -155,7 +155,7 @@ OK ## Quick Start with External Table -To access existing table store table, you can also register them as external tables in Hive. Run the following Hive SQL in Hive CLI. +To access existing paimon table, you can also register them as external tables in Hive. Run the following Hive SQL in Hive CLI. ```sql -- Assume that paimon-hive-connector-{{< version >}}.jar is already in auxlib directory. diff --git a/docs/content/engines/spark2.md b/docs/content/engines/spark2.md index 2d99e1fdf825..ffafa8eeb93c 100644 --- a/docs/content/engines/spark2.md +++ b/docs/content/engines/spark2.md @@ -74,7 +74,7 @@ After the guide, all table files should be stored under the path `/tmp/table_sto **Step 2: Specify Table Store Jar File** -You can append path to table store jar file to the `--jars` argument when starting `spark-shell`. +You can append path to paimon jar file to the `--jars` argument when starting `spark-shell`. ```bash spark-shell ... --jars /path/to/paimon-spark-2-{{< version >}}.jar @@ -84,10 +84,10 @@ Alternatively, you can copy `paimon-spark-2-{{< version >}}.jar` under `spark/ja **Step 3: Query Table** -Table store with Spark 2.4 does not support DDL. You can use the `Dataset` reader and register the `Dataset` as a temporary table. In spark shell: +Paimon with Spark 2.4 does not support DDL. You can use the `Dataset` reader and register the `Dataset` as a temporary table. In spark shell: ```scala -val dataset = spark.read.format("tablestore").load("file:/tmp/table_store/default.db/word_count") +val dataset = spark.read.format("paimon").load("file:/tmp/table_store/default.db/word_count") dataset.createOrReplaceTempView("word_count") spark.sql("SELECT * FROM word_count").show() ``` diff --git a/docs/content/engines/spark3.md b/docs/content/engines/spark3.md index df2111f70b44..c685cb60e17c 100644 --- a/docs/content/engines/spark3.md +++ b/docs/content/engines/spark3.md @@ -72,7 +72,7 @@ If you are using HDFS, make sure that the environment variable `HADOOP_HOME` or **Step 1: Specify Table Store Jar File** -Append path to table store jar file to the `--jars` argument when starting `spark-sql`. +Append path to paimon jar file to the `--jars` argument when starting `spark-sql`. ```bash spark-sql ... --jars /path/to/paimon-spark-3.3-{{< version >}}.jar @@ -82,19 +82,19 @@ Alternatively, you can copy `paimon-spark-3.3-{{< version >}}.jar` under `spark/ **Step 2: Specify Table Store Catalog** -When starting `spark-sql`, use the following command to register Table Store’s Spark catalog with the name `tablestore`. Table files of the warehouse is stored under `/tmp/table_store`. +When starting `spark-sql`, use the following command to register Table Store’s Spark catalog with the name `paimon`. Table files of the warehouse is stored under `/tmp/table_store`. ```bash spark-sql ... \ - --conf spark.sql.catalog.tablestore=org.apache.paimon.spark.SparkCatalog \ - --conf spark.sql.catalog.tablestore.warehouse=file:/tmp/table_store + --conf spark.sql.catalog.paimon=org.apache.paimon.spark.SparkCatalog \ + --conf spark.sql.catalog.paimon.warehouse=file:/tmp/table_store ``` -After `spark-sql` command line has started, run the following SQL to create and switch to database `tablestore.default`. +After `spark-sql` command line has started, run the following SQL to create and switch to database `paimon.default`. ```sql -CREATE DATABASE tablestore.default; -USE tablestore.default; +CREATE DATABASE paimon.default; +USE paimon.default; ``` **Step 3: Create a table and Write Some Records** @@ -142,7 +142,7 @@ spark-shell ... --jars /path/to/paimon-spark-3.3-{{< version >}}.jar ``` ```scala -val dataset = spark.read.format("tablestore").load("file:/tmp/table_store/default.db/my_table") +val dataset = spark.read.format("paimon").load("file:/tmp/table_store/default.db/my_table") dataset.createOrReplaceTempView("my_table") spark.sql("SELECT * FROM my_table").show() ``` diff --git a/docs/content/filesystems/oss.md b/docs/content/filesystems/oss.md index 56e789485b9b..6cb0d7f43b45 100644 --- a/docs/content/filesystems/oss.md +++ b/docs/content/filesystems/oss.md @@ -31,7 +31,7 @@ under the License. ## Download [Download](https://repo.maven.apache.org/maven2/org/apache/flink/paimon-oss/{{< version >}}/paimon-oss-{{< version >}}.jar) -flink table store shaded jar. +flink paimon shaded jar. {{< /stable >}} @@ -78,11 +78,11 @@ Place `paimon-oss-{{< version >}}.jar` together with `paimon-spark-{{< version > ```shell spark-sql \ - --conf spark.sql.catalog.tablestore=org.apache.paimon.spark.SparkCatalog \ - --conf spark.sql.catalog.tablestore.warehouse=oss:/// \ - --conf spark.sql.catalog.tablestore.fs.oss.endpoint=oss-cn-hangzhou.aliyuncs.com \ - --conf spark.sql.catalog.tablestore.fs.oss.accessKeyId=xxx \ - --conf spark.sql.catalog.tablestore.fs.oss.accessKeySecret=yyy + --conf spark.sql.catalog.paimon=org.apache.paimon.spark.SparkCatalog \ + --conf spark.sql.catalog.paimon.warehouse=oss:/// \ + --conf spark.sql.catalog.paimon.fs.oss.endpoint=oss-cn-hangzhou.aliyuncs.com \ + --conf spark.sql.catalog.paimon.fs.oss.accessKeyId=xxx \ + --conf spark.sql.catalog.paimon.fs.oss.accessKeySecret=yyy ``` {{< /tab >}} @@ -94,9 +94,9 @@ NOTE: You need to ensure that Hive metastore can access `oss`. Place `paimon-oss-{{< version >}}.jar` together with `paimon-hive-connector-{{< version >}}.jar` under Hive's auxlib directory, and start like ```sql -SET tablestore.fs.oss.endpoint=oss-cn-hangzhou.aliyuncs.com; -SET tablestore.fs.oss.accessKeyId=xxx; -SET tablestore.fs.oss.accessKeySecret=yyy; +SET paimon.fs.oss.endpoint=oss-cn-hangzhou.aliyuncs.com; +SET paimon.fs.oss.accessKeyId=xxx; +SET paimon.fs.oss.accessKeySecret=yyy; ``` And read table from hive metastore, table can be created by Flink or Spark, see [Catalog with Hive Metastore]({{< ref "how-to/creating-catalogs" >}}) @@ -109,9 +109,9 @@ SELECT COUNT(1) FROM test_table; {{< tab "Trino" >}} -Place `paimon-oss-{{< version >}}.jar` together with `paimon-trino-{{< version >}}.jar` under `plugin/tablestore` directory. +Place `paimon-oss-{{< version >}}.jar` together with `paimon-trino-{{< version >}}.jar` under `plugin/paimon` directory. -Add options in `etc/catalog/tablestore.properties`. +Add options in `etc/catalog/paimon.properties`. ```shell fs.oss.endpoint=oss-cn-hangzhou.aliyuncs.com fs.oss.accessKeyId=xxx diff --git a/docs/content/filesystems/s3.md b/docs/content/filesystems/s3.md index d47bafdfd7c6..866b0ce49355 100644 --- a/docs/content/filesystems/s3.md +++ b/docs/content/filesystems/s3.md @@ -31,7 +31,7 @@ under the License. ## Download [Download](https://repo.maven.apache.org/maven2/org/apache/flink/paimon-s3/{{< version >}}/paimon-s3-{{< version >}}.jar) -flink table store shaded jar. +flink paimon shaded jar. {{< /stable >}} @@ -78,11 +78,11 @@ Place `paimon-s3-{{< version >}}.jar` together with `paimon-spark-{{< version >} ```shell spark-sql \ - --conf spark.sql.catalog.tablestore=org.apache.paimon.spark.SparkCatalog \ - --conf spark.sql.catalog.tablestore.warehouse=s3:/// \ - --conf spark.sql.catalog.tablestore.s3.endpoint=your-endpoint-hostname \ - --conf spark.sql.catalog.tablestore.s3.access-key=xxx \ - --conf spark.sql.catalog.tablestore.s3.secret-key=yyy + --conf spark.sql.catalog.paimon=org.apache.paimon.spark.SparkCatalog \ + --conf spark.sql.catalog.paimon.warehouse=s3:/// \ + --conf spark.sql.catalog.paimon.s3.endpoint=your-endpoint-hostname \ + --conf spark.sql.catalog.paimon.s3.access-key=xxx \ + --conf spark.sql.catalog.paimon.s3.secret-key=yyy ``` {{< /tab >}} @@ -94,9 +94,9 @@ NOTE: You need to ensure that Hive metastore can access `s3`. Place `paimon-s3-{{< version >}}.jar` together with `paimon-hive-connector-{{< version >}}.jar` under Hive's auxlib directory, and start like ```sql -SET tablestore.s3.endpoint=your-endpoint-hostname; -SET tablestore.s3.access-key=xxx; -SET tablestore.s3.secret-key=yyy; +SET paimon.s3.endpoint=your-endpoint-hostname; +SET paimon.s3.access-key=xxx; +SET paimon.s3.secret-key=yyy; ``` And read table from hive metastore, table can be created by Flink or Spark, see [Catalog with Hive Metastore]({{< ref "how-to/creating-catalogs" >}}) @@ -109,9 +109,9 @@ SELECT COUNT(1) FROM test_table; {{< tab "Trino" >}} -Place `paimon-s3-{{< version >}}.jar` together with `paimon-trino-{{< version >}}.jar` under `plugin/tablestore` directory. +Place `paimon-s3-{{< version >}}.jar` together with `paimon-trino-{{< version >}}.jar` under `plugin/paimon` directory. -Add options in `etc/catalog/tablestore.properties`. +Add options in `etc/catalog/paimon.properties`. ```shell s3.endpoint=your-endpoint-hostname s3.access-key=xxx diff --git a/docs/content/how-to/creating-catalogs.md b/docs/content/how-to/creating-catalogs.md index 8fdbba600f56..aaa1d8a6735a 100644 --- a/docs/content/how-to/creating-catalogs.md +++ b/docs/content/how-to/creating-catalogs.md @@ -54,18 +54,18 @@ USE CATALOG my_catalog; {{< tab "Spark3" >}} -The following shell command registers a Table Store catalog named `tablestore`. Metadata and table files are stored under `hdfs://path/to/warehouse`. +The following shell command registers a paimon catalog named `paimon`. Metadata and table files are stored under `hdfs://path/to/warehouse`. ```bash spark-sql ... \ - --conf spark.sql.catalog.tablestore=org.apache.paimon.spark.SparkCatalog \ - --conf spark.sql.catalog.tablestore.warehouse=hdfs://path/to/warehouse + --conf spark.sql.catalog.paimon=org.apache.paimon.spark.SparkCatalog \ + --conf spark.sql.catalog.paimon.warehouse=hdfs://path/to/warehouse ``` -After `spark-sql` is started, you can switch to the `default` database of the `tablestore` catalog with the following SQL. +After `spark-sql` is started, you can switch to the `default` database of the `paimon` catalog with the following SQL. ```sql -USE tablestore.default; +USE paimon.default; ``` {{< /tab >}} @@ -133,20 +133,20 @@ USE CATALOG my_hive; To enable Table Store Hive catalog support in Spark3, append the path of Table Store Hive catalog jar file to `--jars` argument when starting spark. -The following shell command registers a Table tore Hive catalog named `tablestore`. Metadata and table files are stored under `hdfs://path/to/warehouse`. In addition, metadata is also stored in Hive metastore. +The following shell command registers a Table tore Hive catalog named `paimon`. Metadata and table files are stored under `hdfs://path/to/warehouse`. In addition, metadata is also stored in Hive metastore. ```bash spark-sql ... \ - --conf spark.sql.catalog.tablestore=org.apache.paimon.spark.SparkCatalog \ - --conf spark.sql.catalog.tablestore.warehouse=hdfs://path/to/warehouse \ - --conf spark.sql.catalog.tablestore.metastore=hive \ - --conf spark.sql.catalog.tablestore.uri=thrift://: + --conf spark.sql.catalog.paimon=org.apache.paimon.spark.SparkCatalog \ + --conf spark.sql.catalog.paimon.warehouse=hdfs://path/to/warehouse \ + --conf spark.sql.catalog.paimon.metastore=hive \ + --conf spark.sql.catalog.paimon.uri=thrift://: ``` -After `spark-sql` is started, you can switch to the `default` database of the `tablestore` catalog with the following SQL. +After `spark-sql` is started, you can switch to the `default` database of the `paimon` catalog with the following SQL. ```sql -USE tablestore.default; +USE paimon.default; ``` {{< /tab >}} diff --git a/docs/content/how-to/creating-tables.md b/docs/content/how-to/creating-tables.md index 5253f437d958..9d585518f7ee 100644 --- a/docs/content/how-to/creating-tables.md +++ b/docs/content/how-to/creating-tables.md @@ -342,7 +342,7 @@ CREATE TABLE MyTable ( Spark3 only supports creating external tables through Scala API. The following Scala code loads the table located at `hdfs://path/to/table` into a `DataSet`. ```scala -val dataset = spark.read.format("tablestore").load("hdfs://path/to/table") +val dataset = spark.read.format("paimon").load("hdfs://path/to/table") ``` {{< /tab >}} @@ -352,7 +352,7 @@ val dataset = spark.read.format("tablestore").load("hdfs://path/to/table") Spark2 only supports creating external tables through Scala API. The following Scala code loads the table located at `hdfs://path/to/table` into a `DataSet`. ```scala -val dataset = spark.read.format("tablestore").load("hdfs://path/to/table") +val dataset = spark.read.format("paimon").load("hdfs://path/to/table") ``` {{< /tab >}} diff --git a/docs/content/how-to/lookup-joins.md b/docs/content/how-to/lookup-joins.md index ed0a8a5afaf3..04993afff0ea 100644 --- a/docs/content/how-to/lookup-joins.md +++ b/docs/content/how-to/lookup-joins.md @@ -33,7 +33,7 @@ Table Store supports lookup joins on unpartitioned tables with primary keys in F First, let's create a Table Store table and update it in real-time. ```sql --- Create a table store catalog +-- Create a paimon catalog CREATE CATALOG my_catalog WITH ( 'type'='paimon', 'warehouse'='hdfs://nn:8020/warehouse/path' -- or 'file://tmp/foo/bar' diff --git a/docs/content/maintenance/configurations.md b/docs/content/maintenance/configurations.md index 34ea60b0920e..721a8714a839 100644 --- a/docs/content/maintenance/configurations.md +++ b/docs/content/maintenance/configurations.md @@ -28,18 +28,18 @@ under the License. ### CoreOptions -Core options for table store. +Core options for paimon. {{< generated/core_configuration >}} ### CatalogOptions -Options for table store catalog. +Options for paimon catalog. {{< generated/catalog_configuration >}} ### FlinkConnectorOptions -Flink connector options for table store. +Flink connector options for paimon. {{< generated/flink_connector_configuration >}} diff --git a/docs/layouts/shortcodes/generated/catalog_configuration.html b/docs/layouts/shortcodes/generated/catalog_configuration.html index 7d2740df7b97..614763934ad4 100644 --- a/docs/layouts/shortcodes/generated/catalog_configuration.html +++ b/docs/layouts/shortcodes/generated/catalog_configuration.html @@ -36,7 +36,7 @@
metastore
"filesystem" String - Metastore of table store catalog, supports filesystem and hive. + Metastore of paimon catalog, supports filesystem and hive.
table.type
diff --git a/docs/layouts/shortcodes/generated/core_configuration.html b/docs/layouts/shortcodes/generated/core_configuration.html index d93339dc5435..ee58602421d2 100644 --- a/docs/layouts/shortcodes/generated/core_configuration.html +++ b/docs/layouts/shortcodes/generated/core_configuration.html @@ -24,7 +24,7 @@
bucket-key
(none) String - Specify the table store distribution policy. Data is assigned to each bucket according to the hash value of bucket-key.
If you specify multiple fields, delimiter is ','.
If not specified, the primary key will be used; if there is no primary key, the full row will be used. + Specify the paimon distribution policy. Data is assigned to each bucket according to the hash value of bucket-key.
If you specify multiple fields, delimiter is ','.
If not specified, the primary key will be used; if there is no primary key, the full row will be used.
changelog-producer
@@ -348,7 +348,7 @@
write-mode
change-log

Enum

- Specify the write mode for table.

Possible values:
  • "append-only": The table can only accept append-only insert operations. Neither data deduplication nor any primary key constraints will be done when inserting rows into table store.
  • "change-log": The table can accept insert/delete/update operations.
+ Specify the write mode for table.

Possible values:
  • "append-only": The table can only accept append-only insert operations. Neither data deduplication nor any primary key constraints will be done when inserting rows into paimon.
  • "change-log": The table can accept insert/delete/update operations.
write-only
diff --git a/docs/layouts/shortcodes/generated/flink_connector_configuration.html b/docs/layouts/shortcodes/generated/flink_connector_configuration.html index 092452c999ff..a5740c4b2ff8 100644 --- a/docs/layouts/shortcodes/generated/flink_connector_configuration.html +++ b/docs/layouts/shortcodes/generated/flink_connector_configuration.html @@ -42,7 +42,7 @@
sink.partition-shuffle
false Boolean - The option to enable shuffle data by dynamic partition fields in sink phase for table store. + The option to enable shuffle data by dynamic partition fields in sink phase for paimon.
streaming-read-atomic
diff --git a/paimon-benchmark/paimon-cluster-benchmark/src/main/resources/bin/metric_client.sh b/paimon-benchmark/paimon-cluster-benchmark/src/main/resources/bin/metric_client.sh index 57d837b247b5..b4c02f5f4594 100644 --- a/paimon-benchmark/paimon-cluster-benchmark/src/main/resources/bin/metric_client.sh +++ b/paimon-benchmark/paimon-cluster-benchmark/src/main/resources/bin/metric_client.sh @@ -34,7 +34,7 @@ case $STARTSTOP in (start) log=$BENCHMARK_LOG_DIR/metric-client.log log_setting=(-Dlog.file="$log" -Dlog4j.configuration=file:"$BENCHMARK_CONF_DIR"/log4j.properties -Dlog4j.configurationFile=file:"$BENCHMARK_CONF_DIR"/log4j.properties) - java "${log_setting[@]}" -cp "$BENCHMARK_HOME/lib/*:$FLINK_HOME/lib/*" org.apache.flink.table.store.benchmark.metric.cpu.CpuMetricSender & + java "${log_setting[@]}" -cp "$BENCHMARK_HOME/lib/*:$FLINK_HOME/lib/*" org.apache.paimon.benchmark.metric.cpu.CpuMetricSender & ;; (stop) diff --git a/paimon-benchmark/paimon-cluster-benchmark/src/main/resources/bin/run_benchmark.sh b/paimon-benchmark/paimon-cluster-benchmark/src/main/resources/bin/run_benchmark.sh index 492d2e51b541..c31de0d84854 100644 --- a/paimon-benchmark/paimon-cluster-benchmark/src/main/resources/bin/run_benchmark.sh +++ b/paimon-benchmark/paimon-cluster-benchmark/src/main/resources/bin/run_benchmark.sh @@ -27,4 +27,4 @@ bin=`cd "$bin"; pwd` log=$BENCHMARK_LOG_DIR/benchmark.log log_setting=(-Dlog.file="$log" -Dlog4j.configuration=file:"$BENCHMARK_CONF_DIR"/log4j.properties -Dlog4j.configurationFile=file:"$BENCHMARK_CONF_DIR"/log4j.properties) -java "${log_setting[@]}" -cp "$BENCHMARK_HOME/lib/*:$FLINK_HOME/lib/*" org.apache.flink.table.store.benchmark.Benchmark --location $BENCHMARK_HOME --queries $1 --sinks $2 \ No newline at end of file +java "${log_setting[@]}" -cp "$BENCHMARK_HOME/lib/*:$FLINK_HOME/lib/*" org.apache.paimon.benchmark.Benchmark --location $BENCHMARK_HOME --queries $1 --sinks $2 \ No newline at end of file diff --git a/paimon-benchmark/paimon-micro-benchmarks/src/test/java/org/apache/paimon/benchmark/TableBenchmark.java b/paimon-benchmark/paimon-micro-benchmarks/src/test/java/org/apache/paimon/benchmark/TableBenchmark.java index f31e14729fa6..023e37f89d46 100644 --- a/paimon-benchmark/paimon-micro-benchmarks/src/test/java/org/apache/paimon/benchmark/TableBenchmark.java +++ b/paimon-benchmark/paimon-micro-benchmarks/src/test/java/org/apache/paimon/benchmark/TableBenchmark.java @@ -18,11 +18,10 @@ package org.apache.paimon.benchmark; -import org.apache.flink.table.store.catalog.CatalogContext; - import org.apache.commons.math3.random.RandomDataGenerator; import org.apache.paimon.CoreOptions; import org.apache.paimon.catalog.Catalog; +import org.apache.paimon.catalog.CatalogContext; import org.apache.paimon.catalog.CatalogFactory; import org.apache.paimon.catalog.Identifier; import org.apache.paimon.data.BinaryString; diff --git a/paimon-common/src/main/java/org/apache/paimon/catalog/CatalogContext.java b/paimon-common/src/main/java/org/apache/paimon/catalog/CatalogContext.java index a4411e9da43f..ae94b3e58978 100644 --- a/paimon-common/src/main/java/org/apache/paimon/catalog/CatalogContext.java +++ b/paimon-common/src/main/java/org/apache/paimon/catalog/CatalogContext.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.flink.table.store.catalog; +package org.apache.paimon.catalog; import org.apache.hadoop.conf.Configuration; import org.apache.paimon.annotation.Experimental; diff --git a/paimon-common/src/main/java/org/apache/paimon/fs/FileIO.java b/paimon-common/src/main/java/org/apache/paimon/fs/FileIO.java index fcf757ce8b1b..9fff5a49d2f2 100644 --- a/paimon-common/src/main/java/org/apache/paimon/fs/FileIO.java +++ b/paimon-common/src/main/java/org/apache/paimon/fs/FileIO.java @@ -18,9 +18,8 @@ package org.apache.paimon.fs; -import org.apache.flink.table.store.catalog.CatalogContext; - import org.apache.paimon.annotation.Experimental; +import org.apache.paimon.catalog.CatalogContext; import org.apache.paimon.fs.local.LocalFileIO; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/paimon-common/src/main/java/org/apache/paimon/fs/PluginFileIO.java b/paimon-common/src/main/java/org/apache/paimon/fs/PluginFileIO.java index b0b660e49ab5..cbde728a3017 100644 --- a/paimon-common/src/main/java/org/apache/paimon/fs/PluginFileIO.java +++ b/paimon-common/src/main/java/org/apache/paimon/fs/PluginFileIO.java @@ -18,8 +18,7 @@ package org.apache.paimon.fs; -import org.apache.flink.table.store.catalog.CatalogContext; - +import org.apache.paimon.catalog.CatalogContext; import org.apache.paimon.options.Options; import java.io.IOException; diff --git a/paimon-common/src/main/java/org/apache/paimon/fs/hadoop/HadoopFileIO.java b/paimon-common/src/main/java/org/apache/paimon/fs/hadoop/HadoopFileIO.java index c7dfaae3d00e..f113a034713b 100644 --- a/paimon-common/src/main/java/org/apache/paimon/fs/hadoop/HadoopFileIO.java +++ b/paimon-common/src/main/java/org/apache/paimon/fs/hadoop/HadoopFileIO.java @@ -18,12 +18,11 @@ package org.apache.paimon.fs.hadoop; -import org.apache.flink.table.store.catalog.CatalogContext; - import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.paimon.annotation.VisibleForTesting; +import org.apache.paimon.catalog.CatalogContext; import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.FileStatus; import org.apache.paimon.fs.Path; diff --git a/paimon-common/src/main/java/org/apache/paimon/fs/local/LocalFileIO.java b/paimon-common/src/main/java/org/apache/paimon/fs/local/LocalFileIO.java index c66ec95d16d1..3f19bf3164ca 100644 --- a/paimon-common/src/main/java/org/apache/paimon/fs/local/LocalFileIO.java +++ b/paimon-common/src/main/java/org/apache/paimon/fs/local/LocalFileIO.java @@ -18,8 +18,7 @@ package org.apache.paimon.fs.local; -import org.apache.flink.table.store.catalog.CatalogContext; - +import org.apache.paimon.catalog.CatalogContext; import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.FileStatus; import org.apache.paimon.fs.Path; diff --git a/paimon-common/src/main/java/org/apache/paimon/options/CatalogOptions.java b/paimon-common/src/main/java/org/apache/paimon/options/CatalogOptions.java index e4a3f26bf0bf..e853b67c5749 100644 --- a/paimon-common/src/main/java/org/apache/paimon/options/CatalogOptions.java +++ b/paimon-common/src/main/java/org/apache/paimon/options/CatalogOptions.java @@ -37,8 +37,7 @@ public class CatalogOptions { ConfigOptions.key("metastore") .stringType() .defaultValue("filesystem") - .withDescription( - "Metastore of table store catalog, supports filesystem and hive."); + .withDescription("Metastore of paimon catalog, supports filesystem and hive."); public static final ConfigOption URI = ConfigOptions.key("uri") diff --git a/paimon-common/src/main/java/org/apache/paimon/types/DataType.java b/paimon-common/src/main/java/org/apache/paimon/types/DataType.java index 89fe7fea1b5d..59d36260522b 100644 --- a/paimon-common/src/main/java/org/apache/paimon/types/DataType.java +++ b/paimon-common/src/main/java/org/apache/paimon/types/DataType.java @@ -32,7 +32,7 @@ import java.util.Set; /** - * Describes the data type in the table store ecosystem. + * Describes the data type in the paimon ecosystem. * * @see DataTypes * @since 0.4.0 diff --git a/paimon-common/src/test/java/org/apache/paimon/fs/FileIOFinder.java b/paimon-common/src/test/java/org/apache/paimon/fs/FileIOFinder.java index b4d13e3acf7e..178129daa83b 100644 --- a/paimon-common/src/test/java/org/apache/paimon/fs/FileIOFinder.java +++ b/paimon-common/src/test/java/org/apache/paimon/fs/FileIOFinder.java @@ -18,8 +18,7 @@ package org.apache.paimon.fs; -import org.apache.flink.table.store.catalog.CatalogContext; - +import org.apache.paimon.catalog.CatalogContext; import org.apache.paimon.options.Options; import java.io.IOException; diff --git a/paimon-core/src/main/java/org/apache/paimon/CoreOptions.java b/paimon-core/src/main/java/org/apache/paimon/CoreOptions.java index 4a046c7f046e..9da783634942 100644 --- a/paimon-core/src/main/java/org/apache/paimon/CoreOptions.java +++ b/paimon-core/src/main/java/org/apache/paimon/CoreOptions.java @@ -44,7 +44,7 @@ import static org.apache.paimon.options.ConfigOptions.key; import static org.apache.paimon.options.description.TextElement.text; -/** Core options for table store. */ +/** Core options for paimon. */ public class CoreOptions implements Serializable { public static final ConfigOption BUCKET = @@ -61,7 +61,7 @@ public class CoreOptions implements Serializable { .withDescription( Description.builder() .text( - "Specify the table store distribution policy. Data is assigned" + "Specify the paimon distribution policy. Data is assigned" + " to each bucket according to the hash value of bucket-key.") .linebreak() .text("If you specify multiple fields, delimiter is ','.") diff --git a/paimon-core/src/main/java/org/apache/paimon/Snapshot.java b/paimon-core/src/main/java/org/apache/paimon/Snapshot.java index e953a1b0cd41..a51ce69cae11 100644 --- a/paimon-core/src/main/java/org/apache/paimon/Snapshot.java +++ b/paimon-core/src/main/java/org/apache/paimon/Snapshot.java @@ -44,21 +44,19 @@ *

Versioned change list: * *

    - *
  • Version 1: Initial version for table store <= 0.2. There is no "version" field in json - * file. - *
  • Version 2: Introduced in table store 0.3. Add "version" field and "changelogManifestList" - * field. - *
  • Version 3: Introduced in table store 0.4. Add "baseRecordCount" field, "deltaRecordCount" - * field and "changelogRecordCount" field. + *
  • Version 1: Initial version for paimon <= 0.2. There is no "version" field in json file. + *
  • Version 2: Introduced in paimon 0.3. Add "version" field and "changelogManifestList" field. + *
  • Version 3: Introduced in paimon 0.4. Add "baseRecordCount" field, "deltaRecordCount" field + * and "changelogRecordCount" field. *
* *

Unversioned change list: * *

    - *
  • Since table store 0.22 and table store 0.3, commitIdentifier is changed from a String to a - * long value. For table store < 0.22, only Flink connectors have table store sink and they - * use checkpointId as commitIdentifier (which is a long value). Json can automatically - * perform type conversion so there is no compatibility issue. + *
  • Since paimon 0.22 and paimon 0.3, commitIdentifier is changed from a String to a long + * value. For paimon < 0.22, only Flink connectors have paimon sink and they use checkpointId + * as commitIdentifier (which is a long value). Json can automatically perform type conversion + * so there is no compatibility issue. *
*/ public class Snapshot { @@ -85,7 +83,7 @@ public class Snapshot { private static final String FILED_WATERMARK = "watermark"; // version of snapshot - // null for table store <= 0.2 + // null for paimon <= 0.2 @JsonProperty(FIELD_VERSION) @Nullable private final Integer version; @@ -106,7 +104,7 @@ public class Snapshot { private final String deltaManifestList; // a manifest list recording all changelog produced in this snapshot - // null if no changelog is produced, or for table store <= 0.2 + // null if no changelog is produced, or for paimon <= 0.2 @JsonProperty(FIELD_CHANGELOG_MANIFEST_LIST) @Nullable private final String changelogManifestList; @@ -134,25 +132,25 @@ public class Snapshot { private final Map logOffsets; // record count of all changes occurred in this snapshot - // null for table store <= 0.3 + // null for paimon <= 0.3 @JsonProperty(FIELD_TOTAL_RECORD_COUNT) @Nullable private final Long totalRecordCount; // record count of all new changes occurred in this snapshot - // null for table store <= 0.3 + // null for paimon <= 0.3 @JsonProperty(FIELD_DELTA_RECORD_COUNT) @Nullable private final Long deltaRecordCount; // record count of all changelog produced in this snapshot - // null for table store <= 0.3 + // null for paimon <= 0.3 @JsonProperty(FIELD_CHANGELOG_RECORD_COUNT) @Nullable private final Long changelogRecordCount; // watermark for input records - // null for table store <= 0.3 + // null for paimon <= 0.3 // null if there is no watermark in new committing, and the previous snapshot does not have a // watermark @JsonProperty(FILED_WATERMARK) @@ -228,7 +226,7 @@ public Snapshot( @JsonGetter(FIELD_VERSION) public int version() { - // there is no version field for table store <= 0.2 + // there is no version field for paimon <= 0.2 return version == null ? TABLE_STORE_02_VERSION : version; } diff --git a/paimon-core/src/main/java/org/apache/paimon/WriteMode.java b/paimon-core/src/main/java/org/apache/paimon/WriteMode.java index ea353845fbe3..13d221e7c709 100644 --- a/paimon-core/src/main/java/org/apache/paimon/WriteMode.java +++ b/paimon-core/src/main/java/org/apache/paimon/WriteMode.java @@ -23,12 +23,12 @@ import static org.apache.paimon.options.description.TextElement.text; -/** Defines the write mode for flink table store. */ +/** Defines the write mode for flink paimon. */ public enum WriteMode implements DescribedEnum { APPEND_ONLY( "append-only", "The table can only accept append-only insert operations. Neither data deduplication nor any " - + "primary key constraints will be done when inserting rows into table store."), + + "primary key constraints will be done when inserting rows into paimon."), CHANGE_LOG("change-log", "The table can accept insert/delete/update operations."); private final String value; diff --git a/paimon-core/src/main/java/org/apache/paimon/catalog/Catalog.java b/paimon-core/src/main/java/org/apache/paimon/catalog/Catalog.java index 29d561df0cba..8e2814e0a1c9 100644 --- a/paimon-core/src/main/java/org/apache/paimon/catalog/Catalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/catalog/Catalog.java @@ -28,7 +28,7 @@ /** * This interface is responsible for reading and writing metadata such as database/table from a - * table store catalog. + * paimon catalog. * * @see CatalogFactory * @since 0.4.0 diff --git a/paimon-core/src/main/java/org/apache/paimon/catalog/CatalogFactory.java b/paimon-core/src/main/java/org/apache/paimon/catalog/CatalogFactory.java index 611da3e18c21..25717c6cb385 100644 --- a/paimon-core/src/main/java/org/apache/paimon/catalog/CatalogFactory.java +++ b/paimon-core/src/main/java/org/apache/paimon/catalog/CatalogFactory.java @@ -18,8 +18,6 @@ package org.apache.paimon.catalog; -import org.apache.flink.table.store.catalog.CatalogContext; - import org.apache.paimon.annotation.Experimental; import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.Path; @@ -52,7 +50,7 @@ static Path warehouse(CatalogContext context) { String warehouse = Preconditions.checkNotNull( context.options().get(WAREHOUSE), - "Table store '" + WAREHOUSE.key() + "' path must be set"); + "Paimon '" + WAREHOUSE.key() + "' path must be set"); return new Path(warehouse); } diff --git a/paimon-core/src/main/java/org/apache/paimon/catalog/FileSystemCatalogFactory.java b/paimon-core/src/main/java/org/apache/paimon/catalog/FileSystemCatalogFactory.java index 84afbaa5360b..bb09de29dcb3 100644 --- a/paimon-core/src/main/java/org/apache/paimon/catalog/FileSystemCatalogFactory.java +++ b/paimon-core/src/main/java/org/apache/paimon/catalog/FileSystemCatalogFactory.java @@ -18,8 +18,6 @@ package org.apache.paimon.catalog; -import org.apache.flink.table.store.catalog.CatalogContext; - import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.Path; import org.apache.paimon.table.TableType; diff --git a/paimon-core/src/main/java/org/apache/paimon/sort/BinaryInMemorySortBuffer.java b/paimon-core/src/main/java/org/apache/paimon/sort/BinaryInMemorySortBuffer.java index 3692157f8f7d..88c632f90620 100644 --- a/paimon-core/src/main/java/org/apache/paimon/sort/BinaryInMemorySortBuffer.java +++ b/paimon-core/src/main/java/org/apache/paimon/sort/BinaryInMemorySortBuffer.java @@ -45,7 +45,7 @@ *
  • 1. Add clear method to clean all memory. *
  • 2. Add tryInitialized() method to initialize memory before write and read in buffer, while * the old buffer will do it in the constructor and reset(). - *
  • 3. Remove reset() and etc. methods which are not used in flink table store. + *
  • 3. Remove reset() and etc. methods which are not used in flink paimon. * */ public class BinaryInMemorySortBuffer extends BinaryIndexedSortable implements SortBuffer { diff --git a/paimon-core/src/main/java/org/apache/paimon/table/AbstractFileStoreTable.java b/paimon-core/src/main/java/org/apache/paimon/table/AbstractFileStoreTable.java index 3819924b83d4..9f19331ae7b5 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/AbstractFileStoreTable.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/AbstractFileStoreTable.java @@ -119,7 +119,7 @@ public FileStoreTable copy(Map dynamicOptions) { // set dynamic options with default values CoreOptions.setDefaultValues(newOptions); - // copy a new table store to contain dynamic options + // copy a new paimon to contain dynamic options TableSchema newTableSchema = tableSchema.copy(newOptions.toMap()); // validate schema wit new options diff --git a/paimon-core/src/main/java/org/apache/paimon/table/FileStoreTableFactory.java b/paimon-core/src/main/java/org/apache/paimon/table/FileStoreTableFactory.java index 886a4fcca0fe..7a0d099b8ab3 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/FileStoreTableFactory.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/FileStoreTableFactory.java @@ -18,10 +18,9 @@ package org.apache.paimon.table; -import org.apache.flink.table.store.catalog.CatalogContext; - import org.apache.paimon.CoreOptions; import org.apache.paimon.WriteMode; +import org.apache.paimon.catalog.CatalogContext; import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.Path; import org.apache.paimon.options.Options; diff --git a/paimon-core/src/main/java/org/apache/paimon/utils/JsonSerdeUtil.java b/paimon-core/src/main/java/org/apache/paimon/utils/JsonSerdeUtil.java index d088ea060ebe..4a61dde664f6 100644 --- a/paimon-core/src/main/java/org/apache/paimon/utils/JsonSerdeUtil.java +++ b/paimon-core/src/main/java/org/apache/paimon/utils/JsonSerdeUtil.java @@ -77,7 +77,7 @@ public static String toFlatJson(T t) { } private static Module createTableStoreJacksonModule() { - SimpleModule module = new SimpleModule("Table store"); + SimpleModule module = new SimpleModule("Paimon"); registerJsonObjects( module, TableSchema.class, SchemaSerializer.INSTANCE, SchemaSerializer.INSTANCE); registerJsonObjects( diff --git a/paimon-core/src/main/resources/META-INF/services/org.apache.paimon.file.org.apache.paimon.catalog.CatalogFactory b/paimon-core/src/main/resources/META-INF/services/org.apache.paimon.catalog.CatalogFactory similarity index 100% rename from paimon-core/src/main/resources/META-INF/services/org.apache.paimon.file.org.apache.paimon.catalog.CatalogFactory rename to paimon-core/src/main/resources/META-INF/services/org.apache.paimon.catalog.CatalogFactory diff --git a/paimon-core/src/test/java/org/apache/paimon/catalog/CatalogFactoryTest.java b/paimon-core/src/test/java/org/apache/paimon/catalog/CatalogFactoryTest.java index f9076427e32c..0123c2cc4f6a 100644 --- a/paimon-core/src/test/java/org/apache/paimon/catalog/CatalogFactoryTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/catalog/CatalogFactoryTest.java @@ -18,8 +18,6 @@ package org.apache.paimon.catalog; -import org.apache.flink.table.store.catalog.CatalogContext; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.paimon.fs.Path; diff --git a/paimon-core/src/test/java/org/apache/paimon/utils/TraceableFileIO.java b/paimon-core/src/test/java/org/apache/paimon/utils/TraceableFileIO.java index 4418fdd10466..b3d3a937c30c 100644 --- a/paimon-core/src/test/java/org/apache/paimon/utils/TraceableFileIO.java +++ b/paimon-core/src/test/java/org/apache/paimon/utils/TraceableFileIO.java @@ -18,8 +18,7 @@ package org.apache.paimon.utils; -import org.apache.flink.table.store.catalog.CatalogContext; - +import org.apache.paimon.catalog.CatalogContext; import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.FileIOLoader; import org.apache.paimon.fs.FileStatus; diff --git a/paimon-e2e-tests/pom.xml b/paimon-e2e-tests/pom.xml index 6833e4195b2e..dc367fdf024c 100644 --- a/paimon-e2e-tests/pom.xml +++ b/paimon-e2e-tests/pom.xml @@ -150,7 +150,7 @@ under the License. /tmp/paimon-e2e-tests-jars - + org.apache.flink ${flink.sql.connector.kafka} diff --git a/paimon-e2e-tests/src/test/java/org/apache/paimon/tests/FileStoreBatchE2eTest.java b/paimon-e2e-tests/src/test/java/org/apache/paimon/tests/FileStoreBatchE2eTest.java index 9c6554991334..aed5b06863ad 100644 --- a/paimon-e2e-tests/src/test/java/org/apache/paimon/tests/FileStoreBatchE2eTest.java +++ b/paimon-e2e-tests/src/test/java/org/apache/paimon/tests/FileStoreBatchE2eTest.java @@ -92,7 +92,7 @@ public void testWithoutPk() throws Exception { // prepare test data writeSharedFile(testDataSourceFile, String.join("\n", data)); - // insert data into table store + // insert data into paimon runSql( "INSERT INTO ts_table SELECT * FROM test_source;", catalogDdl, @@ -100,7 +100,7 @@ public void testWithoutPk() throws Exception { testDataSourceDdl, tableStoreDdl); - // test #1: read all data from table store + // test #1: read all data from paimon runSql( "INSERT INTO result1 SELECT * FROM ts_table;", catalogDdl, diff --git a/paimon-e2e-tests/src/test/java/org/apache/paimon/tests/FileStoreStreamE2eTest.java b/paimon-e2e-tests/src/test/java/org/apache/paimon/tests/FileStoreStreamE2eTest.java index de86aa9db276..c7273daf91c6 100644 --- a/paimon-e2e-tests/src/test/java/org/apache/paimon/tests/FileStoreStreamE2eTest.java +++ b/paimon-e2e-tests/src/test/java/org/apache/paimon/tests/FileStoreStreamE2eTest.java @@ -78,7 +78,7 @@ public void testWithoutPk() throws Exception { + " 'bucket' = '3'\n" + ");"; - // insert data into table store + // insert data into paimon runSql( "SET 'execution.checkpointing.interval' = '5s';\n" + "INSERT INTO ts_table SELECT a, b, rn FROM (\n" @@ -89,7 +89,7 @@ public void testWithoutPk() throws Exception { tableDdl, testDataSourceDdl); - // read all data from table store + // read all data from paimon runSql( "INSERT INTO result1 SELECT * FROM ts_table;", catalogDdl, diff --git a/paimon-e2e-tests/src/test/java/org/apache/paimon/tests/FlinkActionsE2eTest.java b/paimon-e2e-tests/src/test/java/org/apache/paimon/tests/FlinkActionsE2eTest.java index 51dba9c393a8..5c82e5195d7c 100644 --- a/paimon-e2e-tests/src/test/java/org/apache/paimon/tests/FlinkActionsE2eTest.java +++ b/paimon-e2e-tests/src/test/java/org/apache/paimon/tests/FlinkActionsE2eTest.java @@ -88,7 +88,7 @@ public void testCompact() throws Exception { + " 'write-only' = 'true'\n" + ");"; - // insert data into table store + // insert data into paimon runSql( "SET 'execution.checkpointing.interval' = '1s';\n" + "INSERT INTO ts_table SELECT * FROM test_source;", @@ -123,7 +123,7 @@ public void testCompact() throws Exception { LOG.info(execResult.getStdout()); LOG.info(execResult.getStderr()); - // read all data from table store + // read all data from paimon runSql( "INSERT INTO result1 SELECT * FROM ts_table;", catalogDdl, @@ -185,7 +185,7 @@ public void testDropPartition() throws Exception { LOG.info(execResult.getStdout()); LOG.info(execResult.getStderr()); - // read all data from table store + // read all data from paimon runSql( "INSERT INTO result1 SELECT * FROM ts_table;", catalogDdl, @@ -244,7 +244,7 @@ public void testDelete() throws Exception { LOG.info(execResult.getStdout()); LOG.info(execResult.getStderr()); - // read all data from table store + // read all data from paimon runSql( "INSERT INTO result1 SELECT * FROM ts_table;", catalogDdl, @@ -318,7 +318,7 @@ public void testMergeInto() throws Exception { LOG.info(execResult.getStdout()); LOG.info(execResult.getStderr()); - // read all data from table store + // read all data from paimon runSql( "INSERT INTO result1 SELECT * FROM T;", catalogDdl, diff --git a/paimon-e2e-tests/src/test/java/org/apache/paimon/tests/HiveE2eTest.java b/paimon-e2e-tests/src/test/java/org/apache/paimon/tests/HiveE2eTest.java index 1a8215ac99d5..a3378b2ab710 100644 --- a/paimon-e2e-tests/src/test/java/org/apache/paimon/tests/HiveE2eTest.java +++ b/paimon-e2e-tests/src/test/java/org/apache/paimon/tests/HiveE2eTest.java @@ -27,7 +27,7 @@ import java.util.UUID; /** - * Tests for reading table store from Hive. + * Tests for reading paimon from Hive. * *

    NOTE: This test runs a complete Hadoop cluster in Docker, which requires a lot of memory. If * you're running this test locally, make sure that the memory limit of your Docker is at least 8GB. diff --git a/paimon-e2e-tests/src/test/java/org/apache/paimon/tests/LogStoreE2eTest.java b/paimon-e2e-tests/src/test/java/org/apache/paimon/tests/LogStoreE2eTest.java index bc6c074cd869..ec06bd2eb53e 100644 --- a/paimon-e2e-tests/src/test/java/org/apache/paimon/tests/LogStoreE2eTest.java +++ b/paimon-e2e-tests/src/test/java/org/apache/paimon/tests/LogStoreE2eTest.java @@ -101,7 +101,7 @@ public void testWithPk() throws Exception { + ");", testTopicName); - // insert data into table store + // insert data into paimon runSql( // long checkpoint interval ensures that new data are only visible from log store "SET 'execution.checkpointing.interval' = '9999s';\n" @@ -111,7 +111,7 @@ public void testWithPk() throws Exception { streamTableDdl, testDataSourceDdl); - // read all data from table store + // read all data from paimon runSql( "INSERT INTO result1 SELECT * FROM ts_table;", catalogDdl, diff --git a/paimon-e2e-tests/src/test/java/org/apache/paimon/tests/SparkE2eTest.java b/paimon-e2e-tests/src/test/java/org/apache/paimon/tests/SparkE2eTest.java index 61dd25c8d346..b8e8752168f2 100644 --- a/paimon-e2e-tests/src/test/java/org/apache/paimon/tests/SparkE2eTest.java +++ b/paimon-e2e-tests/src/test/java/org/apache/paimon/tests/SparkE2eTest.java @@ -29,7 +29,7 @@ import java.util.UUID; import java.util.stream.Collectors; -/** Tests for reading table store from Spark3. */ +/** Tests for reading paimon from Spark3. */ @DisabledIfSystemProperty(named = "test.flink.version", matches = "1.14.*") public class SparkE2eTest extends E2eReaderTestBase { @@ -43,7 +43,7 @@ public SparkE2eTest() { public void testFlinkWriteAndSparkRead() throws Exception { String warehousePath = TEST_DATA_DIR + "/" + UUID.randomUUID().toString() + "_warehouse"; final String table = "T"; - final String sparkTable = String.format("tablestore.default.%s", table); + final String sparkTable = String.format("paimon.default.%s", table); runSql( String.join( "\n", @@ -60,9 +60,9 @@ public void testFlinkWriteAndSparkRead() throws Exception { "--master", "spark://spark-master:7077", "--conf", - "spark.sql.catalog.tablestore=org.apache.paimon.spark.SparkCatalog", + "spark.sql.catalog.paimon=org.apache.paimon.spark.SparkCatalog", "--conf", - "spark.sql.catalog.tablestore.warehouse=file:" + "spark.sql.catalog.paimon.warehouse=file:" + warehousePath, "-f", TEST_DATA_DIR + "/" + sql); diff --git a/paimon-filesystems/paimon-oss-impl/src/main/java/org/apache/paimon/oss/OSSFileIO.java b/paimon-filesystems/paimon-oss-impl/src/main/java/org/apache/paimon/oss/OSSFileIO.java index 1d62acab330a..9f705378644f 100644 --- a/paimon-filesystems/paimon-oss-impl/src/main/java/org/apache/paimon/oss/OSSFileIO.java +++ b/paimon-filesystems/paimon-oss-impl/src/main/java/org/apache/paimon/oss/OSSFileIO.java @@ -18,11 +18,10 @@ package org.apache.paimon.oss; -import org.apache.flink.table.store.catalog.CatalogContext; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.aliyun.oss.AliyunOSSFileSystem; +import org.apache.paimon.catalog.CatalogContext; import org.apache.paimon.fs.FileIO; import org.apache.paimon.options.Options; import org.slf4j.Logger; diff --git a/paimon-filesystems/paimon-oss/src/main/java/org/apache/paimon/oss/OSSLoader.java b/paimon-filesystems/paimon-oss/src/main/java/org/apache/paimon/oss/OSSLoader.java index 9a5c4cdd4fce..1bc160ca569e 100644 --- a/paimon-filesystems/paimon-oss/src/main/java/org/apache/paimon/oss/OSSLoader.java +++ b/paimon-filesystems/paimon-oss/src/main/java/org/apache/paimon/oss/OSSLoader.java @@ -18,8 +18,7 @@ package org.apache.paimon.oss; -import org.apache.flink.table.store.catalog.CatalogContext; - +import org.apache.paimon.catalog.CatalogContext; import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.FileIOLoader; import org.apache.paimon.fs.Path; diff --git a/paimon-filesystems/paimon-s3-impl/src/main/java/org/apache/paimon/s3/S3FileIO.java b/paimon-filesystems/paimon-s3-impl/src/main/java/org/apache/paimon/s3/S3FileIO.java index 82bdd54b42cd..4222f0d328d5 100644 --- a/paimon-filesystems/paimon-s3-impl/src/main/java/org/apache/paimon/s3/S3FileIO.java +++ b/paimon-filesystems/paimon-s3-impl/src/main/java/org/apache/paimon/s3/S3FileIO.java @@ -18,11 +18,10 @@ package org.apache.paimon.s3; -import org.apache.flink.table.store.catalog.CatalogContext; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.s3a.S3AFileSystem; +import org.apache.paimon.catalog.CatalogContext; import org.apache.paimon.fs.FileIO; import org.apache.paimon.options.Options; import org.slf4j.Logger; diff --git a/paimon-filesystems/paimon-s3/src/main/java/org/apache/paimon/s3/S3Loader.java b/paimon-filesystems/paimon-s3/src/main/java/org/apache/paimon/s3/S3Loader.java index a823673733c9..98c39e1703e9 100644 --- a/paimon-filesystems/paimon-s3/src/main/java/org/apache/paimon/s3/S3Loader.java +++ b/paimon-filesystems/paimon-s3/src/main/java/org/apache/paimon/s3/S3Loader.java @@ -18,8 +18,7 @@ package org.apache.paimon.s3; -import org.apache.flink.table.store.catalog.CatalogContext; - +import org.apache.paimon.catalog.CatalogContext; import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.FileIOLoader; import org.apache.paimon.fs.Path; diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/connector/AbstractTableStoreFactory.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/connector/AbstractTableStoreFactory.java index e1da476aa96c..d4431e616ece 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/connector/AbstractTableStoreFactory.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/connector/AbstractTableStoreFactory.java @@ -28,12 +28,12 @@ import org.apache.flink.table.factories.DynamicTableFactory; import org.apache.flink.table.factories.DynamicTableSinkFactory; import org.apache.flink.table.factories.DynamicTableSourceFactory; -import org.apache.flink.table.store.catalog.CatalogContext; import org.apache.flink.table.types.logical.RowType; import org.apache.paimon.CoreOptions.LogChangelogMode; import org.apache.paimon.CoreOptions.LogConsistency; import org.apache.paimon.annotation.VisibleForTesting; +import org.apache.paimon.catalog.CatalogContext; import org.apache.paimon.connector.log.LogStoreTableFactory; import org.apache.paimon.connector.sink.TableStoreSink; import org.apache.paimon.connector.source.SystemTableSource; @@ -59,7 +59,7 @@ import static org.apache.paimon.connector.LogicalTypeConversion.toLogicalType; import static org.apache.paimon.connector.log.LogStoreTableFactory.discoverLogStoreFactory; -/** Abstract table store factory to create table source and table sink. */ +/** Abstract paimon factory to create table source and table sink. */ public abstract class AbstractTableStoreFactory implements DynamicTableSourceFactory, DynamicTableSinkFactory { diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/connector/FlinkCatalog.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/connector/FlinkCatalog.java index 90bb7b537417..ee71bbf5cdbc 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/connector/FlinkCatalog.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/connector/FlinkCatalog.java @@ -67,7 +67,7 @@ import static org.apache.paimon.connector.LogicalTypeConversion.toDataType; import static org.apache.paimon.connector.LogicalTypeConversion.toLogicalType; -/** Catalog for table store. */ +/** Catalog for paimon. */ public class FlinkCatalog extends AbstractCatalog { private final Catalog catalog; @@ -204,7 +204,7 @@ public void createTable(ObjectPath tablePath, CatalogBaseTable table, boolean ig if (options.containsKey(CONNECTOR.key())) { throw new CatalogException( String.format( - "Table Store Catalog only supports table store tables, not '%s' connector." + "Table Store Catalog only supports paimon tables, not '%s' connector." + " You can create TEMPORARY table instead.", options.get(CONNECTOR.key()))); } diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/connector/FlinkCatalogFactory.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/connector/FlinkCatalogFactory.java index 9474c667bac4..4620afa430dc 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/connector/FlinkCatalogFactory.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/connector/FlinkCatalogFactory.java @@ -18,9 +18,8 @@ package org.apache.paimon.connector; -import org.apache.flink.table.store.catalog.CatalogContext; - import org.apache.paimon.catalog.Catalog; +import org.apache.paimon.catalog.CatalogContext; import org.apache.paimon.catalog.CatalogFactory; import org.apache.paimon.options.ConfigOption; import org.apache.paimon.options.ConfigOptions; diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/connector/FlinkConnectorOptions.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/connector/FlinkConnectorOptions.java index bc1b75a44172..644a2f1b7083 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/connector/FlinkConnectorOptions.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/connector/FlinkConnectorOptions.java @@ -71,7 +71,7 @@ public class FlinkConnectorOptions { .booleanType() .defaultValue(false) .withDescription( - "The option to enable shuffle data by dynamic partition fields in sink phase for table store."); + "The option to enable shuffle data by dynamic partition fields in sink phase for paimon."); public static final ConfigOption SCAN_PARALLELISM = ConfigOptions.key("scan.parallelism") diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/connector/FlinkFileIO.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/connector/FlinkFileIO.java index f3272fb502d6..42e62893dbb9 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/connector/FlinkFileIO.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/connector/FlinkFileIO.java @@ -23,8 +23,8 @@ import org.apache.flink.core.fs.FileSystem; import org.apache.flink.core.fs.FileSystem.WriteMode; import org.apache.flink.core.fs.FileSystemKind; -import org.apache.flink.table.store.catalog.CatalogContext; +import org.apache.paimon.catalog.CatalogContext; import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.FileStatus; import org.apache.paimon.fs.Path; diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/connector/FlinkUtils.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/connector/FlinkUtils.java index 462578c29af5..7ac8663738e3 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/connector/FlinkUtils.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/connector/FlinkUtils.java @@ -19,8 +19,8 @@ package org.apache.paimon.connector; import org.apache.flink.configuration.ReadableConfig; -import org.apache.flink.table.store.catalog.CatalogContext; +import org.apache.paimon.catalog.CatalogContext; import org.apache.paimon.options.Options; import java.util.Map; diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/connector/TableStoreConnectorFactory.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/connector/TableStoreConnectorFactory.java index a78e38c5201b..4159bba8fd9c 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/connector/TableStoreConnectorFactory.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/connector/TableStoreConnectorFactory.java @@ -37,7 +37,7 @@ import static org.apache.paimon.CoreOptions.AUTO_CREATE; import static org.apache.paimon.connector.FlinkCatalogFactory.IDENTIFIER; -/** A table store {@link DynamicTableFactory} to create source and sink. */ +/** A paimon {@link DynamicTableFactory} to create source and sink. */ public class TableStoreConnectorFactory extends AbstractTableStoreFactory { @Nullable private final CatalogLock.Factory lockFactory; diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/connector/action/ActionBase.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/connector/action/ActionBase.java index 385b0612cfcd..f29b0443f67d 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/connector/action/ActionBase.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/connector/action/ActionBase.java @@ -24,10 +24,10 @@ import org.apache.flink.table.api.EnvironmentSettings; import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; import org.apache.flink.table.data.RowData; -import org.apache.flink.table.store.catalog.CatalogContext; import org.apache.flink.table.types.logical.LogicalType; import org.apache.paimon.catalog.Catalog; +import org.apache.paimon.catalog.CatalogContext; import org.apache.paimon.catalog.CatalogFactory; import org.apache.paimon.catalog.Identifier; import org.apache.paimon.connector.FlinkCatalog; diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/connector/sink/FileStoreSink.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/connector/sink/FileStoreSink.java index 2b6e86a9a4aa..b1a3bfaa6387 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/connector/sink/FileStoreSink.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/connector/sink/FileStoreSink.java @@ -31,7 +31,7 @@ import java.util.Map; -/** {@link FlinkSink} for writing records into table store. */ +/** {@link FlinkSink} for writing records into paimon. */ public class FileStoreSink extends FlinkSink { private static final long serialVersionUID = 1L; diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/connector/sink/FlinkSink.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/connector/sink/FlinkSink.java index 2e39d76e39a7..e7097c4164b6 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/connector/sink/FlinkSink.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/connector/sink/FlinkSink.java @@ -44,7 +44,7 @@ import static org.apache.paimon.connector.FlinkConnectorOptions.CHANGELOG_PRODUCER_FULL_COMPACTION_TRIGGER_INTERVAL; import static org.apache.paimon.connector.FlinkConnectorOptions.CHANGELOG_PRODUCER_LOOKUP_WAIT; -/** Abstract sink of table store. */ +/** Abstract sink of paimon. */ public abstract class FlinkSink implements Serializable { private static final long serialVersionUID = 1L; diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/connector/sink/StoreSinkWrite.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/connector/sink/StoreSinkWrite.java index efd4f71b4126..5cb6af3281f6 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/connector/sink/StoreSinkWrite.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/connector/sink/StoreSinkWrite.java @@ -32,7 +32,7 @@ import java.io.Serializable; import java.util.List; -/** Helper class of {@link StoreWriteOperator} for different types of table store sinks. */ +/** Helper class of {@link StoreWriteOperator} for different types of paimon sinks. */ interface StoreSinkWrite { SinkRecord write(InternalRow rowData) throws Exception; diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/connector/sink/TableStoreSink.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/connector/sink/TableStoreSink.java index f88f947458a6..9dff38a6fa1e 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/connector/sink/TableStoreSink.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/connector/sink/TableStoreSink.java @@ -116,7 +116,7 @@ public ChangelogMode getChangelogMode(ChangelogMode requestedMode) { public SinkRuntimeProvider getSinkRuntimeProvider(Context context) { if (overwrite && !context.isBounded()) { throw new UnsupportedOperationException( - "Table store doesn't support streaming INSERT OVERWRITE."); + "Paimon doesn't support streaming INSERT OVERWRITE."); } LogSinkProvider logSinkProvider = null; diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/connector/source/FlinkSource.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/connector/source/FlinkSource.java index 493ef56dc065..871529869f98 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/connector/source/FlinkSource.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/connector/source/FlinkSource.java @@ -30,7 +30,7 @@ import javax.annotation.Nullable; -/** A Flink {@link Source} for table store. */ +/** A Flink {@link Source} for paimon. */ public abstract class FlinkSource implements Source { diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/connector/source/FlinkTableSource.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/connector/source/FlinkTableSource.java index cf04c07ca75c..2bd77a21840c 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/connector/source/FlinkTableSource.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/connector/source/FlinkTableSource.java @@ -36,7 +36,7 @@ import java.util.ArrayList; import java.util.List; -/** A Flink {@link ScanTableSource} for table store. */ +/** A Flink {@link ScanTableSource} for paimon. */ public abstract class FlinkTableSource implements ScanTableSource, SupportsFilterPushDown, diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/connector/CompositePkAndMultiPartitionedTableITCase.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/connector/CompositePkAndMultiPartitionedTableITCase.java index 86ceaf33f97e..7f3a1d95a9f6 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/connector/CompositePkAndMultiPartitionedTableITCase.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/connector/CompositePkAndMultiPartitionedTableITCase.java @@ -41,7 +41,7 @@ import static org.apache.paimon.connector.util.ReadWriteTableTestUtil.insertOverwritePartition; import static org.apache.paimon.connector.util.ReadWriteTableTestUtil.testBatchRead; -/** Table store IT case when the table has composite primary keys and multiple partition fields. */ +/** Paimon IT case when the table has composite primary keys and multiple partition fields. */ public class CompositePkAndMultiPartitionedTableITCase extends AbstractTestBase { @BeforeEach diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/connector/ComputedColumnAndWatermarkTableITCase.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/connector/ComputedColumnAndWatermarkTableITCase.java index 07a49645d9d1..94eb61dc56e6 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/connector/ComputedColumnAndWatermarkTableITCase.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/connector/ComputedColumnAndWatermarkTableITCase.java @@ -44,7 +44,7 @@ import static org.apache.paimon.connector.util.ReadWriteTableTestUtil.testStreamingRead; import static org.apache.paimon.connector.util.ReadWriteTableTestUtil.testStreamingReadWithReadFirst; -/** Table store IT case when the table has computed column and watermark spec. */ +/** Paimon IT case when the table has computed column and watermark spec. */ public class ComputedColumnAndWatermarkTableITCase extends KafkaTableTestBase { @BeforeEach diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/connector/CreateTableITCase.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/connector/CreateTableITCase.java index 40e1fbbc9d43..fca851ef1d55 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/connector/CreateTableITCase.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/connector/CreateTableITCase.java @@ -93,7 +93,7 @@ protected void testCore() { // check catalog assertThat(((TableEnvironmentImpl) tEnv).getCatalogManager().getTable(tableIdentifier)) .isPresent(); - // check table store + // check paimon assertThat(Paths.get(rootPath, relativeTablePath(tableIdentifier)).toFile()).exists(); } else { // check inconsistency diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/connector/DropTableITCase.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/connector/DropTableITCase.java index a6086632ee5e..29451b4c60fd 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/connector/DropTableITCase.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/connector/DropTableITCase.java @@ -95,7 +95,7 @@ protected void testCore() { // check catalog assertThat(((TableEnvironmentImpl) tEnv).getCatalogManager().getTable(tableIdentifier)) .isNotPresent(); - // check table store + // check paimon assertThat(Paths.get(rootPath, relativeTablePath(tableIdentifier)).toFile()) .doesNotExist(); } else { diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/connector/FileSystemCatalogITCase.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/connector/FileSystemCatalogITCase.java index 795848ef9621..ab154f6f6de4 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/connector/FileSystemCatalogITCase.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/connector/FileSystemCatalogITCase.java @@ -51,8 +51,7 @@ public class FileSystemCatalogITCase extends KafkaTableTestBase { public void before() throws IOException { path = getTempDirPath(); tEnv.executeSql( - String.format( - "CREATE CATALOG fs WITH ('type'='paimon', 'warehouse'='%s')", path)); + String.format("CREATE CATALOG fs WITH ('type'='paimon', 'warehouse'='%s')", path)); tEnv.useCatalog("fs"); env.setParallelism(1); } diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/connector/FlinkCatalogTest.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/connector/FlinkCatalogTest.java index 0a6aa19b810f..c7c4cbc4fdb6 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/connector/FlinkCatalogTest.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/connector/FlinkCatalogTest.java @@ -35,9 +35,9 @@ import org.apache.flink.table.catalog.exceptions.DatabaseNotExistException; import org.apache.flink.table.catalog.exceptions.TableAlreadyExistException; import org.apache.flink.table.catalog.exceptions.TableNotExistException; -import org.apache.flink.table.store.catalog.CatalogContext; import org.apache.paimon.catalog.AbstractCatalog; +import org.apache.paimon.catalog.CatalogContext; import org.apache.paimon.fs.Path; import org.apache.paimon.options.Options; import org.junit.jupiter.api.BeforeEach; @@ -212,7 +212,7 @@ public void testCreateFlinkTable() { assertThatThrownBy(() -> catalog.createTable(this.path1, newTable, false)) .isInstanceOf(CatalogException.class) .hasMessageContaining( - "Table Store Catalog only supports table store tables," + "Table Store Catalog only supports paimon tables," + " not 'filesystem' connector. You can create TEMPORARY table instead."); } diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/connector/LogSystemITCase.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/connector/LogSystemITCase.java index e00b709e5695..bc9b196fb277 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/connector/LogSystemITCase.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/connector/LogSystemITCase.java @@ -38,8 +38,7 @@ public class LogSystemITCase extends KafkaTableTestBase { public void before() throws IOException { tEnv.executeSql( String.format( - "CREATE CATALOG TABLE_STORE WITH (" - + "'type'='paimon', 'warehouse'='%s')", + "CREATE CATALOG TABLE_STORE WITH (" + "'type'='paimon', 'warehouse'='%s')", getTempDirPath())); tEnv.useCatalog("TABLE_STORE"); } diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/connector/ReadWriteTableITCase.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/connector/ReadWriteTableITCase.java index 40eefc034976..cca08b28b8a9 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/connector/ReadWriteTableITCase.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/connector/ReadWriteTableITCase.java @@ -1078,7 +1078,7 @@ public void testStreamingInsertOverwrite() { "INSERT OVERWRITE `%s` VALUES('US Dollar', 102, '2022-06-20')", table))) .isInstanceOf(UnsupportedOperationException.class) - .hasMessage("Table store doesn't support streaming INSERT OVERWRITE."); + .hasMessage("Paimon doesn't support streaming INSERT OVERWRITE."); } // ---------------------------------------------------------------------------------------------------------------- diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/connector/StreamingWarehouseITCase.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/connector/StreamingWarehouseITCase.java index 0acfc9b695dc..666c90d1b7c2 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/connector/StreamingWarehouseITCase.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/connector/StreamingWarehouseITCase.java @@ -34,7 +34,7 @@ import static org.apache.paimon.connector.util.ReadWriteTableTestUtil.sEnv; import static org.assertj.core.api.Assertions.assertThat; -/** Table store IT case to test concurrent batch overwrite and streaming insert into. */ +/** Paimon IT case to test concurrent batch overwrite and streaming insert into. */ public class StreamingWarehouseITCase extends KafkaTableTestBase { @Test diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/connector/TableStoreTestBase.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/connector/TableStoreTestBase.java index bd2185b4eb02..86d5215e2618 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/connector/TableStoreTestBase.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/connector/TableStoreTestBase.java @@ -49,7 +49,7 @@ import java.util.stream.Collectors; import java.util.stream.IntStream; -/** End-to-end test base for table store. */ +/** End-to-end test base for paimon. */ public abstract class TableStoreTestBase extends AbstractTestBase { public static final String CURRENT_CATALOG = "catalog"; diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/connector/action/ActionITCaseBase.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/connector/action/ActionITCaseBase.java index a1cd5bce9943..9fef09af206a 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/connector/action/ActionITCaseBase.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/connector/action/ActionITCaseBase.java @@ -18,9 +18,8 @@ package org.apache.paimon.connector.action; -import org.apache.flink.table.store.catalog.CatalogContext; - import org.apache.paimon.catalog.Catalog; +import org.apache.paimon.catalog.CatalogContext; import org.apache.paimon.catalog.CatalogFactory; import org.apache.paimon.catalog.Identifier; import org.apache.paimon.connector.util.AbstractTestBase; diff --git a/paimon-format/pom.xml b/paimon-format/pom.xml index eb8f140b2ad6..8c8a9507ef5e 100644 --- a/paimon-format/pom.xml +++ b/paimon-format/pom.xml @@ -47,6 +47,12 @@ under the License. provided + + org.xerial.snappy + snappy-java + ${snappy.version} + + @@ -138,6 +144,12 @@ under the License. org.apache.parquet parquet-hadoop ${parquet.version} + + + org.xerial.snappy + snappy-java + + @@ -286,49 +298,49 @@ under the License. org.apache.orc - org.apache.flink.table.store.shaded.org.apache.orc + org.apache.paimon.shaded.org.apache.orc org.apache.hive - org.apache.flink.table.store.shaded.org.apache.hive + org.apache.paimon.shaded.org.apache.hive org.apache.hadoop.hive - org.apache.flink.table.store.shaded.org.apache.hadoop.hive + org.apache.paimon.shaded.org.apache.hadoop.hive io.airlift - org.apache.flink.table.store.shaded.io.airlift + org.apache.paimon.shaded.io.airlift com.google.protobuf - org.apache.flink.table.store.shaded.com.google.protobuf + org.apache.paimon.shaded.com.google.protobuf org.apache.avro - org.apache.flink.table.store.shaded.org.apache.avro + org.apache.paimon.shaded.org.apache.avro com.fasterxml.jackson - org.apache.flink.table.store.shaded.com.fasterxml.jackson + org.apache.paimon.shaded.com.fasterxml.jackson org.apache.parquet - org.apache.flink.table.store.shaded.org.apache.parquet + org.apache.paimon.shaded.org.apache.parquet shaded.parquet - org.apache.flink.table.store.shaded.parquet + org.apache.paimon.shaded.parquet org.apache.commons - org.apache.flink.table.store.shaded.org.apache.commons + org.apache.paimon.shaded.org.apache.commons diff --git a/paimon-format/src/main/java/org/apache/paimon/format/parquet/ParquetInputFile.java b/paimon-format/src/main/java/org/apache/paimon/format/parquet/ParquetInputFile.java index c2c6d014f5f4..812d9ec5deec 100644 --- a/paimon-format/src/main/java/org/apache/paimon/format/parquet/ParquetInputFile.java +++ b/paimon-format/src/main/java/org/apache/paimon/format/parquet/ParquetInputFile.java @@ -26,7 +26,7 @@ import java.io.IOException; -/** A {@link InputFile} for table store. */ +/** A {@link InputFile} for paimon. */ public class ParquetInputFile implements InputFile { private final FileIO fileIO; diff --git a/paimon-format/src/main/java/org/apache/paimon/format/parquet/ParquetInputStream.java b/paimon-format/src/main/java/org/apache/paimon/format/parquet/ParquetInputStream.java index 0728aa666ea0..d9331383c564 100644 --- a/paimon-format/src/main/java/org/apache/paimon/format/parquet/ParquetInputStream.java +++ b/paimon-format/src/main/java/org/apache/paimon/format/parquet/ParquetInputStream.java @@ -23,7 +23,7 @@ import java.io.IOException; -/** A {@link SeekableInputStream} for table store. */ +/** A {@link SeekableInputStream} for paimon. */ public class ParquetInputStream extends DelegatingSeekableInputStream { private final SeekableInputStream in; diff --git a/paimon-hive/paimon-hive-catalog/pom.xml b/paimon-hive/paimon-hive-catalog/pom.xml index 8413984d8e36..d6426e17bf4d 100644 --- a/paimon-hive/paimon-hive-catalog/pom.xml +++ b/paimon-hive/paimon-hive-catalog/pom.xml @@ -592,35 +592,35 @@ under the License. com.facebook.fb303 - org.apache.flink.table.store.shaded.com.facebook.fb303 + org.apache.paimon.shaded.com.facebook.fb303 com.google.common - org.apache.flink.table.store.shaded.com.google.common + org.apache.paimon.shaded.com.google.common org.apache.hadoop.fs.ProxyLocalFileSystem - org.apache.flink.table.store.shaded.org.apache.hadoop.fs.ProxyLocalFileSystem + org.apache.paimon.shaded.org.apache.hadoop.fs.ProxyLocalFileSystem org.apache.hadoop.fs.ProxyFileSystem - org.apache.flink.table.store.shaded.org.apache.hadoop.fs.ProxyFileSystem + org.apache.paimon.shaded.org.apache.hadoop.fs.ProxyFileSystem org.apache.hadoop.fs.DefaultFileAccess - org.apache.flink.table.store.shaded.org.apache.hadoop.fs.DefaultFileAccess + org.apache.paimon.shaded.org.apache.hadoop.fs.DefaultFileAccess org.apache.hadoop.mapred.WebHCatJTShim23 - org.apache.flink.table.store.shaded.org.apache.hadoop.mapred.WebHCatJTShim23 + org.apache.paimon.shaded.org.apache.hadoop.mapred.WebHCatJTShim23 org.apache.hadoop.security.token.delegation.HiveDelegationTokenSupport - org.apache.flink.table.store.shaded.org.apache.hadoop.security.token.delegation.HiveDelegationTokenSupport + org.apache.paimon.shaded.org.apache.hadoop.security.token.delegation.HiveDelegationTokenSupport org.apache.thrift - org.apache.flink.table.store.shaded.org.apache.thrift + org.apache.paimon.shaded.org.apache.thrift diff --git a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java index 7d719ccbaefe..fa35cff2b439 100644 --- a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java +++ b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java @@ -184,8 +184,7 @@ public TableSchema getDataTableSchema(Identifier identifier) throws TableNotExis Path tableLocation = getDataTableLocation(identifier); return new SchemaManager(fileIO, tableLocation) .latest() - .orElseThrow( - () -> new RuntimeException("There is no table stored in " + tableLocation)); + .orElseThrow(() -> new RuntimeException("There is no paimond in " + tableLocation)); } @Override @@ -431,9 +430,7 @@ private boolean tableStoreTableExists(Identifier identifier, boolean throwExcept return false; } catch (TException e) { throw new RuntimeException( - "Cannot determine if table " - + identifier.getFullName() - + " is a table store table.", + "Cannot determine if table " + identifier.getFullName() + " is a paimon table.", e); } @@ -443,7 +440,7 @@ private boolean tableStoreTableExists(Identifier identifier, boolean throwExcept throw new IllegalArgumentException( "Table " + identifier.getFullName() - + " is not a table store table. It's input format is " + + " is not a paimon table. It's input format is " + table.getSd().getInputFormat() + " and its output format is " + table.getSd().getOutputFormat()); diff --git a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalogFactory.java b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalogFactory.java index 1c97929f34da..120502e86540 100644 --- a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalogFactory.java +++ b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalogFactory.java @@ -18,13 +18,12 @@ package org.apache.paimon.hive; -import org.apache.flink.table.store.catalog.CatalogContext; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; import org.apache.hadoop.hive.metastore.IMetaStoreClient; import org.apache.paimon.catalog.Catalog; +import org.apache.paimon.catalog.CatalogContext; import org.apache.paimon.catalog.CatalogFactory; import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.Path; @@ -59,7 +58,7 @@ public Catalog create(FileIO fileIO, Path warehouse, CatalogContext context) { Preconditions.checkNotNull( context.options().get(CatalogOptions.URI), CatalogOptions.URI.key() - + " must be set for table store " + + " must be set for paimon " + IDENTIFIER + " catalog"); diff --git a/paimon-hive/paimon-hive-catalog/src/main/resources/META-INF/services/org.apache.paimon.file.org.apache.paimon.catalog.CatalogFactory b/paimon-hive/paimon-hive-catalog/src/main/resources/META-INF/services/org.apache.paimon.catalog.CatalogFactory similarity index 100% rename from paimon-hive/paimon-hive-catalog/src/main/resources/META-INF/services/org.apache.paimon.file.org.apache.paimon.catalog.CatalogFactory rename to paimon-hive/paimon-hive-catalog/src/main/resources/META-INF/services/org.apache.paimon.catalog.CatalogFactory diff --git a/paimon-hive/paimon-hive-catalog/src/test/java/org/apache/paimon/hive/HiveCatalogITCase.java b/paimon-hive/paimon-hive-catalog/src/test/java/org/apache/paimon/hive/HiveCatalogITCase.java index a333863cdb7c..5e9ce987d815 100644 --- a/paimon-hive/paimon-hive-catalog/src/test/java/org/apache/paimon/hive/HiveCatalogITCase.java +++ b/paimon-hive/paimon-hive-catalog/src/test/java/org/apache/paimon/hive/HiveCatalogITCase.java @@ -211,7 +211,7 @@ public void testTableOperations() throws Exception { Assert.fail("No exception is thrown"); } catch (Throwable t) { ExceptionUtils.assertThrowableWithMessage( - t, "Table test_db.hive_table is not a table store table"); + t, "Table test_db.hive_table is not a paimon table"); } // alter table @@ -236,7 +236,7 @@ public void testTableOperations() throws Exception { Assert.fail("No exception is thrown"); } catch (Throwable t) { ExceptionUtils.assertThrowableWithMessage( - t, "Table test_db.hive_table is not a table store table"); + t, "Table test_db.hive_table is not a paimon table"); } } @@ -281,7 +281,7 @@ public void testFlinkWriteAndHiveRead() throws Exception { Assert.fail("No exception is thrown"); } catch (Throwable t) { ExceptionUtils.assertThrowableWithMessage( - t, "Table test_db.hive_table is not a table store table"); + t, "Table test_db.hive_table is not a paimon table"); } } @@ -454,7 +454,7 @@ public void testRenameTable() throws Exception { // TODO: the hiverunner (4.0) has a bug ,it can not rename the table path correctly , // we should upgrade it to the 6.0 later ,and update the test case for query. assertThatThrownBy(() -> tEnv.executeSql("SELECT * FROM t3")) - .hasMessageContaining("SQL validation failed. There is no table stored in"); + .hasMessageContaining("SQL validation failed. There is no paimond in"); } @Test diff --git a/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/HiveSchema.java b/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/HiveSchema.java index ed9be151f1c9..06e3e77e9de0 100644 --- a/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/HiveSchema.java +++ b/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/HiveSchema.java @@ -18,8 +18,6 @@ package org.apache.paimon.hive; -import org.apache.flink.table.store.catalog.CatalogContext; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants; import org.apache.hadoop.hive.serde.serdeConstants; @@ -27,6 +25,7 @@ import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; import org.apache.paimon.CoreOptions; +import org.apache.paimon.catalog.CatalogContext; import org.apache.paimon.fs.Path; import org.apache.paimon.options.Options; import org.apache.paimon.schema.TableSchema; @@ -74,7 +73,7 @@ public static HiveSchema extract(@Nullable Configuration configuration, Properti throw new UnsupportedOperationException( "Location property is missing for table " + tableName - + ". Currently Flink table store only supports external table for Hive " + + ". Currently Flink paimon only supports external table for Hive " + "so location property must be set."); } Path path = new Path(location); @@ -152,9 +151,9 @@ private static void checkSchemaMatched( if (mismatched.size() > 0) { throw new IllegalArgumentException( - "Hive DDL and table store schema mismatched! " + "Hive DDL and paimon schema mismatched! " + "It is recommended not to write any column definition " - + "as Flink table store external table can read schema from the specified location.\n" + + "as Flink paimon external table can read schema from the specified location.\n" + "Mismatched fields are:\n" + String.join("--------------------\n", mismatched)); } diff --git a/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/TableStoreHiveMetaHook.java b/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/TableStoreHiveMetaHook.java index 48bd76874bde..c793e96885e5 100644 --- a/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/TableStoreHiveMetaHook.java +++ b/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/TableStoreHiveMetaHook.java @@ -26,7 +26,7 @@ import org.apache.paimon.utils.Preconditions; /** - * {@link HiveMetaHook} for table store. Currently this class is only used to set input and output + * {@link HiveMetaHook} for paimon. Currently this class is only used to set input and output * formats. */ public class TableStoreHiveMetaHook implements HiveMetaHook { diff --git a/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/TableStoreHiveStorageHandler.java b/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/TableStoreHiveStorageHandler.java index 2f95b90d6d87..1f66aa92aa9b 100644 --- a/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/TableStoreHiveStorageHandler.java +++ b/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/TableStoreHiveStorageHandler.java @@ -38,7 +38,7 @@ import java.util.Map; import java.util.Properties; -/** {@link HiveStorageHandler} for table store. This is the entrance class of Hive API. */ +/** {@link HiveStorageHandler} for paimon. This is the entrance class of Hive API. */ public class TableStoreHiveStorageHandler implements HiveStoragePredicateHandler, HiveStorageHandler { diff --git a/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/TableStoreJobConf.java b/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/TableStoreJobConf.java index 3131cf80edd1..6ddded952241 100644 --- a/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/TableStoreJobConf.java +++ b/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/TableStoreJobConf.java @@ -37,7 +37,7 @@ public class TableStoreJobConf { private static final String INTERNAL_LOCATION = "paimon.internal.location"; private static final String INTERNAL_CATALOG_CONFIG = "paimon.catalog.config"; - private static final String TABLE_STORE_PREFIX = "tablestore."; + private static final String TABLE_STORE_PREFIX = "paimon."; private final JobConf jobConf; @@ -59,7 +59,7 @@ public String getLocation() { return jobConf.get(INTERNAL_LOCATION); } - /** Extract table store catalog conf from Hive conf. */ + /** Extract paimon catalog conf from Hive conf. */ public static Options extractCatalogConfig(Configuration hiveConf) { Map configMap = new HashMap<>(); for (Map.Entry entry : hiveConf) { diff --git a/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/TableStoreSerDe.java b/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/TableStoreSerDe.java index eba57de5b5a6..c2818a7b231b 100644 --- a/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/TableStoreSerDe.java +++ b/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/TableStoreSerDe.java @@ -31,7 +31,7 @@ import java.util.Properties; /** - * {@link AbstractSerDe} for table store. It transforms map-reduce values to Hive objects. + * {@link AbstractSerDe} for paimon. It transforms map-reduce values to Hive objects. * *

    Currently this class only supports deserialization. */ diff --git a/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/mapred/TableStoreInputFormat.java b/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/mapred/TableStoreInputFormat.java index f769ed652df7..09258b3fb45e 100644 --- a/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/mapred/TableStoreInputFormat.java +++ b/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/mapred/TableStoreInputFormat.java @@ -18,8 +18,6 @@ package org.apache.paimon.hive.mapred; -import org.apache.flink.table.store.catalog.CatalogContext; - import org.apache.hadoop.hive.ql.io.sarg.ConvertAstToSearchArg; import org.apache.hadoop.hive.ql.io.sarg.SearchArgument; import org.apache.hadoop.hive.serde2.ColumnProjectionUtils; @@ -29,6 +27,7 @@ import org.apache.hadoop.mapred.RecordReader; import org.apache.hadoop.mapred.Reporter; import org.apache.paimon.CoreOptions; +import org.apache.paimon.catalog.CatalogContext; import org.apache.paimon.hive.RowDataContainer; import org.apache.paimon.hive.SearchArgumentToPredicateConverter; import org.apache.paimon.hive.TableStoreJobConf; @@ -45,7 +44,7 @@ import java.util.Optional; /** - * {@link InputFormat} for table store. It divides all files into {@link InputSplit}s (one split per + * {@link InputFormat} for paimon. It divides all files into {@link InputSplit}s (one split per * bucket) and creates {@link RecordReader} for each split. */ public class TableStoreInputFormat implements InputFormat { diff --git a/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/mapred/TableStoreInputSplit.java b/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/mapred/TableStoreInputSplit.java index 1a43d1b0d3dd..a7fe4e6efb5d 100644 --- a/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/mapred/TableStoreInputSplit.java +++ b/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/mapred/TableStoreInputSplit.java @@ -31,8 +31,7 @@ import java.util.Objects; /** - * {@link FileSplit} for table store. It contains all files to read from a certain partition and - * bucket. + * {@link FileSplit} for paimon. It contains all files to read from a certain partition and bucket. */ public class TableStoreInputSplit extends FileSplit { diff --git a/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/mapred/TableStoreOutputFormat.java b/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/mapred/TableStoreOutputFormat.java index 0b318ab87da0..b1119993d06d 100644 --- a/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/mapred/TableStoreOutputFormat.java +++ b/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/mapred/TableStoreOutputFormat.java @@ -35,7 +35,7 @@ public RecordWriter getRecordWriter( FileSystem fileSystem, JobConf jobConf, String s, Progressable progressable) throws IOException { throw new UnsupportedOperationException( - "Flink table store currently can only be used as an input format for Hive."); + "Flink paimon currently can only be used as an input format for Hive."); } @Override diff --git a/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/mapred/TableStoreRecordReader.java b/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/mapred/TableStoreRecordReader.java index e208f8b17e36..5485b620af45 100644 --- a/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/mapred/TableStoreRecordReader.java +++ b/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/mapred/TableStoreRecordReader.java @@ -32,7 +32,7 @@ import java.util.List; /** - * Base {@link RecordReader} for table store. Reads {@link KeyValue}s from data files and picks out + * Base {@link RecordReader} for paimon. Reads {@link KeyValue}s from data files and picks out * {@link InternalRow} for Hive to consume. * *

    NOTE: To support projection push down, when {@code selectedColumns} does not match {@code diff --git a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveTableSchemaTest.java b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveTableSchemaTest.java index f61b0ce3dbba..0f2f9d7757a1 100644 --- a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveTableSchemaTest.java +++ b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveTableSchemaTest.java @@ -113,9 +113,9 @@ public void testMismatchedColumnNameAndType() throws Exception { String expected = String.join( "\n", - "Hive DDL and table store schema mismatched! " + "Hive DDL and paimon schema mismatched! " + "It is recommended not to write any column definition " - + "as Flink table store external table can read schema from the specified location.", + + "as Flink paimon external table can read schema from the specified location.", "Mismatched fields are:", "Field #1", "Hive DDL : mismatched string", @@ -142,9 +142,9 @@ public void testTooFewColumns() throws Exception { String expected = String.join( "\n", - "Hive DDL and table store schema mismatched! " + "Hive DDL and paimon schema mismatched! " + "It is recommended not to write any column definition " - + "as Flink table store external table can read schema from the specified location.", + + "as Flink paimon external table can read schema from the specified location.", "Mismatched fields are:", "Field #1", "Hive DDL : null", @@ -180,9 +180,9 @@ public void testTooManyColumns() throws Exception { String expected = String.join( "\n", - "Hive DDL and table store schema mismatched! " + "Hive DDL and paimon schema mismatched! " + "It is recommended not to write any column definition " - + "as Flink table store external table can read schema from the specified location.", + + "as Flink paimon external table can read schema from the specified location.", "Mismatched fields are:", "Field #3", "Hive DDL : d int", diff --git a/paimon-spark/paimon-spark-2/src/main/java/org/apache/paimon/spark/SparkDataSourceReader.java b/paimon-spark/paimon-spark-2/src/main/java/org/apache/paimon/spark/SparkDataSourceReader.java index 2203b4d85a66..bad4b6a7fc34 100644 --- a/paimon-spark/paimon-spark-2/src/main/java/org/apache/paimon/spark/SparkDataSourceReader.java +++ b/paimon-spark/paimon-spark-2/src/main/java/org/apache/paimon/spark/SparkDataSourceReader.java @@ -39,7 +39,7 @@ import java.util.OptionalLong; import java.util.stream.Collectors; -/** A Spark {@link DataSourceReader} for table store. */ +/** A Spark {@link DataSourceReader} for paimon. */ public class SparkDataSourceReader implements SupportsPushDownFilters, SupportsPushDownRequiredColumns, diff --git a/paimon-spark/paimon-spark-2/src/main/java/org/apache/paimon/spark/SparkInputPartition.java b/paimon-spark/paimon-spark-2/src/main/java/org/apache/paimon/spark/SparkInputPartition.java index e922dd9a2c74..12501f2ded3b 100644 --- a/paimon-spark/paimon-spark-2/src/main/java/org/apache/paimon/spark/SparkInputPartition.java +++ b/paimon-spark/paimon-spark-2/src/main/java/org/apache/paimon/spark/SparkInputPartition.java @@ -29,7 +29,7 @@ import java.io.IOException; import java.io.UncheckedIOException; -/** A Spark {@link InputPartition} for table store. */ +/** A Spark {@link InputPartition} for paimon. */ public class SparkInputPartition implements InputPartition { diff --git a/paimon-spark/paimon-spark-2/src/main/java/org/apache/paimon/spark/SparkSource.java b/paimon-spark/paimon-spark-2/src/main/java/org/apache/paimon/spark/SparkSource.java index c60d687976c0..cc743b483484 100644 --- a/paimon-spark/paimon-spark-2/src/main/java/org/apache/paimon/spark/SparkSource.java +++ b/paimon-spark/paimon-spark-2/src/main/java/org/apache/paimon/spark/SparkSource.java @@ -18,8 +18,7 @@ package org.apache.paimon.spark; -import org.apache.flink.table.store.catalog.CatalogContext; - +import org.apache.paimon.catalog.CatalogContext; import org.apache.paimon.options.Options; import org.apache.paimon.table.FileStoreTableFactory; import org.apache.spark.sql.SparkSession; @@ -30,11 +29,11 @@ import org.apache.spark.sql.sources.v2.reader.DataSourceReader; import org.apache.spark.sql.types.StructType; -/** The Spark source for table store. */ +/** The Spark source for paimon. */ public class SparkSource implements DataSourceRegister, ReadSupport, SessionConfigSupport { /** Not use 'paimon' here, the '-' is not allowed in SQL. */ - private static final String SHORT_NAME = "tablestore"; + private static final String SHORT_NAME = "paimon"; @Override public String shortName() { diff --git a/paimon-spark/paimon-spark-2/src/test/java/org/apache/paimon/spark/SparkReadITCase.java b/paimon-spark/paimon-spark-2/src/test/java/org/apache/paimon/spark/SparkReadITCase.java index 51f7e9fbc903..e222dc545f12 100644 --- a/paimon-spark/paimon-spark-2/src/test/java/org/apache/paimon/spark/SparkReadITCase.java +++ b/paimon-spark/paimon-spark-2/src/test/java/org/apache/paimon/spark/SparkReadITCase.java @@ -103,14 +103,13 @@ public static void stopSpark() throws IOException { @Test public void testNormal() { - innerTestNormal( - spark.read().format("tablestore").option("path", tablePath1.toString()).load()); + innerTestNormal(spark.read().format("paimon").option("path", tablePath1.toString()).load()); } @Test public void testFilterPushDown() { innerTestFilterPushDown( - spark.read().format("tablestore").option("path", tablePath2.toString()).load()); + spark.read().format("paimon").option("path", tablePath2.toString()).load()); } private void innerTestNormal(Dataset dataset) { diff --git a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkCatalog.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkCatalog.java index 1741f7eea31c..3851e1b9ce76 100644 --- a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkCatalog.java +++ b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkCatalog.java @@ -18,9 +18,8 @@ package org.apache.paimon.spark; -import org.apache.flink.table.store.catalog.CatalogContext; - import org.apache.paimon.catalog.Catalog; +import org.apache.paimon.catalog.CatalogContext; import org.apache.paimon.catalog.CatalogFactory; import org.apache.paimon.operation.Lock; import org.apache.paimon.options.Options; @@ -63,7 +62,7 @@ import static org.apache.paimon.spark.SparkTypeUtils.toFlinkType; -/** Spark {@link TableCatalog} for table store. */ +/** Spark {@link TableCatalog} for paimon. */ public class SparkCatalog implements TableCatalog, SupportsNamespaces { private static final String PRIMARY_KEY_IDENTIFIER = "primary-key"; diff --git a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkInputPartition.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkInputPartition.java index 407f312ce030..d5bbf9863cd0 100644 --- a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkInputPartition.java +++ b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkInputPartition.java @@ -21,7 +21,7 @@ import org.apache.paimon.table.source.Split; import org.apache.spark.sql.connector.read.InputPartition; -/** A Spark {@link InputPartition} for table store. */ +/** A Spark {@link InputPartition} for paimon. */ public class SparkInputPartition implements InputPartition { private static final long serialVersionUID = 1L; diff --git a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkReaderFactory.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkReaderFactory.java index d8ac74ec7259..acd49feeff39 100644 --- a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkReaderFactory.java +++ b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkReaderFactory.java @@ -28,7 +28,7 @@ import java.io.IOException; import java.io.UncheckedIOException; -/** A Spark {@link PartitionReaderFactory} for table store. */ +/** A Spark {@link PartitionReaderFactory} for paimon. */ public class SparkReaderFactory implements PartitionReaderFactory { private static final long serialVersionUID = 1L; diff --git a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkScan.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkScan.java index 9b3af437f24f..a84c2b80bded 100644 --- a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkScan.java +++ b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkScan.java @@ -32,7 +32,7 @@ import java.util.OptionalLong; /** - * A Spark {@link Scan} for table store. + * A Spark {@link Scan} for paimon. * *

    TODO Introduce a SparkRFScan to implement SupportsRuntimeFiltering. */ @@ -49,7 +49,7 @@ public SparkScan(ReadBuilder readBuilder) { @Override public String description() { // TODO add filters - return String.format("tablestore(%s)", readBuilder.tableName()); + return String.format("paimon(%s)", readBuilder.tableName()); } @Override diff --git a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkScanBuilder.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkScanBuilder.java index 65ed57124c2c..735b7a94458a 100644 --- a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkScanBuilder.java +++ b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkScanBuilder.java @@ -30,7 +30,7 @@ import java.util.ArrayList; import java.util.List; -/** A Spark {@link ScanBuilder} for table store. */ +/** A Spark {@link ScanBuilder} for paimon. */ public class SparkScanBuilder implements ScanBuilder, SupportsPushDownFilters, SupportsPushDownRequiredColumns { diff --git a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkSource.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkSource.java index 433df840bc7b..2b6cc3368aa9 100644 --- a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkSource.java +++ b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkSource.java @@ -18,8 +18,7 @@ package org.apache.paimon.spark; -import org.apache.flink.table.store.catalog.CatalogContext; - +import org.apache.paimon.catalog.CatalogContext; import org.apache.paimon.operation.Lock; import org.apache.paimon.options.Options; import org.apache.paimon.table.FileStoreTableFactory; @@ -33,11 +32,11 @@ import java.util.Map; -/** The spark source for table store. */ +/** The spark source for paimon. */ public class SparkSource implements DataSourceRegister, SessionConfigSupport { /** Not use 'paimon' here, the '-' is not allowed in SQL. */ - private static final String SHORT_NAME = "tablestore"; + private static final String SHORT_NAME = "paimon"; @Override public String shortName() { diff --git a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkTable.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkTable.java index b0aef728f096..6630113a220d 100644 --- a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkTable.java +++ b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkTable.java @@ -45,7 +45,7 @@ import java.util.Map; import java.util.Set; -/** A spark {@link org.apache.spark.sql.connector.catalog.Table} for table store. */ +/** A spark {@link org.apache.spark.sql.connector.catalog.Table} for paimon. */ public class SparkTable implements org.apache.spark.sql.connector.catalog.Table, SupportsRead, diff --git a/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/SparkReadITCase.java b/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/SparkReadITCase.java index d0eae1fa0d5d..81200543ae64 100644 --- a/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/SparkReadITCase.java +++ b/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/SparkReadITCase.java @@ -52,42 +52,41 @@ public class SparkReadITCase extends SparkReadTestBase { @Test public void testNormal() { - innerTestSimpleType(spark.table("tablestore.default.t1")); + innerTestSimpleType(spark.table("paimon.default.t1")); - innerTestNestedType(spark.table("tablestore.default.t2")); + innerTestNestedType(spark.table("paimon.default.t2")); } @Test public void testFilterPushDown() { - innerTestSimpleTypeFilterPushDown(spark.table("tablestore.default.t1")); + innerTestSimpleTypeFilterPushDown(spark.table("paimon.default.t1")); - innerTestNestedTypeFilterPushDown(spark.table("tablestore.default.t2")); + innerTestNestedTypeFilterPushDown(spark.table("paimon.default.t2")); } @Test public void testCatalogNormal() { - innerTestSimpleType(spark.table("tablestore.default.t1")); - innerTestNestedType(spark.table("tablestore.default.t2")); + innerTestSimpleType(spark.table("paimon.default.t1")); + innerTestNestedType(spark.table("paimon.default.t2")); } @Test public void testSnapshotsTable() { List rows = - spark.table("tablestore.default.`t1$snapshots`") + spark.table("paimon.default.`t1$snapshots`") .select("snapshot_id", "schema_id", "commit_user", "commit_kind") .collectAsList(); assertThat(rows.toString()).isEqualTo("[[1,0,user,APPEND]]"); - spark.sql("USE tablestore"); + spark.sql("USE paimon"); spark.sql( "CREATE TABLE default.schemasTable (\n" + "a BIGINT,\n" - + "b STRING) USING tablestore\n" + + "b STRING) USING paimon\n" + "COMMENT 'table comment'\n" + "TBLPROPERTIES ('primary-key' = 'a')"); spark.sql("ALTER TABLE default.schemasTable ADD COLUMN c STRING"); - List schemas = - spark.table("tablestore.default.`schemasTable$schemas`").collectAsList(); + List schemas = spark.table("paimon.default.`schemasTable$schemas`").collectAsList(); List fieldsList = schemas.stream().map(row -> row.get(1)).collect(Collectors.toList()); assertThat(fieldsList.stream().map(Object::toString).collect(Collectors.toList())) .containsExactlyInAnyOrder( @@ -101,7 +100,7 @@ public void testSnapshotsTable() { @Test public void testSnapshotsTableWithRecordCount() { List rows = - spark.table("tablestore.default.`t1$snapshots`") + spark.table("paimon.default.`t1$snapshots`") .select( "snapshot_id", "total_record_count", @@ -113,27 +112,27 @@ public void testSnapshotsTableWithRecordCount() { @Test public void testCatalogFilterPushDown() { - innerTestSimpleTypeFilterPushDown(spark.table("tablestore.default.t1")); + innerTestSimpleTypeFilterPushDown(spark.table("paimon.default.t1")); - innerTestNestedTypeFilterPushDown(spark.table("tablestore.default.t2")); + innerTestNestedTypeFilterPushDown(spark.table("paimon.default.t2")); } @Test public void testDefaultNamespace() { - spark.sql("USE tablestore"); + spark.sql("USE paimon"); assertThat(spark.sql("SHOW CURRENT NAMESPACE").collectAsList().toString()) - .isEqualTo("[[tablestore,default]]"); + .isEqualTo("[[paimon,default]]"); } @Test public void testCreateTable() { spark.sql( - "CREATE TABLE tablestore.default.testCreateTable(\n" + "CREATE TABLE paimon.default.testCreateTable(\n" + "a BIGINT,\n" + "b VARCHAR(10),\n" + "c CHAR(10))"); assertThat( - spark.sql("SELECT fields FROM tablestore.default.`testCreateTable$schemas`") + spark.sql("SELECT fields FROM paimon.default.`testCreateTable$schemas`") .collectAsList() .toString()) .isEqualTo( @@ -162,7 +161,7 @@ public void testCreateTableAs() { "CREATE TABLE default.partitionedTable (\n" + "a BIGINT,\n" + "b STRING," - + "c STRING) USING tablestore\n" + + "c STRING) USING paimon\n" + "COMMENT 'table comment'\n" + "PARTITIONED BY (a,b)"); spark.sql("INSERT INTO default.partitionedTable VALUES(1,'aaa','bbb')"); @@ -279,11 +278,11 @@ public void testCreateTableAs() { @Test public void testCreateTableWithNullablePk() { - spark.sql("USE tablestore"); + spark.sql("USE paimon"); spark.sql( "CREATE TABLE default.PkTable (\n" + "a BIGINT,\n" - + "b STRING) USING tablestore\n" + + "b STRING) USING paimon\n" + "COMMENT 'table comment'\n" + "TBLPROPERTIES ('primary-key' = 'a')"); Path tablePath = new Path(warehousePath, "default.db/PkTable"); @@ -293,11 +292,11 @@ public void testCreateTableWithNullablePk() { @Test public void testDescribeTable() { - spark.sql("USE tablestore"); + spark.sql("USE paimon"); spark.sql( "CREATE TABLE default.PartitionedTable (\n" + "a BIGINT,\n" - + "b STRING) USING tablestore\n" + + "b STRING) USING paimon\n" + "COMMENT 'table comment'\n" + "PARTITIONED BY (a)\n" + "TBLPROPERTIES ('foo' = 'bar')"); @@ -307,7 +306,7 @@ public void testDescribeTable() { @Test public void testShowTableProperties() { - spark.sql("USE tablestore"); + spark.sql("USE paimon"); spark.sql( "CREATE TABLE default.tbl (\n" + "a INT\n" @@ -325,14 +324,14 @@ public void testShowTableProperties() { @Test public void testCreateTableWithInvalidPk() { - spark.sql("USE tablestore"); + spark.sql("USE paimon"); assertThatThrownBy( () -> spark.sql( "CREATE TABLE default.PartitionedPkTable (\n" + "a BIGINT,\n" + "b STRING,\n" - + "c DOUBLE) USING tablestore\n" + + "c DOUBLE) USING paimon\n" + "COMMENT 'table comment'\n" + "PARTITIONED BY (b)" + "TBLPROPERTIES ('primary-key' = 'a')")) @@ -343,14 +342,14 @@ public void testCreateTableWithInvalidPk() { @Test public void testCreateTableWithNonexistentPk() { - spark.sql("USE tablestore"); + spark.sql("USE paimon"); assertThatThrownBy( () -> spark.sql( "CREATE TABLE default.PartitionedPkTable (\n" + "a BIGINT,\n" + "b STRING,\n" - + "c DOUBLE) USING tablestore\n" + + "c DOUBLE) USING paimon\n" + "COMMENT 'table comment'\n" + "PARTITIONED BY (b)" + "TBLPROPERTIES ('primary-key' = 'd')")) @@ -361,14 +360,14 @@ public void testCreateTableWithNonexistentPk() { @Test public void testCreateTableWithNonexistentPartition() { - spark.sql("USE tablestore"); + spark.sql("USE paimon"); assertThatThrownBy( () -> spark.sql( "CREATE TABLE default.PartitionedPkTable (\n" + "a BIGINT,\n" + "b STRING,\n" - + "c DOUBLE) USING tablestore\n" + + "c DOUBLE) USING paimon\n" + "COMMENT 'table comment'\n" + "PARTITIONED BY (d)" + "TBLPROPERTIES ('primary-key' = 'a')")) @@ -388,7 +387,7 @@ public void testCreateAndDropTable() throws Exception { private void innerTest( String tableName, boolean hasPk, boolean partitioned, boolean appendOnly) { - spark.sql("USE tablestore"); + spark.sql("USE paimon"); String ddlTemplate = "CREATE TABLE default.%s (\n" + "order_id BIGINT NOT NULL comment 'order_id',\n" @@ -396,7 +395,7 @@ private void innerTest( + "coupon_info ARRAY NOT NULL COMMENT 'coupon_info',\n" + "order_amount DOUBLE NOT NULL COMMENT 'order_amount',\n" + "dt STRING NOT NULL COMMENT 'dt',\n" - + "hh STRING NOT NULL COMMENT 'hh') USING tablestore\n" + + "hh STRING NOT NULL COMMENT 'hh') USING paimon\n" + "COMMENT 'table comment'\n" + "%s\n" + "TBLPROPERTIES (%s)"; @@ -503,7 +502,7 @@ private void innerTest( tableName, "(1L, 10L, array('loyalty_discount', 'shipping_discount'), 199.0d, '2022-07-20', '12')"); - Dataset dataset = spark.read().format("tablestore").load(tablePath.toString()); + Dataset dataset = spark.read().format("paimon").load(tablePath.toString()); assertThat(dataset.select("order_id", "buyer_id", "dt").collectAsList().toString()) .isEqualTo("[[1,10,2022-07-20]]"); assertThat(dataset.select("coupon_info").collectAsList().toString()) @@ -513,19 +512,19 @@ private void innerTest( assertThat( spark.sql( String.format( - "SHOW TABLES IN tablestore.default LIKE '%s'", + "SHOW TABLES IN paimon.default LIKE '%s'", tableName)) .select("namespace", "tableName") .collectAsList() .toString()) .isEqualTo(String.format("[[default,%s]]", tableName)); - spark.sql(String.format("DROP TABLE tablestore.default.%s", tableName)); + spark.sql(String.format("DROP TABLE paimon.default.%s", tableName)); assertThat( spark.sql( String.format( - "SHOW TABLES IN tablestore.default LIKE '%s'", + "SHOW TABLES IN paimon.default LIKE '%s'", tableName)) .select("namespace", "tableName") .collectAsList() @@ -538,7 +537,7 @@ private void innerTest( @Test public void testCreateAndDropNamespace() { // create namespace - spark.sql("USE tablestore"); + spark.sql("USE paimon"); spark.sql("CREATE NAMESPACE bar"); assertThatThrownBy(() -> spark.sql("CREATE NAMESPACE bar")) @@ -631,9 +630,9 @@ private void innerTestNestedTypeFilterPushDown(Dataset dataset) { @Test public void testCreateNestedField() { spark.sql( - "CREATE TABLE tablestore.default.nested_table ( a INT, b STRUCT, b2 BIGINT>)"); + "CREATE TABLE paimon.default.nested_table ( a INT, b STRUCT, b2 BIGINT>)"); assertThat( - spark.sql("SHOW CREATE TABLE tablestore.default.nested_table") + spark.sql("SHOW CREATE TABLE paimon.default.nested_table") .collectAsList() .toString()) .isEqualTo( diff --git a/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/SparkReadTestBase.java b/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/SparkReadTestBase.java index e2d12c420a10..eb76e8149958 100644 --- a/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/SparkReadTestBase.java +++ b/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/SparkReadTestBase.java @@ -61,9 +61,9 @@ public abstract class SparkReadTestBase { public static void startMetastoreAndSpark(@TempDir java.nio.file.Path tempDir) { warehousePath = new Path("file:" + tempDir.toString()); spark = SparkSession.builder().master("local[2]").getOrCreate(); - spark.conf().set("spark.sql.catalog.tablestore", SparkCatalog.class.getName()); - spark.conf().set("spark.sql.catalog.tablestore.warehouse", warehousePath.toString()); - spark.sql("USE tablestore"); + spark.conf().set("spark.sql.catalog.paimon", SparkCatalog.class.getName()); + spark.conf().set("spark.sql.catalog.paimon.warehouse", warehousePath.toString()); + spark.sql("USE paimon"); spark.sql("CREATE NAMESPACE default"); } @@ -93,7 +93,7 @@ public void beforeEach() throws Exception { // c row not null> not null, bigint> not null tablePath2 = new Path(warehousePath, "default.db/t2"); spark.sql( - "CREATE TABLE tablestore.default.t2 (" + "CREATE TABLE paimon.default.t2 (" + "a INT NOT NULL COMMENT 'comment about a', " + "b ARRAY NOT NULL, " + "c STRUCT NOT NULL> NOT NULL, " @@ -161,7 +161,7 @@ protected DataField getNestedField(DataField field, int index) { protected static void createTable(String tableName) { spark.sql( String.format( - "CREATE TABLE tablestore.default.%s (a INT NOT NULL, b BIGINT, c STRING) TBLPROPERTIES ('file.format'='avro')", + "CREATE TABLE paimon.default.%s (a INT NOT NULL, b BIGINT, c STRING) TBLPROPERTIES ('file.format'='avro')", tableName)); } @@ -182,7 +182,7 @@ private static void writeTable(String tableName, GenericRow... rows) throws Exce protected static void writeTable(String tableName, String... values) { spark.sql( String.format( - "INSERT INTO tablestore.default.%s VALUES %s", + "INSERT INTO paimon.default.%s VALUES %s", tableName, StringUtils.join(values, ","))); } } diff --git a/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/SparkS3ITCase.java b/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/SparkS3ITCase.java index b648b54dede4..83a7c9ddbd24 100644 --- a/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/SparkS3ITCase.java +++ b/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/SparkS3ITCase.java @@ -51,13 +51,13 @@ public static void startMetastoreAndSpark() { String path = MINIO_CONTAINER.getS3UriForDefaultBucket() + "/" + UUID.randomUUID(); Path warehousePath = new Path(path); spark = SparkSession.builder().master("local[2]").getOrCreate(); - spark.conf().set("spark.sql.catalog.tablestore", SparkCatalog.class.getName()); - spark.conf().set("spark.sql.catalog.tablestore.warehouse", warehousePath.toString()); + spark.conf().set("spark.sql.catalog.paimon", SparkCatalog.class.getName()); + spark.conf().set("spark.sql.catalog.paimon.warehouse", warehousePath.toString()); MINIO_CONTAINER .getS3ConfigOptions() - .forEach((k, v) -> spark.conf().set("spark.sql.catalog.tablestore." + k, v)); - spark.sql("CREATE DATABASE tablestore.db"); - spark.sql("USE tablestore.db"); + .forEach((k, v) -> spark.conf().set("spark.sql.catalog.paimon." + k, v)); + spark.sql("CREATE DATABASE paimon.db"); + spark.sql("USE paimon.db"); } @AfterAll diff --git a/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/SparkSchemaEvolutionITCase.java b/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/SparkSchemaEvolutionITCase.java index 983544847b73..e21df7adfe20 100644 --- a/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/SparkSchemaEvolutionITCase.java +++ b/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/SparkSchemaEvolutionITCase.java @@ -37,24 +37,21 @@ public class SparkSchemaEvolutionITCase extends SparkReadTestBase { @Test public void testSetAndRemoveOption() { - spark.sql("ALTER TABLE tablestore.default.t1 SET TBLPROPERTIES('xyc' 'unknown1')"); + spark.sql("ALTER TABLE paimon.default.t1 SET TBLPROPERTIES('xyc' 'unknown1')"); Map options = - rowsToMap( - spark.sql("SELECT * FROM tablestore.default.`t1$options`").collectAsList()); + rowsToMap(spark.sql("SELECT * FROM paimon.default.`t1$options`").collectAsList()); assertThat(options).containsEntry("xyc", "unknown1"); - spark.sql("ALTER TABLE tablestore.default.t1 UNSET TBLPROPERTIES('xyc')"); + spark.sql("ALTER TABLE paimon.default.t1 UNSET TBLPROPERTIES('xyc')"); - options = - rowsToMap( - spark.sql("SELECT * FROM tablestore.default.`t1$options`").collectAsList()); + options = rowsToMap(spark.sql("SELECT * FROM paimon.default.`t1$options`").collectAsList()); assertThat(options).doesNotContainKey("xyc"); assertThatThrownBy( () -> spark.sql( - "ALTER TABLE tablestore.default.t1 SET TBLPROPERTIES('primary-key' = 'a')")) + "ALTER TABLE paimon.default.t1 SET TBLPROPERTIES('primary-key' = 'a')")) .isInstanceOf(UnsupportedOperationException.class) .hasMessageContaining("Alter primary key is not supported"); } @@ -71,9 +68,9 @@ public void testAddColumn() { createTable("testAddColumn"); writeTable("testAddColumn", "(1, 2L, '1')", "(5, 6L, '3')"); - spark.sql("ALTER TABLE tablestore.default.testAddColumn ADD COLUMN d STRING"); + spark.sql("ALTER TABLE paimon.default.testAddColumn ADD COLUMN d STRING"); - Dataset table = spark.table("tablestore.default.testAddColumn"); + Dataset table = spark.table("paimon.default.testAddColumn"); List results = table.collectAsList(); assertThat(results.toString()).isEqualTo("[[1,2,1,null], [5,6,3,null]]"); @@ -89,8 +86,7 @@ public void testAddNotNullColumn() { createTable("testAddNotNullColumn"); List beforeAdd = - spark.sql("SHOW CREATE TABLE tablestore.default.testAddNotNullColumn") - .collectAsList(); + spark.sql("SHOW CREATE TABLE paimon.default.testAddNotNullColumn").collectAsList(); assertThat(beforeAdd.toString()) .isEqualTo( "[[CREATE TABLE testAddNotNullColumn (\n" @@ -103,7 +99,7 @@ public void testAddNotNullColumn() { assertThatThrownBy( () -> spark.sql( - "ALTER TABLE tablestore.default.testAddNotNullColumn ADD COLUMNS (d INT NOT NULL)")) + "ALTER TABLE paimon.default.testAddNotNullColumn ADD COLUMNS (d INT NOT NULL)")) .isInstanceOf(RuntimeException.class) .hasMessage( "java.lang.IllegalArgumentException: ADD COLUMN cannot specify NOT NULL."); @@ -125,8 +121,7 @@ public void testRenameTable() { assertThat(tables.stream().map(Row::toString)) .containsExactlyInAnyOrder("[default,t2,false]", "[default,t3,false]"); - List afterRename = - spark.sql("SHOW CREATE TABLE tablestore.default.t3").collectAsList(); + List afterRename = spark.sql("SHOW CREATE TABLE paimon.default.t3").collectAsList(); assertThat(afterRename.toString()) .isEqualTo( "[[CREATE TABLE t3 (\n" @@ -146,7 +141,7 @@ public void testRenameColumn() { writeTable("testRenameColumn", "(1, 2L, '1')", "(5, 6L, '3')"); List beforeRename = - spark.sql("SHOW CREATE TABLE tablestore.default.testRenameColumn").collectAsList(); + spark.sql("SHOW CREATE TABLE paimon.default.testRenameColumn").collectAsList(); assertThat(beforeRename.toString()) .isEqualTo( "[[CREATE TABLE testRenameColumn (\n" @@ -155,14 +150,14 @@ public void testRenameColumn() { + " `c` STRING)\n" + buildTableProperties("default.db/testRenameColumn") + "]]"); - Dataset table1 = spark.table("tablestore.default.testRenameColumn"); + Dataset table1 = spark.table("paimon.default.testRenameColumn"); List results = table1.select("a", "c").collectAsList(); assertThat(results.toString()).isEqualTo("[[1,1], [5,3]]"); // Rename "a" to "aa" - spark.sql("ALTER TABLE tablestore.default.testRenameColumn RENAME COLUMN a to aa"); + spark.sql("ALTER TABLE paimon.default.testRenameColumn RENAME COLUMN a to aa"); List afterRename = - spark.sql("SHOW CREATE TABLE tablestore.default.testRenameColumn").collectAsList(); + spark.sql("SHOW CREATE TABLE paimon.default.testRenameColumn").collectAsList(); assertThat(afterRename.toString()) .isEqualTo( "[[CREATE TABLE testRenameColumn (\n" @@ -171,7 +166,7 @@ public void testRenameColumn() { + " `c` STRING)\n" + buildTableProperties("default.db/testRenameColumn") + "]]"); - Dataset table2 = spark.table("tablestore.default.testRenameColumn"); + Dataset table2 = spark.table("paimon.default.testRenameColumn"); results = table2.select("aa", "c").collectAsList(); assertThat(results.toString()).isEqualTo("[[1,1], [5,3]]"); assertThatThrownBy(() -> table2.select("a", "c").collectAsList()) @@ -181,24 +176,24 @@ public void testRenameColumn() { @Test public void testRenamePartitionKey() { - spark.sql("USE tablestore"); + spark.sql("USE paimon"); spark.sql( "CREATE TABLE default.testRenamePartitionKey (\n" + "a BIGINT,\n" - + "b STRING) USING tablestore\n" + + "b STRING) USING paimon\n" + "COMMENT 'table comment'\n" + "PARTITIONED BY (a)\n" + "TBLPROPERTIES ('foo' = 'bar')"); List beforeRename = - spark.sql("SHOW CREATE TABLE tablestore.default.testRenamePartitionKey") + spark.sql("SHOW CREATE TABLE paimon.default.testRenamePartitionKey") .collectAsList(); assertThat(beforeRename.toString()) .isEqualTo( "[[CREATE TABLE testRenamePartitionKey (\n" + " `a` BIGINT,\n" + " `b` STRING)\n" - + "USING tablestore\n" + + "USING paimon\n" + "PARTITIONED BY (a)\n" + "COMMENT 'table comment'\n" + "TBLPROPERTIES(\n" @@ -211,7 +206,7 @@ public void testRenamePartitionKey() { assertThatThrownBy( () -> spark.sql( - "ALTER TABLE tablestore.default.testRenamePartitionKey RENAME COLUMN a to aa")) + "ALTER TABLE paimon.default.testRenamePartitionKey RENAME COLUMN a to aa")) .isInstanceOf(RuntimeException.class) .hasMessage( String.format( @@ -225,8 +220,7 @@ public void testDropSingleColumn() { writeTable("testDropSingleColumn", "(1, 2L, '1')", "(5, 6L, '3')"); List beforeDrop = - spark.sql("SHOW CREATE TABLE tablestore.default.testDropSingleColumn") - .collectAsList(); + spark.sql("SHOW CREATE TABLE paimon.default.testDropSingleColumn").collectAsList(); assertThat(beforeDrop.toString()) .isEqualTo( "[[CREATE TABLE testDropSingleColumn (\n" @@ -236,11 +230,10 @@ public void testDropSingleColumn() { + buildTableProperties("default.db/testDropSingleColumn") + "]]"); - spark.sql("ALTER TABLE tablestore.default.testDropSingleColumn DROP COLUMN a"); + spark.sql("ALTER TABLE paimon.default.testDropSingleColumn DROP COLUMN a"); List afterDrop = - spark.sql("SHOW CREATE TABLE tablestore.default.testDropSingleColumn") - .collectAsList(); + spark.sql("SHOW CREATE TABLE paimon.default.testDropSingleColumn").collectAsList(); assertThat(afterDrop.toString()) .isEqualTo( "[[CREATE TABLE testDropSingleColumn (\n" @@ -249,7 +242,7 @@ public void testDropSingleColumn() { + buildTableProperties("default.db/testDropSingleColumn") + "]]"); - Dataset table = spark.table("tablestore.default.testDropSingleColumn"); + Dataset table = spark.table("paimon.default.testDropSingleColumn"); List results = table.collectAsList(); assertThat(results.toString()).isEqualTo("[[2,1], [6,3]]"); } @@ -259,7 +252,7 @@ public void testDropColumns() { createTable("testDropColumns"); List beforeRename = - spark.sql("SHOW CREATE TABLE tablestore.default.testDropColumns").collectAsList(); + spark.sql("SHOW CREATE TABLE paimon.default.testDropColumns").collectAsList(); assertThat(beforeRename.toString()) .isEqualTo( "[[CREATE TABLE testDropColumns (\n" @@ -269,10 +262,10 @@ public void testDropColumns() { + buildTableProperties("default.db/testDropColumns") + "]]"); - spark.sql("ALTER TABLE tablestore.default.testDropColumns DROP COLUMNS a, b"); + spark.sql("ALTER TABLE paimon.default.testDropColumns DROP COLUMNS a, b"); List afterRename = - spark.sql("SHOW CREATE TABLE tablestore.default.testDropColumns").collectAsList(); + spark.sql("SHOW CREATE TABLE paimon.default.testDropColumns").collectAsList(); assertThat(afterRename.toString()) .isEqualTo( "[[CREATE TABLE testDropColumns (\n" @@ -283,24 +276,23 @@ public void testDropColumns() { @Test public void testDropPartitionKey() { - spark.sql("USE tablestore"); + spark.sql("USE paimon"); spark.sql( "CREATE TABLE default.testDropPartitionKey (\n" + "a BIGINT,\n" - + "b STRING) USING tablestore\n" + + "b STRING) USING paimon\n" + "COMMENT 'table comment'\n" + "PARTITIONED BY (a)\n" + "TBLPROPERTIES ('foo' = 'bar')"); List beforeRename = - spark.sql("SHOW CREATE TABLE tablestore.default.testDropPartitionKey") - .collectAsList(); + spark.sql("SHOW CREATE TABLE paimon.default.testDropPartitionKey").collectAsList(); assertThat(beforeRename.toString()) .isEqualTo( "[[CREATE TABLE testDropPartitionKey (\n" + " `a` BIGINT,\n" + " `b` STRING)\n" - + "USING tablestore\n" + + "USING paimon\n" + "PARTITIONED BY (a)\n" + "COMMENT 'table comment'\n" + "TBLPROPERTIES(\n" @@ -313,7 +305,7 @@ public void testDropPartitionKey() { assertThatThrownBy( () -> spark.sql( - "ALTER TABLE tablestore.default.testDropPartitionKey DROP COLUMN a")) + "ALTER TABLE paimon.default.testDropPartitionKey DROP COLUMN a")) .isInstanceOf(RuntimeException.class) .hasMessage( String.format( @@ -323,24 +315,23 @@ public void testDropPartitionKey() { @Test public void testDropPrimaryKey() { - spark.sql("USE tablestore"); + spark.sql("USE paimon"); spark.sql( "CREATE TABLE default.testDropPrimaryKey (\n" + "a BIGINT,\n" - + "b STRING) USING tablestore\n" + + "b STRING) USING paimon\n" + "COMMENT 'table comment'\n" + "PARTITIONED BY (a)\n" + "TBLPROPERTIES ('primary-key' = 'a, b')"); List beforeRename = - spark.sql("SHOW CREATE TABLE tablestore.default.testDropPrimaryKey") - .collectAsList(); + spark.sql("SHOW CREATE TABLE paimon.default.testDropPrimaryKey").collectAsList(); assertThat(beforeRename.toString()) .isEqualTo( "[[CREATE TABLE testDropPrimaryKey (\n" + " `a` BIGINT NOT NULL,\n" + " `b` STRING NOT NULL)\n" - + "USING tablestore\n" + + "USING paimon\n" + "PARTITIONED BY (a)\n" + "COMMENT 'table comment'\n" + "TBLPROPERTIES(\n" @@ -352,7 +343,7 @@ public void testDropPrimaryKey() { assertThatThrownBy( () -> spark.sql( - "ALTER TABLE tablestore.default.testDropPrimaryKey DROP COLUMN b")) + "ALTER TABLE paimon.default.testDropPrimaryKey DROP COLUMN b")) .isInstanceOf(RuntimeException.class) .hasMessage( String.format( @@ -412,12 +403,11 @@ public void testUpdateColumnPosition() { // missing column spark.sql("CREATE TABLE tableMissing (a INT , b BIGINT, c STRING)"); assertThatThrownBy(() -> spark.sql("ALTER TABLE tableMissing ALTER COLUMN d FIRST")) - .hasMessageContaining("Missing field d in table tablestore.default.tableMissing"); + .hasMessageContaining("Missing field d in table paimon.default.tableMissing"); spark.sql("CREATE TABLE tableMissingAfter (a INT , b BIGINT, c STRING)"); assertThatThrownBy(() -> spark.sql("ALTER TABLE tableMissingAfter ALTER COLUMN a AFTER d")) - .hasMessageContaining( - "Missing field d in table tablestore.default.tableMissingAfter"); + .hasMessageContaining("Missing field d in table paimon.default.tableMissingAfter"); } @Test @@ -425,18 +415,16 @@ public void testAlterColumnType() { createTable("testAlterColumnType"); writeTable("testAlterColumnType", "(1, 2L, '1')", "(5, 6L, '3')"); - spark.sql("ALTER TABLE tablestore.default.testAlterColumnType ALTER COLUMN b TYPE DOUBLE"); + spark.sql("ALTER TABLE paimon.default.testAlterColumnType ALTER COLUMN b TYPE DOUBLE"); assertThat( - spark.table("tablestore.default.testAlterColumnType").collectAsList() - .stream() + spark.table("paimon.default.testAlterColumnType").collectAsList().stream() .map(Row::toString) .collect(Collectors.toList())) .containsExactlyInAnyOrder("[1,2.0,1]", "[5,6.0,3]"); - spark.sql("ALTER TABLE tablestore.default.testAlterColumnType DROP COLUMNS a"); + spark.sql("ALTER TABLE paimon.default.testAlterColumnType DROP COLUMNS a"); assertThat( - spark.table("tablestore.default.testAlterColumnType").collectAsList() - .stream() + spark.table("paimon.default.testAlterColumnType").collectAsList().stream() .map(Row::toString) .collect(Collectors.toList())) .containsExactlyInAnyOrder("[2.0,1]", "[6.0,3]"); @@ -455,30 +443,30 @@ public void testAlterTableColumnNullability() { .isFalse(); // note: for Spark, it is illegal to change nullable column to non-nullable - spark.sql("ALTER TABLE tablestore.default.t2 ALTER COLUMN a DROP NOT NULL"); + spark.sql("ALTER TABLE paimon.default.t2 ALTER COLUMN a DROP NOT NULL"); assertThat(fieldIsNullable(getField(schema2(), 0))).isTrue(); - spark.sql("ALTER TABLE tablestore.default.t2 ALTER COLUMN b DROP NOT NULL"); + spark.sql("ALTER TABLE paimon.default.t2 ALTER COLUMN b DROP NOT NULL"); assertThat(fieldIsNullable(getField(schema2(), 1))).isTrue(); - spark.sql("ALTER TABLE tablestore.default.t2 ALTER COLUMN c DROP NOT NULL"); + spark.sql("ALTER TABLE paimon.default.t2 ALTER COLUMN c DROP NOT NULL"); assertThat(fieldIsNullable(getField(schema2(), 2))).isTrue(); - spark.sql("ALTER TABLE tablestore.default.t2 ALTER COLUMN c.c1 DROP NOT NULL"); + spark.sql("ALTER TABLE paimon.default.t2 ALTER COLUMN c.c1 DROP NOT NULL"); assertThat(fieldIsNullable(getNestedField(getField(schema2(), 2), 0))).isTrue(); - spark.sql("ALTER TABLE tablestore.default.t2 ALTER COLUMN c.c1.c12 DROP NOT NULL"); + spark.sql("ALTER TABLE paimon.default.t2 ALTER COLUMN c.c1.c12 DROP NOT NULL"); assertThat(fieldIsNullable(getNestedField(getNestedField(getField(schema2(), 2), 0), 1))) .isTrue(); } @Test public void testAlterPrimaryKeyNullability() { - spark.sql("USE tablestore"); + spark.sql("USE paimon"); spark.sql( "CREATE TABLE default.testAlterPkNullability (\n" + "a BIGINT,\n" - + "b STRING) USING tablestore\n" + + "b STRING) USING paimon\n" + "COMMENT 'table comment'\n" + "TBLPROPERTIES ('primary-key' = 'a')"); assertThatThrownBy( @@ -495,10 +483,10 @@ public void testAlterTableColumnComment() { createTable("testAlterTableColumnComment"); assertThat(getField(schema1(), 0).description()).isNull(); - spark.sql("ALTER TABLE tablestore.default.t1 ALTER COLUMN a COMMENT 'a new comment'"); + spark.sql("ALTER TABLE paimon.default.t1 ALTER COLUMN a COMMENT 'a new comment'"); assertThat(getField(schema1(), 0).description()).isEqualTo("a new comment"); - spark.sql("ALTER TABLE tablestore.default.t1 ALTER COLUMN a COMMENT 'yet another comment'"); + spark.sql("ALTER TABLE paimon.default.t1 ALTER COLUMN a COMMENT 'yet another comment'"); assertThat(getField(schema1(), 0).description()).isEqualTo("yet another comment"); assertThat(getField(schema2(), 2).description()).isEqualTo("comment about c"); @@ -511,13 +499,11 @@ public void testAlterTableColumnComment() { .isNull(); spark.sql( - "ALTER TABLE tablestore.default.t2 ALTER COLUMN c COMMENT 'yet another comment about c'"); - spark.sql("ALTER TABLE tablestore.default.t2 ALTER COLUMN c.c1 COMMENT 'a nested type'"); - spark.sql("ALTER TABLE tablestore.default.t2 ALTER COLUMN c.c2 COMMENT 'a bigint type'"); - spark.sql( - "ALTER TABLE tablestore.default.t2 ALTER COLUMN c.c1.c11 COMMENT 'a double type'"); - spark.sql( - "ALTER TABLE tablestore.default.t2 ALTER COLUMN c.c1.c12 COMMENT 'a boolean array'"); + "ALTER TABLE paimon.default.t2 ALTER COLUMN c COMMENT 'yet another comment about c'"); + spark.sql("ALTER TABLE paimon.default.t2 ALTER COLUMN c.c1 COMMENT 'a nested type'"); + spark.sql("ALTER TABLE paimon.default.t2 ALTER COLUMN c.c2 COMMENT 'a bigint type'"); + spark.sql("ALTER TABLE paimon.default.t2 ALTER COLUMN c.c1.c11 COMMENT 'a double type'"); + spark.sql("ALTER TABLE paimon.default.t2 ALTER COLUMN c.c1.c12 COMMENT 'a boolean array'"); assertThat(getField(schema2(), 2).description()).isEqualTo("yet another comment about c"); assertThat(getNestedField(getField(schema2(), 2), 0).description()) @@ -551,7 +537,7 @@ public void testAlterTableColumnComment() { public void testSchemaEvolution() { // Create table with fields [a, b, c] and insert 2 records spark.sql( - "CREATE TABLE tablestore.default.testSchemaEvolution(\n" + "CREATE TABLE paimon.default.testSchemaEvolution(\n" + "a INT NOT NULL, \n" + "b BIGINT NOT NULL, \n" + "c VARCHAR(10), \n" @@ -561,19 +547,18 @@ public void testSchemaEvolution() { + "TBLPROPERTIES ('file.format'='avro')"); writeTable("testSchemaEvolution", "(1, 2L, '3', 4, 5, 6)", "(7, 8L, '9', 10, 11, 12)"); assertThat( - spark.table("tablestore.default.testSchemaEvolution").collectAsList() - .stream() + spark.table("paimon.default.testSchemaEvolution").collectAsList().stream() .map(Row::toString) .collect(Collectors.toList())) .containsExactlyInAnyOrder("[1,2,3,4,5,6]", "[7,8,9,10,11,12]"); assertThat( - spark.table("tablestore.default.testSchemaEvolution").select("a", "c", "e") + spark.table("paimon.default.testSchemaEvolution").select("a", "c", "e") .collectAsList().stream() .map(Row::toString) .collect(Collectors.toList())) .containsExactlyInAnyOrder("[1,3,5]", "[7,9,11]"); assertThat( - spark.table("tablestore.default.testSchemaEvolution").filter("a>1") + spark.table("paimon.default.testSchemaEvolution").filter("a>1") .collectAsList().stream() .map(Row::toString) .collect(Collectors.toList())) @@ -582,21 +567,20 @@ public void testSchemaEvolution() { // Rename (a, int)->(aa, bigint), c->a, b->c, (d, int)->(b, bigint), (f, int)->(ff, float), // the fields are [(1, aa, bigint), (2, c, bigint), (3, a, string), (4, b, bigint), (5, e, // int), (6, ff, float)] and insert 2 records - spark.sql("ALTER TABLE tablestore.default.testSchemaEvolution RENAME COLUMN a to aa"); - spark.sql("ALTER TABLE tablestore.default.testSchemaEvolution ALTER COLUMN aa TYPE bigint"); - spark.sql("ALTER TABLE tablestore.default.testSchemaEvolution RENAME COLUMN c to a"); - spark.sql("ALTER TABLE tablestore.default.testSchemaEvolution RENAME COLUMN b to c"); - spark.sql("ALTER TABLE tablestore.default.testSchemaEvolution RENAME COLUMN d to b"); - spark.sql("ALTER TABLE tablestore.default.testSchemaEvolution ALTER COLUMN b TYPE bigint"); - spark.sql("ALTER TABLE tablestore.default.testSchemaEvolution RENAME COLUMN f to ff"); - spark.sql("ALTER TABLE tablestore.default.testSchemaEvolution ALTER COLUMN ff TYPE float"); + spark.sql("ALTER TABLE paimon.default.testSchemaEvolution RENAME COLUMN a to aa"); + spark.sql("ALTER TABLE paimon.default.testSchemaEvolution ALTER COLUMN aa TYPE bigint"); + spark.sql("ALTER TABLE paimon.default.testSchemaEvolution RENAME COLUMN c to a"); + spark.sql("ALTER TABLE paimon.default.testSchemaEvolution RENAME COLUMN b to c"); + spark.sql("ALTER TABLE paimon.default.testSchemaEvolution RENAME COLUMN d to b"); + spark.sql("ALTER TABLE paimon.default.testSchemaEvolution ALTER COLUMN b TYPE bigint"); + spark.sql("ALTER TABLE paimon.default.testSchemaEvolution RENAME COLUMN f to ff"); + spark.sql("ALTER TABLE paimon.default.testSchemaEvolution ALTER COLUMN ff TYPE float"); writeTable( "testSchemaEvolution", "(13L, 14L, '15', 16L, 17, 18.0F)", "(19L, 20L, '21', 22L, 23, 24.0F)"); assertThat( - spark.table("tablestore.default.testSchemaEvolution").collectAsList() - .stream() + spark.table("paimon.default.testSchemaEvolution").collectAsList().stream() .map(Row::toString) .collect(Collectors.toList())) .containsExactlyInAnyOrder( @@ -605,27 +589,26 @@ public void testSchemaEvolution() { "[13,14,15,16,17,18.0]", "[19,20,21,22,23,24.0]"); assertThat( - spark.table("tablestore.default.testSchemaEvolution") - .select("aa", "b", "ff").collectAsList().stream() + spark.table("paimon.default.testSchemaEvolution").select("aa", "b", "ff") + .collectAsList().stream() .map(Row::toString) .collect(Collectors.toList())) .containsExactlyInAnyOrder( "[1,4,6.0]", "[7,10,12.0]", "[13,16,18.0]", "[19,22,24.0]"); assertThat( - spark.table("tablestore.default.testSchemaEvolution") - .select("aa", "a", "ff").filter("b>10L").collectAsList().stream() + spark.table("paimon.default.testSchemaEvolution").select("aa", "a", "ff") + .filter("b>10L").collectAsList().stream() .map(Row::toString) .collect(Collectors.toList())) .containsExactlyInAnyOrder("[13,15,18.0]", "[19,21,24.0]"); // Drop fields aa, c, e, the fields are [(3, a, string), (4, b, bigint), (6, ff, float)] and // insert 2 records - spark.sql("ALTER TABLE tablestore.default.testSchemaEvolution DROP COLUMNS aa, c, e"); + spark.sql("ALTER TABLE paimon.default.testSchemaEvolution DROP COLUMNS aa, c, e"); writeTable("testSchemaEvolution", "('25', 26L, 27.0F)", "('28', 29L, 30.0)"); assertThat( - spark.table("tablestore.default.testSchemaEvolution").collectAsList() - .stream() + spark.table("paimon.default.testSchemaEvolution").collectAsList().stream() .map(Row::toString) .collect(Collectors.toList())) .containsExactlyInAnyOrder( @@ -636,7 +619,7 @@ public void testSchemaEvolution() { "[25,26,27.0]", "[28,29,30.0]"); assertThat( - spark.table("tablestore.default.testSchemaEvolution").select("a", "ff") + spark.table("paimon.default.testSchemaEvolution").select("a", "ff") .filter("b>10L").collectAsList().stream() .map(Row::toString) .collect(Collectors.toList())) @@ -645,15 +628,14 @@ public void testSchemaEvolution() { // Add new fields d, c, e, the fields are [(3, a, string), (4, b, bigint), (6, ff, float), // (7, d, int), (8, c, int), (9, e, int)] insert 2 records spark.sql( - "ALTER TABLE tablestore.default.testSchemaEvolution ADD COLUMNS (d INT, c INT, e INT)"); + "ALTER TABLE paimon.default.testSchemaEvolution ADD COLUMNS (d INT, c INT, e INT)"); writeTable( "testSchemaEvolution", "('31', 32L, 33.0F, 34, 35, 36)", "('37', 38L, 39.0F, 40, 41, 42)"); assertThat( - spark.table("tablestore.default.testSchemaEvolution").collectAsList() - .stream() + spark.table("paimon.default.testSchemaEvolution").collectAsList().stream() .map(Row::toString) .collect(Collectors.toList())) .containsExactlyInAnyOrder( @@ -666,7 +648,7 @@ public void testSchemaEvolution() { "[31,32,33.0,34,35,36]", "[37,38,39.0,40,41,42]"); assertThat( - spark.table("tablestore.default.testSchemaEvolution").filter("b>10") + spark.table("paimon.default.testSchemaEvolution").filter("b>10") .collectAsList().stream() .map(Row::toString) .collect(Collectors.toList())) @@ -678,7 +660,7 @@ public void testSchemaEvolution() { "[31,32,33.0,34,35,36]", "[37,38,39.0,40,41,42]"); assertThat( - spark.table("tablestore.default.testSchemaEvolution") + spark.table("paimon.default.testSchemaEvolution") .select("e", "a", "ff", "d", "b").filter("b>10L").collectAsList() .stream() .map(Row::toString) @@ -691,7 +673,7 @@ public void testSchemaEvolution() { "[36,31,33.0,34,32]", "[42,37,39.0,40,38]"); assertThat( - spark.table("tablestore.default.testSchemaEvolution") + spark.table("paimon.default.testSchemaEvolution") .select("e", "a", "ff", "d", "b").filter("b>10 and e is not null") .collectAsList().stream() .map(Row::toString) @@ -706,28 +688,27 @@ public void testFilesTable() { writeTable("testFilesTable", "(1, 2L, '3')", "(4, 5L, '6')"); assertThat( getFieldStatsList( - spark.sql("SELECT * FROM tablestore.default.`testFilesTable$files`") + spark.sql("SELECT * FROM paimon.default.`testFilesTable$files`") .collectAsList())) .containsExactlyInAnyOrder("{a=0, b=0, c=0},{a=1, b=2, c=3},{a=4, b=5, c=6}"); // Add new fields d, e, f and the fields are [a, b, c, d, e, f], insert 2 records - spark.sql( - "ALTER TABLE tablestore.default.testFilesTable ADD COLUMNS (d INT, e INT, f INT)"); + spark.sql("ALTER TABLE paimon.default.testFilesTable ADD COLUMNS (d INT, e INT, f INT)"); writeTable("testFilesTable", "(7, 8L, '9', 10, 11, 12)", "(13, 14L, '15', 16, 17, 18)"); assertThat( getFieldStatsList( - spark.sql("SELECT * FROM tablestore.default.`testFilesTable$files`") + spark.sql("SELECT * FROM paimon.default.`testFilesTable$files`") .collectAsList())) .containsExactlyInAnyOrder( "{a=0, b=0, c=0, d=2, e=2, f=2},{a=1, b=2, c=3, d=null, e=null, f=null},{a=4, b=5, c=6, d=null, e=null, f=null}", "{a=0, b=0, c=0, d=0, e=0, f=0},{a=7, b=8, c=15, d=10, e=11, f=12},{a=13, b=14, c=9, d=16, e=17, f=18}"); // Drop fields c, e and the fields are [a, b, d, f], insert 2 records - spark.sql("ALTER TABLE tablestore.default.testFilesTable DROP COLUMNS c, e"); + spark.sql("ALTER TABLE paimon.default.testFilesTable DROP COLUMNS c, e"); writeTable("testFilesTable", "(19, 20L, 21, 22)", "(23, 24L, 25, 26)"); assertThat( getFieldStatsList( - spark.sql("SELECT * FROM tablestore.default.`testFilesTable$files`") + spark.sql("SELECT * FROM paimon.default.`testFilesTable$files`") .collectAsList())) .containsExactlyInAnyOrder( "{a=0, b=0, d=2, f=2},{a=1, b=2, d=null, f=null},{a=4, b=5, d=null, f=null}", diff --git a/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/SparkWriteITCase.java b/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/SparkWriteITCase.java index 891c7bfee966..486bfce4d900 100644 --- a/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/SparkWriteITCase.java +++ b/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/SparkWriteITCase.java @@ -41,10 +41,10 @@ public class SparkWriteITCase { public static void startMetastoreAndSpark(@TempDir java.nio.file.Path tempDir) { Path warehousePath = new Path("file:" + tempDir.toString()); spark = SparkSession.builder().master("local[2]").getOrCreate(); - spark.conf().set("spark.sql.catalog.tablestore", SparkCatalog.class.getName()); - spark.conf().set("spark.sql.catalog.tablestore.warehouse", warehousePath.toString()); - spark.sql("CREATE DATABASE tablestore.db"); - spark.sql("USE tablestore.db"); + spark.conf().set("spark.sql.catalog.paimon", SparkCatalog.class.getName()); + spark.conf().set("spark.sql.catalog.paimon.warehouse", warehousePath.toString()); + spark.sql("CREATE DATABASE paimon.db"); + spark.sql("USE paimon.db"); } @AfterAll