diff --git a/.github/workflows/publish_snapshot.yml b/.github/workflows/publish_snapshot.yml index 7d370814da94..c7f97cab991f 100644 --- a/.github/workflows/publish_snapshot.yml +++ b/.github/workflows/publish_snapshot.yml @@ -64,6 +64,6 @@ jobs: echo "$ASF_PASSWORD" >> $tmp_settings echo "" >> $tmp_settings - mvn --settings $tmp_settings clean deploy -Dgpg.skip -Drat.skip -DskipTests -Papache-release + mvn --settings $tmp_settings clean deploy -Dgpg.skip -Drat.skip -DskipTests -Papache-release,spark3 rm $tmp_settings diff --git a/.github/workflows/unitcase-flink-jdk11.yml b/.github/workflows/utitcase-flink-jdk11.yml similarity index 100% rename from .github/workflows/unitcase-flink-jdk11.yml rename to .github/workflows/utitcase-flink-jdk11.yml diff --git a/.github/workflows/unitcase-jdk11.yml b/.github/workflows/utitcase-jdk11.yml similarity index 82% rename from .github/workflows/unitcase-jdk11.yml rename to .github/workflows/utitcase-jdk11.yml index 1baed87f9027..878ce5f96898 100644 --- a/.github/workflows/unitcase-jdk11.yml +++ b/.github/workflows/utitcase-jdk11.yml @@ -16,7 +16,7 @@ # limitations under the License. ################################################################################ -name: UTCase and ITCase Non Flink on JDK 11 +name: UTCase and ITCase Others on JDK 11 on: issue_comment: @@ -52,6 +52,11 @@ jobs: . .github/workflows/utils.sh jvm_timezone=$(random_timezone) echo "JVM timezone is set to $jvm_timezone" - mvn -T 1C -B clean install -pl '!paimon-e2e-tests,!org.apache.paimon:paimon-hive-connector-3.1' -Pskip-paimon-flink-tests -Duser.timezone=$jvm_timezone + test_modules="!paimon-e2e-tests,!org.apache.paimon:paimon-hive-connector-3.1," + for suffix in 3.5 3.4 3.3 3.2 ut; do + test_modules+="!org.apache.paimon:paimon-spark-${suffix}," + done + test_modules="${test_modules%,}" + mvn -T 1C -B clean install -pl "${test_modules}" -Pskip-paimon-flink-tests -Duser.timezone=$jvm_timezone env: MAVEN_OPTS: -Xmx4096m \ No newline at end of file diff --git a/.github/workflows/utitcase-spark-3.x.yml b/.github/workflows/utitcase-spark-3.x.yml new file mode 100644 index 000000000000..2d3df5f4d005 --- /dev/null +++ b/.github/workflows/utitcase-spark-3.x.yml @@ -0,0 +1,63 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +name: UTCase and ITCase Spark 3.x + +on: + push: + pull_request: + paths-ignore: + - 'docs/**' + - '**/*.md' + +env: + JDK_VERSION: 8 + +concurrency: + group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.number || github.run_id }} + cancel-in-progress: true + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v2 + - name: Set up JDK ${{ env.JDK_VERSION }} + uses: actions/setup-java@v2 + with: + java-version: ${{ env.JDK_VERSION }} + distribution: 'adopt' + - name: Build Spark + run: mvn -T 1C -B clean install -DskipTests + - name: Test Spark + timeout-minutes: 60 + run: | + # run tests with random timezone to find out timezone related bugs + . .github/workflows/utils.sh + jvm_timezone=$(random_timezone) + echo "JVM timezone is set to $jvm_timezone" + test_modules="" + for suffix in ut 3.5 3.4 3.3 3.2; do + test_modules+="org.apache.paimon:paimon-spark-${suffix}," + done + test_modules="${test_modules%,}" + mvn -T 1C -B test -pl "${test_modules}" -Duser.timezone=$jvm_timezone + env: + MAVEN_OPTS: -Xmx4096m \ No newline at end of file diff --git a/.github/workflows/utitcase-spark-4.x.yml b/.github/workflows/utitcase-spark-4.x.yml new file mode 100644 index 000000000000..c58fd7c03be2 --- /dev/null +++ b/.github/workflows/utitcase-spark-4.x.yml @@ -0,0 +1,63 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +name: UTCase and ITCase Spark 4.x + +on: + push: + pull_request: + paths-ignore: + - 'docs/**' + - '**/*.md' + +env: + JDK_VERSION: 17 + +concurrency: + group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.number || github.run_id }} + cancel-in-progress: true + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v2 + - name: Set up JDK ${{ env.JDK_VERSION }} + uses: actions/setup-java@v2 + with: + java-version: ${{ env.JDK_VERSION }} + distribution: 'adopt' + - name: Build Spark + run: mvn -T 1C -B clean install -DskipTests -Pspark4 + - name: Test Spark + timeout-minutes: 60 + run: | + # run tests with random timezone to find out timezone related bugs + . .github/workflows/utils.sh + jvm_timezone=$(random_timezone) + echo "JVM timezone is set to $jvm_timezone" + test_modules="" + for suffix in ut 4.0; do + test_modules+="org.apache.paimon:paimon-spark-${suffix}," + done + test_modules="${test_modules%,}" + mvn -T 1C -B test -pl "${test_modules}" -Duser.timezone=$jvm_timezone -Pspark4 + env: + MAVEN_OPTS: -Xmx4096m \ No newline at end of file diff --git a/.github/workflows/utitcase.yml b/.github/workflows/utitcase.yml index 431b44332232..8aa33f5b8218 100644 --- a/.github/workflows/utitcase.yml +++ b/.github/workflows/utitcase.yml @@ -16,7 +16,7 @@ # limitations under the License. ################################################################################ -name: UTCase and ITCase Non Flink +name: UTCase and ITCase Others on: push: @@ -53,6 +53,11 @@ jobs: . .github/workflows/utils.sh jvm_timezone=$(random_timezone) echo "JVM timezone is set to $jvm_timezone" - mvn -T 1C -B clean install -pl '!paimon-e2e-tests' -Pskip-paimon-flink-tests -Duser.timezone=$jvm_timezone + test_modules="!paimon-e2e-tests," + for suffix in 3.5 3.4 3.3 3.2 ut; do + test_modules+="!org.apache.paimon:paimon-spark-${suffix}," + done + test_modules="${test_modules%,}" + mvn -T 1C -B clean install -pl "${test_modules}" -Pskip-paimon-flink-tests -Duser.timezone=$jvm_timezone env: MAVEN_OPTS: -Xmx4096m \ No newline at end of file diff --git a/.gitignore b/.gitignore index 3831eb7e9ef1..25ebf232470a 100644 --- a/.gitignore +++ b/.gitignore @@ -17,4 +17,5 @@ target .DS_Store *.ipr *.iws +.java-version dependency-reduced-pom.xml diff --git a/docs/content/append-table/query.md b/docs/content/append-table/query-performance.md similarity index 94% rename from docs/content/append-table/query.md rename to docs/content/append-table/query-performance.md index 4f7e0ab9f66f..7ec745468ef5 100644 --- a/docs/content/append-table/query.md +++ b/docs/content/append-table/query-performance.md @@ -1,9 +1,9 @@ --- -title: "Query" +title: "Query Performance" weight: 3 type: docs aliases: -- /append-table/query.html +- /append-table/query-performance.html --- -# Query +# Query Performance ## Data Skipping By Order @@ -57,8 +57,6 @@ multiple columns. Different file index may be efficient in different scenario. For example bloom filter may speed up query in point lookup scenario. Using a bitmap may consume more space but can result in greater accuracy. -Currently, file index is only supported in append-only table. - `Bloom Filter`: * `file-index.bloom-filter.columns`: specify the columns that need bloom filter index. * `file-index.bloom-filter..fpp` to config false positive probability. @@ -67,6 +65,9 @@ Currently, file index is only supported in append-only table. `Bitmap`: * `file-index.bitmap.columns`: specify the columns that need bitmap index. +`Bit-Slice Index Bitmap` +* `file-index.bsi.columns`: specify the columns that need bsi index. + More filter types will be supported... If you want to add file index to existing table, without any rewrite, you can use `rewrite_file_index` procedure. Before diff --git a/docs/content/flink/cdc-ingestion/_index.md b/docs/content/cdc-ingestion/_index.md similarity index 90% rename from docs/content/flink/cdc-ingestion/_index.md rename to docs/content/cdc-ingestion/_index.md index 0c76825b0f6a..b3d4dce2a0c7 100644 --- a/docs/content/flink/cdc-ingestion/_index.md +++ b/docs/content/cdc-ingestion/_index.md @@ -1,7 +1,9 @@ --- title: CDC Ingestion +icon: +bold: true bookCollapseSection: true -weight: 95 +weight: 91 --- + +# Catalog + +Paimon provides a Catalog abstraction to manage the table of contents and metadata. The Catalog abstraction provides +a series of ways to help you better integrate with computing engines. We always recommend that you use Catalog to +access the Paimon table. + +## Catalogs + +Paimon catalogs currently support three types of metastores: + +* `filesystem` metastore (default), which stores both metadata and table files in filesystems. +* `hive` metastore, which additionally stores metadata in Hive metastore. Users can directly access the tables from Hive. +* `jdbc` metastore, which additionally stores metadata in relational databases such as MySQL, Postgres, etc. + +## Filesystem Catalog + +Metadata and table files are stored under `hdfs:///path/to/warehouse`. + +```sql +-- Flink SQL +CREATE CATALOG my_catalog WITH ( + 'type' = 'paimon', + 'warehouse' = 'hdfs:///path/to/warehouse' +); +``` + +## Hive Catalog + +By using Paimon Hive catalog, changes to the catalog will directly affect the corresponding Hive metastore. Tables +created in such catalog can also be accessed directly from Hive. Metadata and table files are stored under +`hdfs:///path/to/warehouse`. In addition, schema is also stored in Hive metastore. + +```sql +-- Flink SQL +CREATE CATALOG my_hive WITH ( + 'type' = 'paimon', + 'metastore' = 'hive', + -- 'warehouse' = 'hdfs:///path/to/warehouse', default use 'hive.metastore.warehouse.dir' in HiveConf +); +``` + +By default, Paimon does not synchronize newly created partitions into Hive metastore. Users will see an unpartitioned +table in Hive. Partition push-down will be carried out by filter push-down instead. + +If you want to see a partitioned table in Hive and also synchronize newly created partitions into Hive metastore, +please set the table option `metastore.partitioned-table` to true. + +## JDBC Catalog + +By using the Paimon JDBC catalog, changes to the catalog will be directly stored in relational databases such as SQLite, +MySQL, postgres, etc. + +```sql +-- Flink SQL +CREATE CATALOG my_jdbc WITH ( + 'type' = 'paimon', + 'metastore' = 'jdbc', + 'uri' = 'jdbc:mysql://:/', + 'jdbc.user' = '...', + 'jdbc.password' = '...', + 'catalog-key'='jdbc', + 'warehouse' = 'hdfs:///path/to/warehouse' +); +``` diff --git a/docs/content/concepts/data-types.md b/docs/content/concepts/data-types.md new file mode 100644 index 000000000000..b33dcd428399 --- /dev/null +++ b/docs/content/concepts/data-types.md @@ -0,0 +1,179 @@ +--- +title: "Data Types" +weight: 7 +type: docs +aliases: +- /concepts/data-types.html +--- + + +# Data Types + +A data type describes the logical type of a value in the table ecosystem. It can be used to declare input and/or output types of operations. + +All data types supported by Paimon are as follows: + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
DataTypeDescription
BOOLEANData type of a boolean with a (possibly) three-valued logic of TRUE, FALSE, and UNKNOWN.
CHAR
+ CHAR(n) + 		Data type of a fixed-length character string.

+ The type can be declared using CHAR(n) where n is the number of code points. n must have a value between 1 and 2,147,483,647 (both inclusive). If no length is specified, n is equal to 1. +
VARCHAR
+ VARCHAR(n)

+ STRING + 		Data type of a variable-length character string.

+ The type can be declared using VARCHAR(n) where n is the maximum number of code points. n must have a value between 1 and 2,147,483,647 (both inclusive). If no length is specified, n is equal to 1.

+ STRING is a synonym for VARCHAR(2147483647). +
BINARY
+ BINARY(n)

+ 		Data type of a fixed-length binary string (=a sequence of bytes).

+ The type can be declared using BINARY(n) where n is the number of bytes. n must have a value between 1 and 2,147,483,647 (both inclusive). If no length is specified, n is equal to 1. +
VARBINARY
+ VARBINARY(n)

+ BYTES + 		Data type of a variable-length binary string (=a sequence of bytes).

+ The type can be declared using VARBINARY(n) where n is the maximum number of bytes. n must have a value between 1 and 2,147,483,647 (both inclusive). If no length is specified, n is equal to 1.

+ BYTES is a synonym for VARBINARY(2147483647). +
DECIMAL
+ DECIMAL(p)
+ DECIMAL(p, s) + 		Data type of a decimal number with fixed precision and scale.

+ The type can be declared using DECIMAL(p, s) where p is the number of digits in a number (precision) and s is the number of digits to the right of the decimal point in a number (scale). p must have a value between 1 and 38 (both inclusive). s must have a value between 0 and p (both inclusive). The default value for p is 10. The default value for s is 0. +
TINYINTData type of a 1-byte signed integer with values from -128 to 127.
SMALLINTData type of a 2-byte signed integer with values from -32,768 to 32,767.
INTData type of a 4-byte signed integer with values from -2,147,483,648 to 2,147,483,647.
BIGINTData type of an 8-byte signed integer with values from -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807.
FLOATData type of a 4-byte single precision floating point number.

+ Compared to the SQL standard, the type does not take parameters. +
DOUBLEData type of an 8-byte double precision floating point number.
DATEData type of a date consisting of year-month-day with values ranging from 0000-01-01 to 9999-12-31.

+ Compared to the SQL standard, the range starts at year 0000. +
TIME
+ TIME(p) + 		Data type of a time without time zone consisting of hour:minute:second[.fractional] with up to nanosecond precision and values ranging from 00:00:00.000000000 to 23:59:59.999999999.

+ The type can be declared using TIME(p) where p is the number of digits of fractional seconds (precision). p must have a value between 0 and 9 (both inclusive). If no precision is specified, p is equal to 0. +
TIMESTAMP
+ TIMESTAMP(p) + 		Data type of a timestamp without time zone consisting of year-month-day hour:minute:second[.fractional] with up to nanosecond precision and values ranging from 0000-01-01 00:00:00.000000000 to 9999-12-31 23:59:59.999999999.

+ The type can be declared using TIMESTAMP(p) where p is the number of digits of fractional seconds (precision). p must have a value between 0 and 9 (both inclusive). If no precision is specified, p is equal to 6. +
TIMESTAMP WITH TIME ZONE
+ TIMESTAMP(p) WITH TIME ZONE + 		Data type of a timestamp with time zone consisting of year-month-day hour:minute:second[.fractional] zone with up to nanosecond precision and values ranging from 0000-01-01 00:00:00.000000000 +14:59 to 9999-12-31 23:59:59.999999999 -14:59.

+ This type fills the gap between time zone free and time zone mandatory timestamp types by allowing the interpretation of UTC timestamps according to the configured session time zone. A conversion from and to int describes the number of seconds since epoch. A conversion from and to long describes the number of milliseconds since epoch. +
ARRAY<t>Data type of an array of elements with same subtype.

+ Compared to the SQL standard, the maximum cardinality of an array cannot be specified but is fixed at 2,147,483,647. Also, any valid type is supported as a subtype.

+ The type can be declared using ARRAY<t> where t is the data type of the contained elements. +
MAP<kt, vt>Data type of an associative array that maps keys (including NULL) to values (including NULL). A map cannot contain duplicate keys; each key can map to at most one value.

+ There is no restriction of element types; it is the responsibility of the user to ensure uniqueness.

+ The type can be declared using MAP<kt, vt> where kt is the data type of the key elements and vt is the data type of the value elements. +
MULTISET<t>Data type of a multiset (=bag). Unlike a set, it allows for multiple instances for each of its elements with a common subtype. Each unique value (including NULL) is mapped to some multiplicity.

+ There is no restriction of element types; it is the responsibility of the user to ensure uniqueness.

+ The type can be declared using MULTISET<t> where t is the data type of the contained elements. +
ROW<n0 t0, n1 t1, ...>
+ ROW<n0 t0 'd0', n1 t1 'd1', ...> + 		Data type of a sequence of fields.

+ A field consists of a field name, field type, and an optional description. The most specific type of a row of a table is a row type. In this case, each column of the row corresponds to the field of the row type that has the same ordinal position as the column.

+ Compared to the SQL standard, an optional field description simplifies the handling with complex structures.

+ A row type is similar to the STRUCT type known from other non-standard-compliant frameworks.

+ The type can be declared using ROW<n0 t0 'd0', n1 t1 'd1', ...> where n is the unique name of a field, t is the logical type of a field, d is the description of a field. +
diff --git a/docs/content/concepts/spec/_index.md b/docs/content/concepts/spec/_index.md index 3bd8e657ffbc..cc148d6a8b53 100644 --- a/docs/content/concepts/spec/_index.md +++ b/docs/content/concepts/spec/_index.md @@ -1,7 +1,7 @@ --- title: Specification bookCollapseSection: true -weight: 4 +weight: 8 --- + +# Table Types + +Paimon supports table types: + +1. table with pk: Paimon Data Table with Primary key +2. table w/o pk: Paimon Data Table without Primary key +3. view: metastore required, views in SQL are a kind of virtual table +4. format-table: file format table refers to a directory that contains multiple files of the same format, where + operations on this table allow for reading or writing to these files, compatible with Hive tables +5. object table: provides metadata indexes for unstructured data objects in the specified Object Storage directory. +6. materialized-table: aimed at simplifying both batch and stream data pipelines, providing a consistent development + experience, see [Flink Materialized Table](https://nightlies.apache.org/flink/flink-docs-master/docs/dev/table/materialized-table/overview/) + +## Table with PK + +See [Paimon with Primary key]({{< ref "primary-key-table/overview" >}}). + +Primary keys consist of a set of columns that contain unique values for each record. Paimon enforces data ordering by +sorting the primary key within each bucket, allowing streaming update and streaming changelog read. + +The definition of primary key is similar to that of standard SQL, as it ensures that there is only one data entry for +the same primary key during batch queries. + +{{< tabs "primary-table" >}} +{{< tab "Flink SQL" >}} + +```sql +CREATE TABLE my_table ( + a INT PRIMARY KEY NOT ENFORCED, + b STRING +) WITH ( + 'bucket'='8' +) +``` +{{< /tab >}} + +{{< tab "Spark SQL" >}} + +```sql +CREATE TABLE my_table ( + a INT, + b STRING +) TBLPROPERTIES ( + 'primary-key' = 'a', + 'bucket' = '8' +) +``` + +{{< /tab >}} +{{< /tabs >}} + +## Table w/o PK + +See [Paimon w/o Primary key]({{< ref "append-table/overview" >}}). + +If a table does not have a primary key defined, it is an append table. Compared to the primary key table, it does not +have the ability to directly receive changelogs. It cannot be directly updated with data through streaming upsert. It +can only receive incoming data from append data. + +However, it also supports batch sql: DELETE, UPDATE, and MERGE-INTO. + +```sql +CREATE TABLE my_table ( + a INT, + b STRING +) +``` + +## View + +View is supported when the metastore can support view, for example, hive metastore. If you don't have metastore, you +can only use temporary View, which only exists in the current session. This chapter mainly describes persistent views. + +View will currently save the original SQL. If you need to use View across engines, you can write a cross engine +SQL statement. For example: + +{{< tabs "view" >}} +{{< tab "Flink SQL" >}} + +```sql +CREATE VIEW [IF NOT EXISTS] [catalog_name.][db_name.]view_name + [( columnName [, columnName ]* )] [COMMENT view_comment] +AS query_expression; + +DROP VIEW [IF EXISTS] [catalog_name.][db_name.]view_name; + +SHOW VIEWS; + +SHOW CREATE VIEW my_view; +``` +{{< /tab >}} + +{{< tab "Spark SQL" >}} + +```sql +CREATE [OR REPLACE] VIEW [IF NOT EXISTS] [catalog_name.][db_name.]view_name + [( columnName [, columnName ]* )] [COMMENT view_comment] +AS query_expression; + +DROP VIEW [IF EXISTS] [catalog_name.][db_name.]view_name; + +SHOW VIEWS; +``` + +{{< /tab >}} + +{{< /tabs >}} + +## Format Table + +Format table is supported when the metastore can support format table, for example, hive metastore. The Hive tables +inside the metastore will be mapped to Paimon's Format Table for computing engines (Spark, Hive, Flink) to read and write. + +Format table refers to a directory that contains multiple files of the same format, where operations on this table +allow for reading or writing to these files, facilitating the retrieval of existing data and the addition of new files. + +Partitioned file format table just like the standard hive format. Partitions are discovered and inferred based on +directory structure. + +Format Table is enabled by default, you can disable it by configuring Catalog option: `'format-table.enabled'`. + +Currently only support `CSV`, `Parquet`, `ORC` formats. + +{{< tabs "format-table" >}} +{{< tab "Flink-CSV" >}} + +```sql +CREATE TABLE my_csv_table ( + a INT, + b STRING +) WITH ( + 'type'='format-table', + 'file.format'='csv', + 'field-delimiter'=',' +) +``` +{{< /tab >}} + +{{< tab "Spark-CSV" >}} + +```sql +CREATE TABLE my_csv_table ( + a INT, + b STRING +) USING csv OPTIONS ('field-delimiter' ',') +``` + +{{< /tab >}} + +{{< tab "Flink-Parquet" >}} + +```sql +CREATE TABLE my_parquet_table ( + a INT, + b STRING +) WITH ( + 'type'='format-table', + 'file.format'='parquet' +) +``` +{{< /tab >}} + +{{< tab "Spark-Parquet" >}} + +```sql +CREATE TABLE my_parquet_table ( + a INT, + b STRING +) USING parquet +``` + +{{< /tab >}} + +{{< /tabs >}} + +## Object Table + +Object Table provides metadata indexes for unstructured data objects in the specified Object Storage storage directory. +Object tables allow users to analyze unstructured data in Object Storage: + +1. Use Python API to manipulate these unstructured data, such as converting images to PDF format. +2. Model functions can also be used to perform inference, and then the results of these operations can be concatenated + with other structured data in the Catalog. + +The object table is managed by Catalog and can also have access permissions and the ability to manage blood relations. + +{{< tabs "object-table" >}} + +{{< tab "Flink-SQL" >}} + +```sql +-- Create Object Table + +CREATE TABLE `my_object_table` WITH ( + 'type' = 'object-table', + 'object-location' = 'oss://my_bucket/my_location' +); + +-- Refresh Object Table + +CALL sys.refresh_object_table('mydb.my_object_table'); + +-- Query Object Table + +SELECT * FROM `my_object_table`; + +-- Query Object Table with Time Travel + +SELECT * FROM `my_object_table` /*+ OPTIONS('scan.snapshot-id' = '1') */; +``` + +{{< /tab >}} + +{{< tab "Spark-SQL" >}} + +```sql +-- Create Object Table + +CREATE TABLE `my_object_table` TBLPROPERTIES ( + 'type' = 'object-table', + 'object-location' = 'oss://my_bucket/my_location' +); + +-- Refresh Object Table + +CALL sys.refresh_object_table('mydb.my_object_table'); + +-- Query Object Table + +SELECT * FROM `my_object_table`; + +-- Query Object Table with Time Travel + +SELECT * FROM `my_object_table` VERSION AS OF 1; +``` + +{{< /tab >}} + +{{< /tabs >}} + +## Materialized Table + +Materialized Table aimed at simplifying both batch and stream data pipelines, providing a consistent development +experience, see [Flink Materialized Table](https://nightlies.apache.org/flink/flink-docs-master/docs/dev/table/materialized-table/overview/). + +Now only Flink SQL integrate to Materialized Table, we plan to support it in Spark SQL too. + +```sql +CREATE MATERIALIZED TABLE continuous_users_shops +PARTITIONED BY (ds) +FRESHNESS = INTERVAL '30' SECOND +AS SELECT + user_id, + ds, + SUM (payment_amount_cents) AS payed_buy_fee_sum, + SUM (1) AS PV +FROM ( + SELECT user_id, order_created_at AS ds, payment_amount_cents + FROM json_source + ) AS tmp +GROUP BY user_id, ds; +``` diff --git a/docs/content/engines/doris.md b/docs/content/engines/doris.md index 634e7f7c71da..6d22bc376a88 100644 --- a/docs/content/engines/doris.md +++ b/docs/content/engines/doris.md @@ -73,13 +73,13 @@ See [Apache Doris Website](https://doris.apache.org/docs/lakehouse/datalake-anal 1. Query Paimon table with full qualified name - ``` + ```sql SELECT * FROM paimon_hdfs.paimon_db.paimon_table; ``` 2. Switch to Paimon Catalog and query - ``` + ```sql SWITCH paimon_hdfs; USE paimon_db; SELECT * FROM paimon_table; @@ -89,11 +89,11 @@ See [Apache Doris Website](https://doris.apache.org/docs/lakehouse/datalake-anal - Read optimized for Primary Key Table - Doris can utilize the [Read optimized](https://paimon.apache.org/releases/release-0.6/#read-optimized) feature for Primary Key Table(release in Paimon 0.6), by reading base data files using native Parquet/ORC reader and delta file using JNI. + Doris can utilize the [Read optimized](https://paimon.apache.org/docs/0.8/primary-key-table/read-optimized/) feature for Primary Key Table(release in Paimon 0.6), by reading base data files using native Parquet/ORC reader and delta file using JNI. - Deletion Vectors - Doris(2.1.4+) natively supports [Deletion Vectors](https://paimon.apache.org/releases/release-0.8/#deletion-vectors)(released in Paimon 0.8). + Doris(2.1.4+) natively supports [Deletion Vectors](https://paimon.apache.org/docs/0.8/primary-key-table/deletion-vectors/)(released in Paimon 0.8). ## Doris to Paimon type mapping diff --git a/docs/content/engines/presto.md b/docs/content/engines/presto.md deleted file mode 100644 index c336226bcf0a..000000000000 --- a/docs/content/engines/presto.md +++ /dev/null @@ -1,321 +0,0 @@ ---- -title: "Presto" -weight: 6 -type: docs -aliases: -- /engines/presto.html ---- - - -# Presto - -This documentation is a guide for using Paimon in Presto. - -## Version - -Paimon currently supports Presto 0.236 and above. - -## Preparing Paimon Jar File - -{{< stable >}} - -Download from master: -https://paimon.apache.org/docs/master/project/download/ - -{{< /stable >}} - -{{< unstable >}} - -| Version | Jar | -|-----------------|-----------------------------------------------------------------------------------------------------------------------------------------------------| -| [0.236, 0.268) | [paimon-presto-0.236-{{< version >}}-plugin.tar.gz](https://repository.apache.org/snapshots/org/apache/paimon/paimon-presto-0.236/{{< version >}}/) | -| [0.268, 0.273) | [paimon-presto-0.268-{{< version >}}-plugin.tar.gz](https://repository.apache.org/snapshots/org/apache/paimon/paimon-presto-0.268/{{< version >}}/) | -| [0.273, latest] | [paimon-presto-0.273-{{< version >}}-plugin.tar.gz](https://repository.apache.org/snapshots/org/apache/paimon/paimon-presto-0.273/{{< version >}}/) | - -{{< /unstable >}} - -You can also manually build a bundled jar from the source code. - -To build from the source code, [clone the git repository]({{< presto_github_repo >}}). - -Build presto connector plugin with the following command. - -``` -mvn clean install -DskipTests -``` - -After the packaging is complete, you can choose the corresponding connector based on your own Presto version: - -| Version | Package | -|-----------------|----------------------------------------------------------------------------------| -| [0.236, 0.268) | `./paimon-presto-0.236/target/paimon-presto-0.236-{{< version >}}-plugin.tar.gz` | -| [0.268, 0.273) | `./paimon-presto-0.268/target/paimon-presto-0.268-{{< version >}}-plugin.tar.gz` | -| [0.273, latest] | `./paimon-presto-0.273/target/paimon-presto-0.273-{{< version >}}-plugin.tar.gz` | - -Of course, we also support different versions of Hive and Hadoop. But note that we utilize -Presto-shaded versions of Hive and Hadoop packages to address dependency conflicts. -You can check the following two links to select the appropriate versions of Hive and Hadoop: - -[hadoop-apache2](https://mvnrepository.com/artifact/com.facebook.presto.hadoop/hadoop-apache2) - -[hive-apache](https://mvnrepository.com/artifact/com.facebook.presto.hive/hive-apache) - -Both Hive 2 and 3, as well as Hadoop 2 and 3, are supported. - -For example, if your presto version is 0.274, hive and hadoop version is 2.x, you could run: - -```bash -mvn clean install -DskipTests -am -pl paimon-presto-0.273 -Dpresto.version=0.274 -Dhadoop.apache2.version=2.7.4-9 -Dhive.apache.version=1.2.2-2 -``` - -## Tmp Dir - -Paimon will unzip some jars to the tmp directory for codegen. By default, Presto will use `'/tmp'` as the temporary -directory, but `'/tmp'` may be periodically deleted. - -You can configure this environment variable when Presto starts: -```shell --Djava.io.tmpdir=/path/to/other/tmpdir -``` - -Let Paimon use a secure temporary directory. - -## Configure Paimon Catalog - -### Install Paimon Connector - -```bash -tar -zxf paimon-presto-${PRESTO_VERSION}/target/paimon-presto-${PRESTO_VERSION}-${PAIMON_VERSION}-plugin.tar.gz -C ${PRESTO_HOME}/plugin -``` - -Note that, the variable `PRESTO_VERSION` is module name, must be one of 0.236, 0.268, 0.273. - -### Configuration - -```bash -cd ${PRESTO_HOME} -mkdir -p etc/catalog -``` - -```properties -connector.name=paimon -# set your filesystem path, such as hdfs://namenode01:8020/path and s3://${YOUR_S3_BUCKET}/path -warehouse=${YOUR_FS_PATH} -``` - -If you are using HDFS FileSystem, you will also need to do one more thing: choose one of the following ways to configure your HDFS: - -- set environment variable HADOOP_HOME. -- set environment variable HADOOP_CONF_DIR. -- configure `hadoop-conf-dir` in the properties. - -If you are using S3 FileSystem, you need to add `paimon-s3-${PAIMON_VERSION}.jar` in `${PRESTO_HOME}/plugin/paimon` and additionally configure the following properties in `paimon.properties`: - -```properties -s3.endpoint=${YOUR_ENDPOINTS} -s3.access-key=${YOUR_AK} -s3.secret-key=${YOUR_SK} -``` - -**Query HiveCatalog table:** - -```bash -vim etc/catalog/paimon.properties -``` - -and set the following config: - -```properties -connector.name=paimon -# set your filesystem path, such as hdfs://namenode01:8020/path and s3://${YOUR_S3_BUCKET}/path -warehouse=${YOUR_FS_PATH} -metastore=hive -uri=thrift://${YOUR_HIVE_METASTORE}:9083 -``` - -## Kerberos - -You can configure kerberos keytab file when using KERBEROS authentication in the properties. - -``` -security.kerberos.login.principal=hadoop-user -security.kerberos.login.keytab=/etc/presto/hdfs.keytab -``` - -Keytab files must be distributed to every node in the cluster that runs Presto. - -## Create Schema - -``` -CREATE SCHEMA paimon.test_db; -``` - -## Create Table - -``` -CREATE TABLE paimon.test_db.orders ( - order_key bigint, - order_status varchar, - total_price decimal(18,4), - order_date date -) -WITH ( - file_format = 'ORC', - primary_key = ARRAY['order_key','order_date'], - partitioned_by = ARRAY['order_date'], - bucket = '2', - bucket_key = 'order_key', - changelog_producer = 'input' -) -``` - -## Add Column - -``` -CREATE TABLE paimon.test_db.orders ( - order_key bigint, - orders_tatus varchar, - total_price decimal(18,4), - order_date date -) -WITH ( - file_format = 'ORC', - primary_key = ARRAY['order_key','order_date'], - partitioned_by = ARRAY['order_date'], - bucket = '2', - bucket_key = 'order_key', - changelog_producer = 'input' -) - -ALTER TABLE paimon.test_db.orders ADD COLUMN "shipping_address varchar; -``` - -## Query - -``` -SELECT * FROM paimon.default.MyTable -``` - -## Presto to Paimon type mapping - -This section lists all supported type conversion between Presto and Paimon. -All Presto's data types are available in package ` com.facebook.presto.common.type`. - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Presto Data TypePaimon Data TypeAtomic Type
RowTypeRowTypefalse
MapTypeMapTypefalse
ArrayTypeArrayTypefalse
BooleanTypeBooleanTypetrue
TinyintTypeTinyIntTypetrue
SmallintTypeSmallIntTypetrue
IntegerTypeIntTypetrue
BigintTypeBigIntTypetrue
RealTypeFloatTypetrue
DoubleTypeDoubleTypetrue
CharType(length)CharType(length)true
VarCharType(VarCharType.MAX_LENGTH)VarCharType(VarCharType.MAX_LENGTH)true
VarCharType(length)VarCharType(length), length is less than VarCharType.MAX_LENGTHtrue
DateTypeDateTypetrue
TimestampTypeTimestampTypetrue
DecimalType(precision, scale)DecimalType(precision, scale)true
VarBinaryType(length)VarBinaryType(length)true
TimestampWithTimeZoneTypeLocalZonedTimestampTypetrue
diff --git a/docs/content/engines/starrocks.md b/docs/content/engines/starrocks.md index 1ab821a9a103..dda22d35f76a 100644 --- a/docs/content/engines/starrocks.md +++ b/docs/content/engines/starrocks.md @@ -81,7 +81,7 @@ SELECT * FROM paimon_catalog.test_db.partition_tbl$partitions; ## StarRocks to Paimon type mapping This section lists all supported type conversion between StarRocks and Paimon. -All StarRocks’s data types can be found in this doc [StarRocks Data type overview](https://docs.starrocks.io/docs/sql-reference/data-types/data-type-list/). +All StarRocks’s data types can be found in this doc [StarRocks Data type overview](https://docs.starrocks.io/docs/sql-reference/data-types/). diff --git a/docs/content/engines/trino.md b/docs/content/engines/trino.md index 0f0fe8b94bf9..bef10f9d2870 100644 --- a/docs/content/engines/trino.md +++ b/docs/content/engines/trino.md @@ -30,36 +30,22 @@ This documentation is a guide for using Paimon in Trino. ## Version -Paimon currently supports Trino 420 and above. +Paimon currently supports Trino 440. ## Filesystem -From version 0.8, paimon share trino filesystem for all actions, which means, you should -config trino filesystem before using trino-paimon. You can find information about how to config -filesystems for trino on trino official website. +From version 0.8, Paimon share Trino filesystem for all actions, which means, you should +config Trino filesystem before using trino-paimon. You can find information about how to config +filesystems for Trino on Trino official website. ## Preparing Paimon Jar File -{{< stable >}} - -Download from master: -https://paimon.apache.org/docs/master/project/download/ - -{{< /stable >}} - -{{< unstable >}} - -| Version | Package | -|---------------|-----------------------------------------------------------------------------------------------------------------------------------------------| -| [420, 426] | [paimon-trino-420-{{< version >}}-plugin.tar.gz](https://repository.apache.org/snapshots/org/apache/paimon/paimon-trino-420/{{< version >}}/) | -| [427, latest] | [paimon-trino-427-{{< version >}}-plugin.tar.gz](https://repository.apache.org/snapshots/org/apache/paimon/paimon-trino-427/{{< version >}}/) | - -{{< /unstable >}} +[Download]({{< ref "project/download" >}}) You can also manually build a bundled jar from the source code. However, there are a few preliminary steps that need to be taken before compiling: - To build from the source code, [clone the git repository]({{< trino_github_repo >}}). -- Install JDK17 locally, and configure JDK17 as a global environment variable; +- Install JDK21 locally, and configure JDK21 as a global environment variable; Then,you can build bundled jar with the following command: @@ -78,28 +64,17 @@ For example, if you want to use Hadoop 3.3.5-1, you can use the following comman mvn clean install -DskipTests -Dhadoop.apache.version=3.3.5-1 ``` -## Tmp Dir - -Paimon will unzip some jars to the tmp directory for codegen. By default, Trino will use `'/tmp'` as the temporary -directory, but `'/tmp'` may be periodically deleted. - -You can configure this environment variable when Trino starts: -```shell --Djava.io.tmpdir=/path/to/other/tmpdir -``` - -Let Paimon use a secure temporary directory. - ## Configure Paimon Catalog ### Install Paimon Connector ```bash tar -zxf paimon-trino--{{< version >}}-plugin.tar.gz -C ${TRINO_HOME}/plugin ``` -the variable `trino-version` is module name, must be one of 420, 427. -> NOTE: For JDK 17, when Deploying Trino, should add jvm options: `--add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED` + +> NOTE: For JDK 21, when Deploying Trino, should add jvm options: `--add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED` ### Configure + Catalogs are registered by creating a catalog properties file in the etc/catalog directory. For example, create etc/catalog/paimon.properties with the following contents to mount the paimon connector as the paimon catalog: ``` @@ -113,7 +88,7 @@ If you are using HDFS, choose one of the following ways to configure your HDFS: - set environment variable HADOOP_CONF_DIR. - configure `hadoop-conf-dir` in the properties. -If you are using a hadoop filesystem, you can still use trino-hdfs and trino-hive to config it. +If you are using a Hadoop filesystem, you can still use trino-hdfs and trino-hive to config it. For example, if you use oss as a storage, you can write in `paimon.properties` according to [Trino Reference](https://trino.io/docs/current/connector/hive.html#hdfs-configuration): ``` @@ -186,9 +161,6 @@ SELECT * FROM paimon.test_db.orders ``` ## Query with Time Traveling -{{< tabs "time-travel-example" >}} - -{{< tab "version >=420" >}} ```sql -- read the snapshot from specified timestamp @@ -208,10 +180,15 @@ you have a tag named '1' based on snapshot 2, the statement `SELECT * FROM paimo instead of snapshot 1. {{< /hint >}} -{{< /tab >}} +## Insert +``` +INSERT INTO paimon.test_db.orders VALUES (.....); +``` -{{< /tabs >}} +Supports: +- primary key table with fixed bucket. +- non-primary-key table with bucket -1. ## Trino to Paimon type mapping @@ -319,3 +296,15 @@ All Trino's data types are available in package `io.trino.spi.type`.
+ +## Tmp Dir + +Paimon will unzip some jars to the tmp directory for codegen. By default, Trino will use `'/tmp'` as the temporary +directory, but `'/tmp'` may be periodically deleted. + +You can configure this environment variable when Trino starts: +```shell +-Djava.io.tmpdir=/path/to/other/tmpdir +``` + +Let Paimon use a secure temporary directory. diff --git a/docs/content/filesystems/hdfs.md b/docs/content/filesystems/hdfs.md deleted file mode 100644 index ace26d1a84a0..000000000000 --- a/docs/content/filesystems/hdfs.md +++ /dev/null @@ -1,129 +0,0 @@ ---- -title: "HDFS" -weight: 2 -type: docs -aliases: -- /filesystems/hdfs.html ---- - - -# HDFS - -You don't need any additional dependencies to access HDFS because you have already taken care of the Hadoop dependencies. - -## HDFS Configuration - -For HDFS, the most important thing is to be able to read your HDFS configuration. - -{{< tabs "hdfs conf" >}} - -{{< tab "Flink/Trino/JavaAPI" >}} - -You may not have to do anything, if you are in a hadoop environment. Otherwise pick one of the following ways to -configure your HDFS: - -1. Set environment variable `HADOOP_HOME` or `HADOOP_CONF_DIR`. -2. Configure `'hadoop-conf-dir'` in the paimon catalog. -3. Configure Hadoop options through prefix `'hadoop.'` in the paimon catalog. - -The first approach is recommended. - -If you do not want to include the value of the environment variable, you can configure `hadoop-conf-loader` to `option`. - -{{< /tab >}} - -{{< tab "Hive/Spark" >}} - -HDFS Configuration is available directly through the computation cluster, see cluster configuration of Hive and Spark for details. - -{{< /tab >}} - -{{< /tabs >}} - -## Hadoop-compatible file systems (HCFS) - -All Hadoop file systems are automatically available when the Hadoop libraries are on the classpath. - -This way, Paimon seamlessly supports all of Hadoop file systems implementing the `org.apache.hadoop.fs.FileSystem` -interface, and all Hadoop-compatible file systems (HCFS). - -- HDFS -- Alluxio (see configuration specifics below) -- XtreemFS -- … - -The Hadoop configuration has to have an entry for the required file system implementation in the `core-site.xml` file. - -For Alluxio support add the following entry into the core-site.xml file: - -```shell - - fs.alluxio.impl - alluxio.hadoop.FileSystem - -``` - -## Kerberos - -{{< tabs "Kerberos" >}} - -{{< tab "Flink" >}} - -It is recommended to use [Flink Kerberos Keytab](https://nightlies.apache.org/flink/flink-docs-stable/docs/deployment/security/security-kerberos/). - -{{< /tab >}} - -{{< tab "Spark" >}} - -It is recommended to use [Spark Kerberos Keytab](https://spark.apache.org/docs/latest/security.html#using-a-keytab). - -{{< /tab >}} - -{{< tab "Hive" >}} - -An intuitive approach is to configure Hive's kerberos authentication. - -{{< /tab >}} - -{{< tab "Trino/JavaAPI" >}} - -Configure the following three options in your catalog configuration: - -- security.kerberos.login.keytab: Absolute path to a Kerberos keytab file that contains the user credentials. - Please make sure it is copied to each machine. -- security.kerberos.login.principal: Kerberos principal name associated with the keytab. -- security.kerberos.login.use-ticket-cache: True or false, indicates whether to read from your Kerberos ticket cache. - -For JavaAPI: -``` -SecurityContext.install(catalogOptions); -``` - -{{< /tab >}} - -{{< /tabs >}} - -## HDFS HA - -Ensure that `hdfs-site.xml` and `core-site.xml` contain the necessary [HA configuration](https://hadoop.apache.org/docs/stable/hadoop-project-dist/hadoop-hdfs/HDFSHighAvailabilityWithNFS.html). - -## HDFS ViewFS - -Ensure that `hdfs-site.xml` and `core-site.xml` contain the necessary [ViewFs configuration](https://hadoop.apache.org/docs/stable/hadoop-project-dist/hadoop-hdfs/ViewFs.html). diff --git a/docs/content/filesystems/oss.md b/docs/content/filesystems/oss.md deleted file mode 100644 index b381350a5c9d..000000000000 --- a/docs/content/filesystems/oss.md +++ /dev/null @@ -1,120 +0,0 @@ ---- -title: "OSS" -weight: 3 -type: docs -aliases: -- /filesystems/oss.html ---- - - -# OSS - -{{< stable >}} - -Download [paimon-oss-{{< version >}}.jar](https://repo.maven.apache.org/maven2/org/apache/paimon/paimon-oss/{{< version >}}/paimon-oss-{{< version >}}.jar). - -{{< /stable >}} - -{{< unstable >}} - -Download [paimon-oss-{{< version >}}.jar](https://repository.apache.org/snapshots/org/apache/paimon/paimon-oss/{{< version >}}/). - -{{< /unstable >}} - -{{< tabs "oss" >}} - -{{< tab "Flink" >}} - -{{< hint info >}} -If you have already configured [oss access through Flink](https://nightlies.apache.org/flink/flink-docs-stable/docs/deployment/filesystems/oss/) (Via Flink FileSystem), -here you can skip the following configuration. -{{< /hint >}} - -Put `paimon-oss-{{< version >}}.jar` into `lib` directory of your Flink home, and create catalog: - -```sql -CREATE CATALOG my_catalog WITH ( - 'type' = 'paimon', - 'warehouse' = 'oss:///', - 'fs.oss.endpoint' = 'oss-cn-hangzhou.aliyuncs.com', - 'fs.oss.accessKeyId' = 'xxx', - 'fs.oss.accessKeySecret' = 'yyy' -); -``` - -{{< /tab >}} - -{{< tab "Spark" >}} - -{{< hint info >}} -If you have already configured oss access through Spark (Via Hadoop FileSystem), here you can skip the following configuration. -{{< /hint >}} - -Place `paimon-oss-{{< version >}}.jar` together with `paimon-spark-{{< version >}}.jar` under Spark's jars directory, and start like - -```shell -spark-sql \ - --conf spark.sql.catalog.paimon=org.apache.paimon.spark.SparkCatalog \ - --conf spark.sql.catalog.paimon.warehouse=oss:/// \ - --conf spark.sql.catalog.paimon.fs.oss.endpoint=oss-cn-hangzhou.aliyuncs.com \ - --conf spark.sql.catalog.paimon.fs.oss.accessKeyId=xxx \ - --conf spark.sql.catalog.paimon.fs.oss.accessKeySecret=yyy -``` - -{{< /tab >}} - -{{< tab "Hive" >}} - -{{< hint info >}} -If you have already configured oss access through Hive (Via Hadoop FileSystem), here you can skip the following configuration. -{{< /hint >}} - -NOTE: You need to ensure that Hive metastore can access `oss`. - -Place `paimon-oss-{{< version >}}.jar` together with `paimon-hive-connector-{{< version >}}.jar` under Hive's auxlib directory, and start like - -```sql -SET paimon.fs.oss.endpoint=oss-cn-hangzhou.aliyuncs.com; -SET paimon.fs.oss.accessKeyId=xxx; -SET paimon.fs.oss.accessKeySecret=yyy; -``` - -And read table from hive metastore, table can be created by Flink or Spark, see [Catalog with Hive Metastore]({{< ref "flink/sql-ddl" >}}) -```sql -SELECT * FROM test_table; -SELECT COUNT(1) FROM test_table; -``` - -{{< /tab >}} - -{{< tab "Trino" >}} - -From version 0.8, paimon-trino uses trino filesystem as basic file read and write system. We strongly recommend you to use jindo-sdk in trino. - -You can find [How to config jindo sdk on trino](https://github.com/aliyun/alibabacloud-jindodata/blob/master/docs/user/4.x/4.6.x/4.6.12/oss/presto/jindosdk_on_presto.md) here. -Please note that: - * Use paimon to replace hive-hadoop2 when you decompress the plugin jar and find location to put in. - * You can specify the `core-site.xml` in `paimon.properties` on configuration [hive.config.resources](https://trino.io/docs/current/connector/hive.html#hdfs-configuration). - * Presto and Jindo use the same configuration method. - - -{{< /tab >}} - -{{< /tabs >}} diff --git a/docs/content/filesystems/overview.md b/docs/content/filesystems/overview.md deleted file mode 100644 index 3de3c2ec500a..000000000000 --- a/docs/content/filesystems/overview.md +++ /dev/null @@ -1,60 +0,0 @@ ---- -title: "Overview" -weight: 1 -type: docs -aliases: -- /filesystems/overview.html ---- - - -# Overview - -Apache Paimon utilizes the same pluggable file systems as Apache Flink. Users can follow the -[standard plugin mechanism](https://nightlies.apache.org/flink/flink-docs-stable/docs/deployment/filesystems/plugins/) -to configure the plugin structure if using Flink as compute engine. However, for other engines like Spark -or Hive, the provided opt jars (by Flink) may get conflicts and cannot be used directly. It is not convenient -for users to fix class conflicts, thus Paimon provides the self-contained and engine-unified -FileSystem pluggable jars for user to query tables from Spark/Hive side. - -## Supported FileSystems - -| FileSystem | URI Scheme | Pluggable | Description | -|:------------------|:-----------|-----------|:-----------------------------------------------------------------------| -| Local File System | file:// | N | Built-in Support | -| HDFS | hdfs:// | N | Built-in Support, ensure that the cluster is in the hadoop environment | -| Aliyun OSS | oss:// | Y | | -| S3 | s3:// | Y | | - -## Dependency - -We recommend you to download the jar directly: [Download Link]({{< ref "project/download#filesystem-jars" >}}). - -You can also manually build bundled jar from the source code. - -To build from source code, [clone the git repository]({{< github_repo >}}). - -Build shaded jar with the following command. - -```bash -mvn clean install -DskipTests -``` - -You can find the shaded jars under -`./paimon-filesystems/paimon-${fs}/target/paimon-${fs}-{{< version >}}.jar`. diff --git a/docs/content/filesystems/s3.md b/docs/content/filesystems/s3.md deleted file mode 100644 index 3085d820b67e..000000000000 --- a/docs/content/filesystems/s3.md +++ /dev/null @@ -1,143 +0,0 @@ ---- -title: "S3" -weight: 4 -type: docs -aliases: -- /filesystems/s3.html ---- - - -# S3 - -{{< stable >}} - -Download [paimon-s3-{{< version >}}.jar](https://repo.maven.apache.org/maven2/org/apache/paimon/paimon-s3/{{< version >}}/paimon-s3-{{< version >}}.jar). - -{{< /stable >}} - -{{< unstable >}} - -Download [paimon-s3-{{< version >}}.jar](https://repository.apache.org/snapshots/org/apache/paimon/paimon-s3/{{< version >}}/). - -{{< /unstable >}} - -{{< tabs "oss" >}} - -{{< tab "Flink" >}} - -{{< hint info >}} -If you have already configured [s3 access through Flink](https://nightlies.apache.org/flink/flink-docs-stable/docs/deployment/filesystems/s3/) (Via Flink FileSystem), -here you can skip the following configuration. -{{< /hint >}} - -Put `paimon-s3-{{< version >}}.jar` into `lib` directory of your Flink home, and create catalog: - -```sql -CREATE CATALOG my_catalog WITH ( - 'type' = 'paimon', - 'warehouse' = 's3:///', - 's3.endpoint' = 'your-endpoint-hostname', - 's3.access-key' = 'xxx', - 's3.secret-key' = 'yyy' -); -``` - -{{< /tab >}} - -{{< tab "Spark" >}} - -{{< hint info >}} -If you have already configured s3 access through Spark (Via Hadoop FileSystem), here you can skip the following configuration. -{{< /hint >}} - -Place `paimon-s3-{{< version >}}.jar` together with `paimon-spark-{{< version >}}.jar` under Spark's jars directory, and start like - -```shell -spark-sql \ - --conf spark.sql.catalog.paimon=org.apache.paimon.spark.SparkCatalog \ - --conf spark.sql.catalog.paimon.warehouse=s3:/// \ - --conf spark.sql.catalog.paimon.s3.endpoint=your-endpoint-hostname \ - --conf spark.sql.catalog.paimon.s3.access-key=xxx \ - --conf spark.sql.catalog.paimon.s3.secret-key=yyy -``` - -{{< /tab >}} - -{{< tab "Hive" >}} - -{{< hint info >}} -If you have already configured s3 access through Hive ((Via Hadoop FileSystem)), here you can skip the following configuration. -{{< /hint >}} - -NOTE: You need to ensure that Hive metastore can access `s3`. - -Place `paimon-s3-{{< version >}}.jar` together with `paimon-hive-connector-{{< version >}}.jar` under Hive's auxlib directory, and start like - -```sql -SET paimon.s3.endpoint=your-endpoint-hostname; -SET paimon.s3.access-key=xxx; -SET paimon.s3.secret-key=yyy; -``` - -And read table from hive metastore, table can be created by Flink or Spark, see [Catalog with Hive Metastore]({{< ref "flink/sql-ddl" >}}) -```sql -SELECT * FROM test_table; -SELECT COUNT(1) FROM test_table; -``` - -{{< /tab >}} - -{{< tab "Trino" >}} - -Paimon use shared trino filesystem as basic read and write system. - -Please refer to [Trino S3](https://trino.io/docs/current/object-storage/file-system-s3.html) to config s3 filesystem in trino. - -{{< /tab >}} - -{{< /tabs >}} - -## S3 Complaint Object Stores - -The S3 Filesystem also support using S3 compliant object stores such as MinIO, Tencent's COS and IBM’s Cloud Object -Storage. Just configure your endpoint to the provider of the object store service. - -```yaml -s3.endpoint: your-endpoint-hostname -``` - -## Configure Path Style Access - -Some S3 compliant object stores might not have virtual host style addressing enabled by default, for example when using Standalone MinIO for testing purpose. -In such cases, you will have to provide the property to enable path style access. - -```yaml -s3.path.style.access: true -``` - -## S3A Performance - -[Tune Performance](https://hadoop.apache.org/docs/stable/hadoop-aws/tools/hadoop-aws/performance.html) for `S3AFileSystem`. - -If you encounter the following exception: -```shell -Caused by: org.apache.http.conn.ConnectionPoolTimeoutException: Timeout waiting for connection from pool. -``` -Try to configure this in catalog options: `fs.s3a.connection.maximum=1000`. diff --git a/docs/content/flink/_index.md b/docs/content/flink/_index.md index c39ff01d8760..6ec757fa520f 100644 --- a/docs/content/flink/_index.md +++ b/docs/content/flink/_index.md @@ -3,7 +3,7 @@ title: Engine Flink icon: bold: true bookCollapseSection: true -weight: 4 +weight: 5 --- + +# Consumer ID + +Consumer id can help you accomplish the following two things: + +1. Safe consumption: When deciding whether a snapshot has expired, Paimon looks at all the consumers of the table in + the file system, and if there are consumers that still depend on this snapshot, then this snapshot will not be + deleted by expiration. +2. Resume from breakpoint: When previous job is stopped, the newly started job can continue to consume from the previous + progress without resuming from the state. + +## Usage + +You can specify the `consumer-id` when streaming read table. + +The consumer will prevent expiration of the snapshot. In order to prevent too many snapshots caused by mistakes, +you need to specify `'consumer.expiration-time'` to manage the lifetime of consumers. + +```sql +ALTER TABLE t SET ('consumer.expiration-time' = '1 d'); +``` + +Then, restart streaming write job of this table, expiration of consumers will be triggered in writing job. + +```sql +SELECT * FROM t /*+ OPTIONS('consumer-id' = 'myid', 'consumer.mode' = 'at-least-once') */; +``` + +## Ignore Progress + +Sometimes, you only want the feature of 'Safe Consumption'. You want to get a new snapshot progress when restarting the +stream consumption job , you can enable the `'consumer.ignore-progress'` option. + +```sql +SELECT * FROM t /*+ OPTIONS('consumer-id' = 'myid', 'consumer.ignore-progress' = 'true') */; +``` + +The startup of this job will retrieve the snapshot that should be read again. + +## Consumer Mode + +By default, the consumption of snapshots is strictly aligned within the checkpoint to make 'Resume from breakpoint' +feature exactly-once. + +But in some scenarios where you don't need 'Resume from breakpoint', or you don't need strict 'Resume from breakpoint', +you can consider enabling `'consumer.mode' = 'at-least-once'` mode. This mode: +1. Allow readers consume snapshots at different rates and record the slowest snapshot-id among all readers into the + consumer. It doesn't affect the checkpoint time and have good performance. +2. This mode can provide more capabilities, such as watermark alignment. + +{{< hint >}} +About `'consumer.mode'`, since the implementation of `exactly-once` mode and `at-least-once` mode are completely +different, the state of flink is incompatible and cannot be restored from the state when switching modes. +{{< /hint >}} + +## Reset Consumer + +You can reset or delete a consumer with a given consumer ID and next snapshot ID and delete a consumer with a given +consumer ID. First, you need to stop the streaming task using this consumer ID, and then execute the reset consumer +action job. + +Run the following command: + +{{< tabs "reset_consumer" >}} + +{{< tab "Flink SQL" >}} + +```sql +CALL sys.reset_consumer( + `table` => 'database_name.table_name', + consumer_id => 'consumer_id', + next_snapshot_id => +); +-- No next_snapshot_id if you want to delete the consumer +``` +{{< /tab >}} + +{{< tab "Flink Action" >}} + +```bash +/bin/flink run \ + /path/to/paimon-flink-action-{{< version >}}.jar \ + reset-consumer \ + --warehouse \ + --database \ + --table \ + --consumer_id \ + [--next_snapshot ] \ + [--catalog_conf [--catalog_conf ...]] + +## No next_snapshot if you want to delete the consumer +``` +{{< /tab >}} + +{{< /tabs >}} diff --git a/docs/content/flink/expire-partition.md b/docs/content/flink/expire-partition.md index 3acf6e59d58c..226017513fee 100644 --- a/docs/content/flink/expire-partition.md +++ b/docs/content/flink/expire-partition.md @@ -134,7 +134,7 @@ More options:
end-input.check-partition-expire
false Boolean - Whether check partition expire after batch mode or bounded stream job finish. + Whether check partition expire after batch mode or bounded stream job finish. diff --git a/docs/content/flink/procedures.md b/docs/content/flink/procedures.md index ce8c8043ae40..8eb1786a08b3 100644 --- a/docs/content/flink/procedures.md +++ b/docs/content/flink/procedures.md @@ -67,14 +67,17 @@ All available procedures are listed below. order_by => 'order_by', options => 'options', `where` => 'where', - partition_idle_time => 'partition_idle_time')

+ partition_idle_time => 'partition_idle_time', + compact_strategy => 'compact_strategy')

-- Use indexed argument
CALL [catalog.]sys.compact('table')

CALL [catalog.]sys.compact('table', 'partitions')

+ CALL [catalog.]sys.compact('table', 'order_strategy', 'order_by')

CALL [catalog.]sys.compact('table', 'partitions', 'order_strategy', 'order_by')

CALL [catalog.]sys.compact('table', 'partitions', 'order_strategy', 'order_by', 'options')

CALL [catalog.]sys.compact('table', 'partitions', 'order_strategy', 'order_by', 'options', 'where')

CALL [catalog.]sys.compact('table', 'partitions', 'order_strategy', 'order_by', 'options', 'where', 'partition_idle_time')

+ CALL [catalog.]sys.compact('table', 'partitions', 'order_strategy', 'order_by', 'options', 'where', 'partition_idle_time', 'compact_strategy')

To compact a table. Arguments: @@ -85,6 +88,7 @@ All available procedures are listed below.
  • options(optional): additional dynamic options of the table.
  • where(optional): partition predicate(Can't be used together with "partitions"). Note: as where is a keyword,a pair of backticks need to add around like `where`.
  • partition_idle_time(optional): this is used to do a full compaction for partition which had not received any new data for 'partition_idle_time'. And only these partitions will be compacted. This argument can not be used with order compact.
    • +
  • compact_strategy(optional): this determines how to pick files to be merged, the default is determined by the runtime execution mode. 'full' strategy only supports batch mode. All files will be selected for merging. 'minor' strategy: Pick the set of files that need to be merged based on specified conditions.
    • -- use partition filter
    @@ -103,7 +107,8 @@ All available procedures are listed below. including_tables => 'includingTables', excluding_tables => 'excludingTables', table_options => 'tableOptions', - partition_idle_time => 'partitionIdleTime')

    + partition_idle_time => 'partitionIdleTime', + compact_strategy => 'compact_strategy')

    -- Use indexed argument
    CALL [catalog.]sys.compact_database()

    CALL [catalog.]sys.compact_database('includingDatabases')

    @@ -111,7 +116,8 @@ All available procedures are listed below. CALL [catalog.]sys.compact_database('includingDatabases', 'mode', 'includingTables')

    CALL [catalog.]sys.compact_database('includingDatabases', 'mode', 'includingTables', 'excludingTables')

    CALL [catalog.]sys.compact_database('includingDatabases', 'mode', 'includingTables', 'excludingTables', 'tableOptions')

    - CALL [catalog.]sys.compact_database('includingDatabases', 'mode', 'includingTables', 'excludingTables', 'tableOptions', 'partitionIdleTime') + CALL [catalog.]sys.compact_database('includingDatabases', 'mode', 'includingTables', 'excludingTables', 'tableOptions', 'partitionIdleTime')

    + CALL [catalog.]sys.compact_database('includingDatabases', 'mode', 'includingTables', 'excludingTables', 'tableOptions', 'partitionIdleTime', 'compact_strategy')

    To compact databases. Arguments: @@ -123,6 +129,7 @@ All available procedures are listed below.
  • excludingTables: to specify tables that are not compacted. You can use regular expression.
  • tableOptions: additional dynamic options of the table.
  • partition_idle_time: this is used to do a full compaction for partition which had not received any new data for 'partition_idle_time'. And only these partitions will be compacted.
    • +
  • compact_strategy(optional): this determines how to pick files to be merged, the default is determined by the runtime execution mode. 'full' strategy only supports batch mode. All files will be selected for merging. 'minor' strategy: Pick the set of files that need to be merged based on specified conditions.
    • CALL sys.compact_database( @@ -130,7 +137,8 @@ All available procedures are listed below. mode => 'combined', including_tables => 'table_.*', excluding_tables => 'ignore', - table_options => 'sink.parallelism=4') + table_options => 'sink.parallelism=4', + compat_strategy => 'full') @@ -221,6 +229,46 @@ All available procedures are listed below. CALL sys.delete_tag(`table` => 'default.T', tag => 'my_tag') + + replace_tag + + -- Use named argument
    + -- replace tag with new time retained
    + CALL [catalog.]sys.replace_tag(`table` => 'identifier', tag => 'tagName', time_retained => 'timeRetained')
    + -- replace tag with new snapshot id and time retained
    + CALL [catalog.]sys.replace_tag(`table` => 'identifier', snapshot_id => 'snapshotId')

    + -- Use indexed argument
    + -- replace tag with new snapshot id and time retained
    + CALL [catalog.]sys.replace_tag('identifier', 'tagName', 'snapshotId', 'timeRetained')
    + + + To replace an existing tag with new tag info. Arguments: +
  • table: the target table identifier. Cannot be empty.
    • +
  • tag: name of the existed tag. Cannot be empty.
    • +
  • snapshot(Long): id of the snapshot which the tag is based on, it is optional.
    • +
  • time_retained: The maximum time retained for the existing tag, it is optional.
    • + + + -- for Flink 1.18
    + CALL sys.replace_tag('default.T', 'my_tag', 5, '1 d')

    + -- for Flink 1.19 and later
    + CALL sys.replace_tag(`table` => 'default.T', tag => 'my_tag', snapshot_id => 5, time_retained => '1 d')

    + + + + expire_tags + + CALL [catalog.]sys.expire_tags('identifier', 'older_than') + + + To expire tags by time. Arguments: +
  • identifier: the target table identifier. Cannot be empty.
    • +
  • older_than: tagCreateTime before which tags will be removed.
    • + + + CALL sys.expire_tags(table => 'default.T', older_than => '2024-09-06 11:00:00') + + merge_into @@ -241,10 +289,13 @@ All available procedures are listed below. matched_upsert_setting => 'matchedUpsertSetting',
    not_matched_insert_condition => 'notMatchedInsertCondition',
    not_matched_insert_values => 'notMatchedInsertValues',
    - matched_delete_condition => 'matchedDeleteCondition')

    + matched_delete_condition => 'matchedDeleteCondition',
    + not_matched_by_source_upsert_condition => 'notMatchedBySourceUpsertCondition',
    + not_matched_by_source_upsert_setting => 'notMatchedBySourceUpsertSetting',
    + not_matched_by_source_delete_condition => 'notMatchedBySourceDeleteCondition')

    - To perform "MERGE INTO" syntax. See merge_into action for + To perform "MERGE INTO" syntax. See merge_into action for details of arguments. @@ -339,11 +390,76 @@ All available procedures are listed below. CALL sys.rollback_to(`table` => 'default.T', snapshot_id => 10) + + rollback_to_timestamp + + -- for Flink 1.18
    + -- rollback to the snapshot which earlier or equal than timestamp.
    + CALL sys.rollback_to_timestamp('identifier', timestamp)

    + -- for Flink 1.19 and later
    + -- rollback to the snapshot which earlier or equal than timestamp.
    + CALL sys.rollback_to_timestamp(`table` => 'default.T', `timestamp` => timestamp)

    + + + To rollback to the snapshot which earlier or equal than timestamp. Argument: +
  • identifier: the target table identifier. Cannot be empty.
    • +
  • timestamp (Long): Roll back to the snapshot which earlier or equal than timestamp.
    • + + + -- for Flink 1.18
    + CALL sys.rollback_to_timestamp('default.T', 10) + -- for Flink 1.19 and later
    + CALL sys.rollback_to_timestamp(`table` => 'default.T', timestamp => 1730292023000) + + + + rollback_to_watermark + + -- for Flink 1.18
    + -- rollback to the snapshot which earlier or equal than watermark.
    + CALL sys.rollback_to_watermark('identifier', watermark)

    + -- for Flink 1.19 and later
    + -- rollback to the snapshot which earlier or equal than watermark.
    + CALL sys.rollback_to_watermark(`table` => 'default.T', `watermark` => watermark)

    + + + To rollback to the snapshot which earlier or equal than watermark. Argument: +
  • identifier: the target table identifier. Cannot be empty.
    • +
  • watermark (Long): Roll back to the snapshot which earlier or equal than watermark.
    • + + + -- for Flink 1.18
    + CALL sys.rollback_to_watermark('default.T', 1730292023000) + -- for Flink 1.19 and later
    + CALL sys.rollback_to_watermark(`table` => 'default.T', watermark => 1730292023000) + + + + purge_files + + -- for Flink 1.18
    + -- clear table with purge files directly.
    + CALL sys.purge_files('identifier')

    + -- for Flink 1.19 and later
    + -- clear table with purge files directly.
    + CALL sys.purge_files(`table` => 'default.T')

    + + + To clear table with purge files directly. Argument: +
  • identifier: the target table identifier. Cannot be empty.
    • + + + -- for Flink 1.18
    + CALL sys.purge_files('default.T') + -- for Flink 1.19 and later
    + CALL sys.purge_files(`table` => 'default.T') + + expire_snapshots -- Use named argument
    - CALL [catalog.]sys.reset_consumer(
    + CALL [catalog.]sys.expire_snapshots(
    `table` => 'identifier',
    retain_max => 'retain_max',
    retain_min => 'retain_min',
    diff --git a/docs/content/flink/quick-start.md b/docs/content/flink/quick-start.md index 62559065ec9a..e50acfe484e1 100644 --- a/docs/content/flink/quick-start.md +++ b/docs/content/flink/quick-start.md @@ -269,11 +269,16 @@ SELECT * FROM ....; ## Setting dynamic options When interacting with the Paimon table, table options can be tuned without changing the options in the catalog. Paimon will extract job-level dynamic options and take effect in the current session. -The dynamic option's key format is `paimon.${catalogName}.${dbName}.${tableName}.${config_key}`. The catalogName/dbName/tableName can be `*`, which means matching all the specific parts. +The dynamic table option's key format is `paimon.${catalogName}.${dbName}.${tableName}.${config_key}`. The catalogName/dbName/tableName can be `*`, which means matching all the specific parts. +The dynamic global option's key format is `${config_key}`. Global options will take effect for all the tables. Table options will override global options if there are conflicts. For example: ```sql +-- set scan.timestamp-millis=1697018249001 for all tables +SET 'scan.timestamp-millis' = '1697018249001'; +SELECT * FROM T; + -- set scan.timestamp-millis=1697018249000 for the table mycatalog.default.T SET 'paimon.mycatalog.default.T.scan.timestamp-millis' = '1697018249000'; SELECT * FROM T; @@ -281,4 +286,10 @@ SELECT * FROM T; -- set scan.timestamp-millis=1697018249000 for the table default.T in any catalog SET 'paimon.*.default.T.scan.timestamp-millis' = '1697018249000'; SELECT * FROM T; + +-- set scan.timestamp-millis=1697018249000 for the table mycatalog.default.T1 +-- set scan.timestamp-millis=1697018249001 for others tables +SET 'paimon.mycatalog.default.T1.scan.timestamp-millis' = '1697018249000'; +SET 'scan.timestamp-millis' = '1697018249001'; +SELECT * FROM T1 JOIN T2 ON xxxx; ``` diff --git a/docs/content/flink/savepoint.md b/docs/content/flink/savepoint.md index 16139f0b0fc8..b9d353c1de33 100644 --- a/docs/content/flink/savepoint.md +++ b/docs/content/flink/savepoint.md @@ -41,12 +41,12 @@ metadata left. This is very safe, so we recommend using this feature to stop and ## Tag with Savepoint -In Flink, we may consume from kafka and then write to paimon. Since flink's checkpoint only retains a limited number, +In Flink, we may consume from Kafka and then write to Paimon. Since Flink's checkpoint only retains a limited number, we will trigger a savepoint at certain time (such as code upgrades, data updates, etc.) to ensure that the state can be retained for a longer time, so that the job can be restored incrementally. -Paimon's snapshot is similar to flink's checkpoint, and both will automatically expire, but the tag feature of paimon -allows snapshots to be retained for a long time. Therefore, we can combine the two features of paimon's tag and flink's +Paimon's snapshot is similar to Flink's checkpoint, and both will automatically expire, but the tag feature of Paimon +allows snapshots to be retained for a long time. Therefore, we can combine the two features of Paimon's tag and Flink's savepoint to achieve incremental recovery of job from the specified savepoint. {{< hint warning >}} @@ -64,17 +64,17 @@ You can set `sink.savepoint.auto-tag` to `true` to enable the feature of automat **Step 2: Trigger savepoint.** -You can refer to [flink savepoint](https://nightlies.apache.org/flink/flink-docs-stable/docs/ops/state/savepoints/#operations) +You can refer to [Flink savepoint](https://nightlies.apache.org/flink/flink-docs-stable/docs/ops/state/savepoints/#operations) to learn how to configure and trigger savepoint. **Step 3: Choose the tag corresponding to the savepoint.** The tag corresponding to the savepoint will be named in the form of `savepoint-${savepointID}`. You can refer to -[Tags Table]({{< ref "maintenance/system-tables#tags-table" >}}) to query. +[Tags Table]({{< ref "concepts/system-tables#tags-table" >}}) to query. **Step 4: Rollback the paimon table.** -[Rollback]({{< ref "maintenance/manage-tags#rollback-to-tag" >}}) the paimon table to the specified tag. +[Rollback]({{< ref "maintenance/manage-tags#rollback-to-tag" >}}) the Paimon table to the specified tag. **Step 5: Restart from the savepoint.** diff --git a/docs/content/flink/sql-alter.md b/docs/content/flink/sql-alter.md index fe96ec413796..877995cc631b 100644 --- a/docs/content/flink/sql-alter.md +++ b/docs/content/flink/sql-alter.md @@ -1,6 +1,6 @@ --- title: "SQL Alter" -weight: 6 +weight: 7 type: docs aliases: - /flink/sql-alter.html @@ -78,6 +78,10 @@ If you use object storage, such as S3 or OSS, please use this syntax carefully, The following SQL adds two columns `c1` and `c2` to table `my_table`. +{{< hint info >}} +To add a column in a row type, see [Changing Column Type](#changing-column-type). +{{< /hint >}} + ```sql ALTER TABLE my_table ADD (c1 INT, c2 STRING); ``` @@ -99,6 +103,10 @@ otherwise this operation may fail, throws an exception like `The following colum ALTER TABLE my_table DROP (c1, c2); ``` +{{< hint info >}} +To drop a column in a row type, see [Changing Column Type](#changing-column-type). +{{< /hint >}} + ## Dropping Partitions The following SQL drops the partitions of the paimon table. @@ -114,6 +122,21 @@ ALTER TABLE my_table DROP PARTITION (`id` = 1), PARTITION (`id` = 2); ``` +## Adding Partitions + +The following SQL adds the partitions of the paimon table. + +For flink sql, you can specify the partial columns of partition columns, and you can also specify multiple partition values at the same time, only with metastore configured metastore.partitioned-table=true. + +```sql +ALTER TABLE my_table ADD PARTITION (`id` = 1); + +ALTER TABLE my_table ADD PARTITION (`id` = 1, `name` = 'paimon'); + +ALTER TABLE my_table ADD PARTITION (`id` = 1), PARTITION (`id` = 2); + +``` + ## Changing Column Nullability The following SQL changes nullability of column `coupon_info`. @@ -170,6 +193,14 @@ The following SQL changes type of column `col_a` to `DOUBLE`. ALTER TABLE my_table MODIFY col_a DOUBLE; ``` +Paimon also supports changing columns of row type, array type, and map type. + +```sql +-- col_a previously has type ARRAY> +-- the following SQL changes f1 to BIGINT, drops f2, and adds f3 +ALTER TABLE my_table MODIFY col_a ARRAY>; +``` + ## Adding watermark The following SQL adds a computed column `ts` from existing column `log_ts`, and a watermark with strategy `ts - INTERVAL '1' HOUR` on column `ts` which is marked as event time attribute of table `my_table`. diff --git a/docs/content/flink/sql-ddl.md b/docs/content/flink/sql-ddl.md index 363d7475761c..a373348861bd 100644 --- a/docs/content/flink/sql-ddl.md +++ b/docs/content/flink/sql-ddl.md @@ -101,7 +101,7 @@ Also, you can create [FlinkGenericCatalog]({{< ref "flink/quick-start" >}}). By default, Paimon does not synchronize newly created partitions into Hive metastore. Users will see an unpartitioned table in Hive. Partition push-down will be carried out by filter push-down instead. -If you want to see a partitioned table in Hive and also synchronize newly created partitions into Hive metastore, please set the table property `metastore.partitioned-table` to true. Also see [CoreOptions]({{< ref "maintenance/configurations#CoreOptions" >}}). +If you want to see a partitioned table in Hive and also synchronize newly created partitions into Hive metastore, please set the table property `metastore.partitioned-table` to true. Also see [CoreOptions]({{< ref "maintenance/configurations#coreoptions" >}}). #### Adding Parameters to a Hive Table @@ -114,7 +114,7 @@ For instance, using the option `hive.table.owner=Jon` will automatically add the If you are using an object storage , and you don't want that the location of paimon table/database is accessed by the filesystem of hive, which may lead to the error such as "No FileSystem for scheme: s3a". You can set location in the properties of table/database by the config of `location-in-properties`. See -[setting the location of table/database in properties ]({{< ref "maintenance/configurations#HiveCatalogOptions" >}}) +[setting the location of table/database in properties ]({{< ref "maintenance/configurations#hivecatalogoptions" >}}) ### Creating JDBC Catalog @@ -203,6 +203,9 @@ Paimon will automatically collect the statistics of the data file for speeding u The statistics collector mode can be configured by `'metadata.stats-mode'`, by default is `'truncate(16)'`. You can configure the field level by setting `'fields.{field_name}.stats-mode'`. +For the stats mode of `none`, by default `metadata.stats-dense-store` is `true`, which will significantly reduce the +storage size of the manifest. But the Paimon sdk in reading engine requires at least version 0.9.1 or 1.0.0 or higher. + ### Field Default Value Paimon table currently supports setting default values for fields in table properties by `'fields.item_id.default-value'`, diff --git a/docs/content/flink/sql-lookup.md b/docs/content/flink/sql-lookup.md index 296b9c862eee..8b95c6cb1b15 100644 --- a/docs/content/flink/sql-lookup.md +++ b/docs/content/flink/sql-lookup.md @@ -1,6 +1,6 @@ --- title: "SQL Lookup" -weight: 5 +weight: 6 type: docs aliases: - /flink/sql-lookup.html diff --git a/docs/content/flink/sql-query.md b/docs/content/flink/sql-query.md index bb26d5d3c66a..89136b0b0635 100644 --- a/docs/content/flink/sql-query.md +++ b/docs/content/flink/sql-query.md @@ -172,70 +172,6 @@ prevent you from reading older incremental data. So, Paimon also provides anothe SELECT * FROM t /*+ OPTIONS('scan.file-creation-time-millis' = '1678883047356') */; ``` -### Consumer ID - -You can specify the `consumer-id` when streaming read table: -```sql -SELECT * FROM t /*+ OPTIONS('consumer-id' = 'myid', 'consumer.expiration-time' = '1 d', 'consumer.mode' = 'at-least-once') */; -``` - -When stream read Paimon tables, the next snapshot id to be recorded into the file system. This has several advantages: - -1. When previous job is stopped, the newly started job can continue to consume from the previous progress without - resuming from the state. The newly reading will start reading from next snapshot id found in consumer files. - If you don't want this behavior, you can set `'consumer.ignore-progress'` to true. -2. When deciding whether a snapshot has expired, Paimon looks at all the consumers of the table in the file system, - and if there are consumers that still depend on this snapshot, then this snapshot will not be deleted by expiration. - -{{< hint warning >}} -NOTE 1: The consumer will prevent expiration of the snapshot. You can specify `'consumer.expiration-time'` to manage the -lifetime of consumers. - -NOTE 2: If you don't want to affect the checkpoint time, you need to configure `'consumer.mode' = 'at-least-once'`. -This mode allow readers consume snapshots at different rates and record the slowest snapshot-id among all readers into -the consumer. This mode can provide more capabilities, such as watermark alignment. - -NOTE 3: About `'consumer.mode'`, since the implementation of `exactly-once` mode and `at-least-once` mode are completely -different, the state of flink is incompatible and cannot be restored from the state when switching modes. -{{< /hint >}} - -You can reset a consumer with a given consumer ID and next snapshot ID and delete a consumer with a given consumer ID. -First, you need to stop the streaming task using this consumer ID, and then execute the reset consumer action job. - -Run the following command: - -{{< tabs "reset_consumer" >}} - -{{< tab "Flink SQL" >}} - -```sql -CALL sys.reset_consumer( - `table` => 'database_name.table_name', - consumer_id => 'consumer_id', - next_snapshot_id -> -); -``` -{{< /tab >}} - -{{< tab "Flink Action" >}} - -```bash -/bin/flink run \ - /path/to/paimon-flink-action-{{< version >}}.jar \ - reset-consumer \ - --warehouse \ - --database \ - --table \ - --consumer_id \ - [--next_snapshot ] \ - [--catalog_conf [--catalog_conf ...]] -``` -{{< /tab >}} - -{{< /tabs >}} - -please don't specify --next_snapshot parameter if you want to delete the consumer. - ### Read Overwrite Streaming reading will ignore the commits generated by `INSERT OVERWRITE` by default. If you want to read the diff --git a/docs/content/flink/sql-write.md b/docs/content/flink/sql-write.md index 008fe498363c..6abbfa01756c 100644 --- a/docs/content/flink/sql-write.md +++ b/docs/content/flink/sql-write.md @@ -79,7 +79,7 @@ The data is clustered using an automatically chosen strategy (such as ORDER, ZOR by setting the `sink.clustering.strategy`. Clustering relies on sampling and sorting. If the clustering process takes too much time, you can decrease the total sample number by setting the `sink.clustering.sample-factor` or disable the sorting step by setting the `sink.clustering.sort-in-cluster` to false. -You can refer to [FlinkConnectorOptions]({{< ref "maintenance/configurations#FlinkConnectorOptions" >}}) for more info about the configurations above. +You can refer to [FlinkConnectorOptions]({{< ref "maintenance/configurations#flinkconnectoroptions" >}}) for more info about the configurations above. ## Overwriting the Whole Table @@ -175,9 +175,9 @@ PARTITION (k0 = 0, k1 = 0) SELECT v FROM my_table WHERE false; {{< hint info >}} Important table properties setting: -1. Only [primary key table]({{< ref "primary-key-table" >}}) supports this feature. -2. [MergeEngine]({{< ref "primary-key-table/merge-engine" >}}) needs to be [deduplicate]({{< ref "primary-key-table/merge-engine#deduplicate" >}}) - or [partial-update]({{< ref "primary-key-table/merge-engine#partial-update" >}}) to support this feature. +1. Only [primary key table]({{< ref "primary-key-table/overview" >}}) supports this feature. +2. [MergeEngine]({{< ref "primary-key-table/merge-engine" >}}) needs to be [deduplicate]({{< ref "primary-key-table/merge-engine/overview#deduplicate" >}}) + or [partial-update]({{< ref "primary-key-table/merge-engine/partial-update" >}}) to support this feature. 3. Do not support updating primary keys. {{< /hint >}} @@ -211,7 +211,9 @@ UPDATE my_table SET b = 1, c = 2 WHERE a = 'myTable'; {{< hint info >}} Important table properties setting: 1. Only primary key tables support this feature. -2. If the table has primary keys, [MergeEngine]({{< ref "primary-key-table/merge-engine" >}}) needs to be [deduplicate]({{< ref "primary-key-table/merge-engine#deduplicate" >}}) to support this feature. +2. If the table has primary keys, the following [MergeEngine]({{< ref "primary-key-table/merge-engine/overview" >}}) support this feature: + * [deduplicate]({{< ref "primary-key-table/merge-engine/overview#deduplicate" >}}). + * [partial-update]({{< ref "primary-key-table/merge-engine/partial-update" >}}) with option 'partial-update.remove-record-on-delete' enabled. 3. Do not support deleting from table in streaming mode. {{< /hint >}} @@ -257,7 +259,8 @@ CREATE TABLE my_partitioned_table ( 'partition.timestamp-formatter'='yyyyMMdd', 'partition.timestamp-pattern'='$dt', 'partition.time-interval'='1 d', - 'partition.idle-time-to-done'='15 m' + 'partition.idle-time-to-done'='15 m', + 'partition.mark-done-action'='done-partition' ); ``` @@ -267,4 +270,5 @@ CREATE TABLE my_partitioned_table ( and then it will be marked as done. 3. Thirdly, by default, partition mark done will create _SUCCESS file, the content of _SUCCESS file is a json, contains `creationTime` and `modificationTime`, they can help you understand if there is any delayed data. You can also - configure other actions. + configure other actions, like `'done-partition'`, for example, partition `'dt=20240501'` with produce + `'dt=20240501.done'` done partition. diff --git a/docs/content/learn-paimon/understand-files.md b/docs/content/learn-paimon/understand-files.md index b18c259993a7..fea6d30a0471 100644 --- a/docs/content/learn-paimon/understand-files.md +++ b/docs/content/learn-paimon/understand-files.md @@ -316,8 +316,8 @@ made and contains the following information: "commitKind" : "COMPACT", "timeMillis" : 1684163217960, "logOffsets" : { }, - "totalRecordCount" : 38, - "deltaRecordCount" : 20, + "totalRecordCount" : 2, + "deltaRecordCount" : -16, "changelogRecordCount" : 0, "watermark" : -9223372036854775808 } @@ -328,9 +328,9 @@ The new file layout as of snapshot-4 looks like Note that `manifest-4-0` contains 20 manifest entries (18 `DELETE` operations and 2 `ADD` operations) 1. For partition `20230503` to `20230510`, two `DELETE` operations for two data files -2. For partition `20230501` to `20230502`, one `DELETE` operation and one `ADD` operation - for the same data file. - +2. For partition `20230501` to `20230502`, one `DELETE` operation and one `ADD` operation for the same data file. + This is because there has been an upgrade of the file from level 0 to the highest level. Please rest assured that + this is only a change in metadata, and the file is still the same. ### Alter Table Execute the following statement to configure full-compaction: diff --git a/docs/content/maintenance/dedicated-compaction.md b/docs/content/maintenance/dedicated-compaction.md index 471bdad22275..63e0aa5e66e4 100644 --- a/docs/content/maintenance/dedicated-compaction.md +++ b/docs/content/maintenance/dedicated-compaction.md @@ -1,6 +1,6 @@ --- title: "Dedicated Compaction" -weight: 3 +weight: 4 type: docs aliases: - /maintenance/dedicated-compaction.html @@ -107,6 +107,7 @@ Run the following command to submit a compaction job for the table. --database \ --table \ [--partition ] \ + [--compact_strategy ] \ [--table_conf ] \ [--catalog_conf [--catalog_conf ...]] ``` @@ -123,10 +124,14 @@ Example: compact table --partition dt=20221126,hh=08 \ --partition dt=20221127,hh=09 \ --table_conf sink.parallelism=10 \ + --compact_strategy minor \ --catalog_conf s3.endpoint=https://****.com \ --catalog_conf s3.access-key=***** \ --catalog_conf s3.secret-key=***** ``` +* `--compact_strategy` Determines how to pick files to be merged, the default is determined by the runtime execution mode, streaming-mode use `minor` strategy and batch-mode use `full` strategy. + * `full` : Only supports batch mode. All files will be selected for merging. + * `minor` : Pick the set of files that need to be merged based on specified conditions. You can use `-D execution.runtime-mode=batch` or `-yD execution.runtime-mode=batch` (for the ON-YARN scenario) to control batch or streaming mode. If you submit a batch job, all current table files will be compacted. If you submit a streaming job, the job will continuously monitor new changes @@ -190,6 +195,7 @@ CALL sys.compact_database( [--including_tables ] \ [--excluding_tables ] \ [--mode ] \ + [--compact_strategy ] \ [--catalog_conf [--catalog_conf ...]] \ [--table_conf [--table_conf ...]] ``` @@ -346,6 +352,7 @@ CALL sys.compact(`table` => 'default.T', 'partition_idle_time' => '1 d') --table \ --partition_idle_time \ [--partition ] \ + [--compact_strategy ] \ [--catalog_conf [--catalog_conf ...]] \ [--table_conf [--table_conf ] ...] ``` @@ -406,6 +413,7 @@ CALL sys.compact_database( [--including_tables ] \ [--excluding_tables ] \ [--mode ] \ + [--compact_strategy ] \ [--catalog_conf [--catalog_conf ...]] \ [--table_conf [--table_conf ...]] ``` diff --git a/docs/content/maintenance/filesystems.md b/docs/content/maintenance/filesystems.md new file mode 100644 index 000000000000..dc030a9ec2bd --- /dev/null +++ b/docs/content/maintenance/filesystems.md @@ -0,0 +1,374 @@ +--- +title: "Filesystems" +weight: 1 +type: docs +aliases: +- /maintenance/filesystems.html +--- + + +# Filesystems + +Apache Paimon utilizes the same pluggable file systems as Apache Flink. Users can follow the +[standard plugin mechanism](https://nightlies.apache.org/flink/flink-docs-stable/docs/deployment/filesystems/plugins/) +to configure the plugin structure if using Flink as compute engine. However, for other engines like Spark +or Hive, the provided opt jars (by Flink) may get conflicts and cannot be used directly. It is not convenient +for users to fix class conflicts, thus Paimon provides the self-contained and engine-unified +FileSystem pluggable jars for user to query tables from Spark/Hive side. + +## Supported FileSystems + +| FileSystem | URI Scheme | Pluggable | Description | +|:------------------|:-----------|-----------|:-----------------------------------------------------------------------| +| Local File System | file:// | N | Built-in Support | +| HDFS | hdfs:// | N | Built-in Support, ensure that the cluster is in the hadoop environment | +| Aliyun OSS | oss:// | Y | | +| S3 | s3:// | Y | | + +## Dependency + +We recommend you to download the jar directly: [Download Link]({{< ref "project/download#filesystem-jars" >}}). + +You can also manually build bundled jar from the source code. + +To build from source code, [clone the git repository]({{< github_repo >}}). + +Build shaded jar with the following command. + +```bash +mvn clean install -DskipTests +``` + +You can find the shaded jars under +`./paimon-filesystems/paimon-${fs}/target/paimon-${fs}-{{< version >}}.jar`. + +## HDFS + +You don't need any additional dependencies to access HDFS because you have already taken care of the Hadoop dependencies. + +### HDFS Configuration + +For HDFS, the most important thing is to be able to read your HDFS configuration. + +{{< tabs "hdfs conf" >}} + +{{< tab "Flink" >}} + +You may not have to do anything, if you are in a hadoop environment. Otherwise pick one of the following ways to +configure your HDFS: + +1. Set environment variable `HADOOP_HOME` or `HADOOP_CONF_DIR`. +2. Configure `'hadoop-conf-dir'` in the paimon catalog. +3. Configure Hadoop options through prefix `'hadoop.'` in the paimon catalog. + +The first approach is recommended. + +If you do not want to include the value of the environment variable, you can configure `hadoop-conf-loader` to `option`. + +{{< /tab >}} + +{{< tab "Hive/Spark" >}} + +HDFS Configuration is available directly through the computation cluster, see cluster configuration of Hive and Spark for details. + +{{< /tab >}} + +{{< /tabs >}} + +### Hadoop-compatible file systems (HCFS) + +All Hadoop file systems are automatically available when the Hadoop libraries are on the classpath. + +This way, Paimon seamlessly supports all of Hadoop file systems implementing the `org.apache.hadoop.fs.FileSystem` +interface, and all Hadoop-compatible file systems (HCFS). + +- HDFS +- Alluxio (see configuration specifics below) +- XtreemFS +- … + +The Hadoop configuration has to have an entry for the required file system implementation in the `core-site.xml` file. + +For Alluxio support add the following entry into the core-site.xml file: + +```shell + + fs.alluxio.impl + alluxio.hadoop.FileSystem + +``` + +### Kerberos + +{{< tabs "Kerberos" >}} + +{{< tab "Flink" >}} + +It is recommended to use [Flink Kerberos Keytab](https://nightlies.apache.org/flink/flink-docs-stable/docs/deployment/security/security-kerberos/). + +{{< /tab >}} + +{{< tab "Spark" >}} + +It is recommended to use [Spark Kerberos Keytab](https://spark.apache.org/docs/latest/security.html#using-a-keytab). + +{{< /tab >}} + +{{< tab "Hive" >}} + +An intuitive approach is to configure Hive's kerberos authentication. + +{{< /tab >}} + +{{< tab "Trino/JavaAPI" >}} + +Configure the following three options in your catalog configuration: + +- security.kerberos.login.keytab: Absolute path to a Kerberos keytab file that contains the user credentials. + Please make sure it is copied to each machine. +- security.kerberos.login.principal: Kerberos principal name associated with the keytab. +- security.kerberos.login.use-ticket-cache: True or false, indicates whether to read from your Kerberos ticket cache. + +For JavaAPI: +``` +SecurityContext.install(catalogOptions); +``` + +{{< /tab >}} + +{{< /tabs >}} + +### HDFS HA + +Ensure that `hdfs-site.xml` and `core-site.xml` contain the necessary [HA configuration](https://hadoop.apache.org/docs/stable/hadoop-project-dist/hadoop-hdfs/HDFSHighAvailabilityWithNFS.html). + +### HDFS ViewFS + +Ensure that `hdfs-site.xml` and `core-site.xml` contain the necessary [ViewFs configuration](https://hadoop.apache.org/docs/stable/hadoop-project-dist/hadoop-hdfs/ViewFs.html). + +## OSS + +{{< stable >}} + +Download [paimon-oss-{{< version >}}.jar](https://repo.maven.apache.org/maven2/org/apache/paimon/paimon-oss/{{< version >}}/paimon-oss-{{< version >}}.jar). + +{{< /stable >}} + +{{< unstable >}} + +Download [paimon-oss-{{< version >}}.jar](https://repository.apache.org/snapshots/org/apache/paimon/paimon-oss/{{< version >}}/). + +{{< /unstable >}} + +{{< tabs "oss" >}} + +{{< tab "Flink" >}} + +{{< hint info >}} +If you have already configured [oss access through Flink](https://nightlies.apache.org/flink/flink-docs-stable/docs/deployment/filesystems/oss/) (Via Flink FileSystem), +here you can skip the following configuration. +{{< /hint >}} + +Put `paimon-oss-{{< version >}}.jar` into `lib` directory of your Flink home, and create catalog: + +```sql +CREATE CATALOG my_catalog WITH ( + 'type' = 'paimon', + 'warehouse' = 'oss:///', + 'fs.oss.endpoint' = 'oss-cn-hangzhou.aliyuncs.com', + 'fs.oss.accessKeyId' = 'xxx', + 'fs.oss.accessKeySecret' = 'yyy' +); +``` + +{{< /tab >}} + +{{< tab "Spark" >}} + +{{< hint info >}} +If you have already configured oss access through Spark (Via Hadoop FileSystem), here you can skip the following configuration. +{{< /hint >}} + +Place `paimon-oss-{{< version >}}.jar` together with `paimon-spark-{{< version >}}.jar` under Spark's jars directory, and start like + +```shell +spark-sql \ + --conf spark.sql.catalog.paimon=org.apache.paimon.spark.SparkCatalog \ + --conf spark.sql.catalog.paimon.warehouse=oss:/// \ + --conf spark.sql.catalog.paimon.fs.oss.endpoint=oss-cn-hangzhou.aliyuncs.com \ + --conf spark.sql.catalog.paimon.fs.oss.accessKeyId=xxx \ + --conf spark.sql.catalog.paimon.fs.oss.accessKeySecret=yyy +``` + +{{< /tab >}} + +{{< tab "Hive" >}} + +{{< hint info >}} +If you have already configured oss access through Hive (Via Hadoop FileSystem), here you can skip the following configuration. +{{< /hint >}} + +NOTE: You need to ensure that Hive metastore can access `oss`. + +Place `paimon-oss-{{< version >}}.jar` together with `paimon-hive-connector-{{< version >}}.jar` under Hive's auxlib directory, and start like + +```sql +SET paimon.fs.oss.endpoint=oss-cn-hangzhou.aliyuncs.com; +SET paimon.fs.oss.accessKeyId=xxx; +SET paimon.fs.oss.accessKeySecret=yyy; +``` + +And read table from hive metastore, table can be created by Flink or Spark, see [Catalog with Hive Metastore]({{< ref "flink/sql-ddl" >}}) +```sql +SELECT * FROM test_table; +SELECT COUNT(1) FROM test_table; +``` + +{{< /tab >}} +{{< tab "Trino" >}} + +From version 0.8, paimon-trino uses trino filesystem as basic file read and write system. We strongly recommend you to use jindo-sdk in trino. + +You can find [How to config jindo sdk on trino](https://github.com/aliyun/alibabacloud-jindodata/blob/master/docs/user/4.x/4.6.x/4.6.12/oss/presto/jindosdk_on_presto.md) here. +Please note that: +* Use paimon to replace hive-hadoop2 when you decompress the plugin jar and find location to put in. +* You can specify the `core-site.xml` in `paimon.properties` on configuration [hive.config.resources](https://trino.io/docs/current/connector/hive.html#hdfs-configuration). +* Presto and Jindo use the same configuration method. + +{{< /tab >}} +{{< /tabs >}} + +## S3 + +{{< stable >}} + +Download [paimon-s3-{{< version >}}.jar](https://repo.maven.apache.org/maven2/org/apache/paimon/paimon-s3/{{< version >}}/paimon-s3-{{< version >}}.jar). + +{{< /stable >}} + +{{< unstable >}} + +Download [paimon-s3-{{< version >}}.jar](https://repository.apache.org/snapshots/org/apache/paimon/paimon-s3/{{< version >}}/). + +{{< /unstable >}} + +{{< tabs "s3" >}} + +{{< tab "Flink" >}} + +{{< hint info >}} +If you have already configured [s3 access through Flink](https://nightlies.apache.org/flink/flink-docs-stable/docs/deployment/filesystems/s3/) (Via Flink FileSystem), +here you can skip the following configuration. +{{< /hint >}} + +Put `paimon-s3-{{< version >}}.jar` into `lib` directory of your Flink home, and create catalog: + +```sql +CREATE CATALOG my_catalog WITH ( + 'type' = 'paimon', + 'warehouse' = 's3:///', + 's3.endpoint' = 'your-endpoint-hostname', + 's3.access-key' = 'xxx', + 's3.secret-key' = 'yyy' +); +``` + +{{< /tab >}} + +{{< tab "Spark" >}} + +{{< hint info >}} +If you have already configured s3 access through Spark (Via Hadoop FileSystem), here you can skip the following configuration. +{{< /hint >}} + +Place `paimon-s3-{{< version >}}.jar` together with `paimon-spark-{{< version >}}.jar` under Spark's jars directory, and start like + +```shell +spark-sql \ + --conf spark.sql.catalog.paimon=org.apache.paimon.spark.SparkCatalog \ + --conf spark.sql.catalog.paimon.warehouse=s3:/// \ + --conf spark.sql.catalog.paimon.s3.endpoint=your-endpoint-hostname \ + --conf spark.sql.catalog.paimon.s3.access-key=xxx \ + --conf spark.sql.catalog.paimon.s3.secret-key=yyy +``` + +{{< /tab >}} + +{{< tab "Hive" >}} + +{{< hint info >}} +If you have already configured s3 access through Hive ((Via Hadoop FileSystem)), here you can skip the following configuration. +{{< /hint >}} + +NOTE: You need to ensure that Hive metastore can access `s3`. + +Place `paimon-s3-{{< version >}}.jar` together with `paimon-hive-connector-{{< version >}}.jar` under Hive's auxlib directory, and start like + +```sql +SET paimon.s3.endpoint=your-endpoint-hostname; +SET paimon.s3.access-key=xxx; +SET paimon.s3.secret-key=yyy; +``` + +And read table from hive metastore, table can be created by Flink or Spark, see [Catalog with Hive Metastore]({{< ref "flink/sql-ddl" >}}) +```sql +SELECT * FROM test_table; +SELECT COUNT(1) FROM test_table; +``` + +{{< /tab >}} + +{{< tab "Trino" >}} + +Paimon use shared trino filesystem as basic read and write system. + +Please refer to [Trino S3](https://trino.io/docs/current/object-storage/file-system-s3.html) to config s3 filesystem in trino. + +{{< /tab >}} + +{{< /tabs >}} + +### S3 Complaint Object Stores + +The S3 Filesystem also support using S3 compliant object stores such as MinIO, Tencent's COS and IBM’s Cloud Object +Storage. Just configure your endpoint to the provider of the object store service. + +```yaml +s3.endpoint: your-endpoint-hostname +``` + +### Configure Path Style Access + +Some S3 compliant object stores might not have virtual host style addressing enabled by default, for example when using Standalone MinIO for testing purpose. +In such cases, you will have to provide the property to enable path style access. + +```yaml +s3.path.style.access: true +``` + +### S3A Performance + +[Tune Performance](https://hadoop.apache.org/docs/stable/hadoop-aws/tools/hadoop-aws/performance.html) for `S3AFileSystem`. + +If you encounter the following exception: +```shell +Caused by: org.apache.http.conn.ConnectionPoolTimeoutException: Timeout waiting for connection from pool. +``` +Try to configure this in catalog options: `fs.s3a.connection.maximum=1000`. diff --git a/docs/content/maintenance/manage-snapshots.md b/docs/content/maintenance/manage-snapshots.md index 00c0322de8c2..721d5d0bafb2 100644 --- a/docs/content/maintenance/manage-snapshots.md +++ b/docs/content/maintenance/manage-snapshots.md @@ -1,6 +1,6 @@ --- title: "Manage Snapshots" -weight: 4 +weight: 5 type: docs aliases: - /maintenance/manage-snapshots.html @@ -308,9 +308,9 @@ submit a `remove_orphan_files` job to clean them: {{< tab "Spark SQL/Flink SQL" >}} ```sql -CALL sys.remove_orphan_files(`table` => "my_db.my_table", [older_than => "2023-10-31 12:00:00"]) +CALL sys.remove_orphan_files(`table` => 'my_db.my_table', [older_than => '2023-10-31 12:00:00']) -CALL sys.remove_orphan_files(`table` => "my_db.*", [older_than => "2023-10-31 12:00:00"]) +CALL sys.remove_orphan_files(`table` => 'my_db.*', [older_than => '2023-10-31 12:00:00']) ``` {{< /tab >}} diff --git a/docs/content/maintenance/metrics.md b/docs/content/maintenance/metrics.md index c21be8ec0302..2c3067267fd7 100644 --- a/docs/content/maintenance/metrics.md +++ b/docs/content/maintenance/metrics.md @@ -67,16 +67,6 @@ Below is lists of Paimon built-in metrics. They are summarized into types of sca Gauge Number of scanned manifest files in the last scan. - - lastSkippedByPartitionAndStats - Gauge - Skipped table files by partition filter and value / key stats information in the last scan. - - - lastSkippedByWholeBucketFilesFilter - Gauge - Skipped table files by bucket level value filter (only primary key table) in the last scan. - lastScanSkippedTableFiles Gauge @@ -181,6 +171,16 @@ Below is lists of Paimon built-in metrics. They are summarized into types of sca Gauge Number of buckets written in the last commit. + + lastCompactionInputFileSize + Gauge + Total size of the input files for the last compaction. + + + lastCompactionOutputFileSize + Gauge + Total size of the output files for the last compaction. + @@ -232,23 +232,53 @@ Below is lists of Paimon built-in metrics. They are summarized into types of sca maxLevel0FileCount Gauge - The maximum number of level 0 files currently handled by this writer. This value will become larger if asynchronous compaction cannot be done in time. + The maximum number of level 0 files currently handled by this task. This value will become larger if asynchronous compaction cannot be done in time. avgLevel0FileCount Gauge - The average number of level 0 files currently handled by this writer. This value will become larger if asynchronous compaction cannot be done in time. + The average number of level 0 files currently handled by this task. This value will become larger if asynchronous compaction cannot be done in time. compactionThreadBusy Gauge - The maximum business of compaction threads in this parallelism. Currently, there is only one compaction thread in each parallelism, so value of business ranges from 0 (idle) to 100 (compaction running all the time). + The maximum business of compaction threads in this task. Currently, there is only one compaction thread in each parallelism, so value of business ranges from 0 (idle) to 100 (compaction running all the time). avgCompactionTime Gauge The average runtime of compaction threads, calculated based on recorded compaction time data in milliseconds. The value represents the average duration of compaction operations. Higher values indicate longer average compaction times, which may suggest the need for performance optimization. + + compactionCompletedCount + Counter + The total number of compactions that have completed. + + + compactionQueuedCount + Counter + The total number of compactions that are queued/running. + + + maxCompactionInputSize + Gauge + The maximum input file size for this task's compaction. + + + avgCompactionInputSize/td> + Gauge + The average input file size for this task's compaction. + + + maxCompactionOutputSize + Gauge + The maximum output file size for this task's compaction. + + + avgCompactionOutputSize + Gauge + The average output file size for this task's compaction. + diff --git a/docs/content/maintenance/write-performance.md b/docs/content/maintenance/write-performance.md index 03e734874c05..ade2c3353e3c 100644 --- a/docs/content/maintenance/write-performance.md +++ b/docs/content/maintenance/write-performance.md @@ -1,6 +1,6 @@ --- title: "Write Performance" -weight: 2 +weight: 3 type: docs aliases: - /maintenance/write-performance.html diff --git a/docs/content/migration/iceberg-compatibility.md b/docs/content/migration/iceberg-compatibility.md new file mode 100644 index 000000000000..b6fcaa282615 --- /dev/null +++ b/docs/content/migration/iceberg-compatibility.md @@ -0,0 +1,535 @@ +--- +title: "Iceberg Compatibility" +weight: 4 +type: docs +aliases: +- /migration/iceberg-compatibility.html +--- + + +# Iceberg Compatibility + +Paimon supports generating Iceberg compatible metadata, +so that Paimon tables can be consumed directly by Iceberg readers. + +Set the following table options, so that Paimon tables can generate Iceberg compatible metadata. + + + + + + + + + + + + + + + + + + +
    OptionDefaultTypeDescription
    metadata.iceberg.storage
    		disabledEnum + When set, produce Iceberg metadata after a snapshot is committed, so that Iceberg readers can read Paimon's raw data files. +
      +
    • disabled: Disable Iceberg compatibility support.
      • +
    • table-location: Store Iceberg metadata in each table's directory.
      • +
    • hadoop-catalog: Store Iceberg metadata in a separate directory. This directory can be specified as the warehouse directory of an Iceberg Hadoop catalog.
      • +
    • hive-catalog: Not only store Iceberg metadata like hadoop-catalog, but also create Iceberg external table in Hive.
      • +
    +
    + +For most SQL users, we recommend setting `'metadata.iceberg.storage' = 'hadoop-catalog'` +or `'metadata.iceberg.storage' = 'hive-catalog'`, +so that all tables can be visited as an Iceberg warehouse. +For Iceberg Java API users, you might consider setting `'metadata.iceberg.storage' = 'table-location'`, +so you can visit each table with its table path. + +## Append Tables + +Let's walk through a simple example, where we query Paimon tables with Iceberg connectors in Flink and Spark. +Before trying out this example, make sure that your compute engine already supports Iceberg. +Please refer to Iceberg's document if you haven't set up Iceberg. +* Flink: [Preparation when using Flink SQL Client](https://iceberg.apache.org/docs/latest/flink/#preparation-when-using-flink-sql-client) +* Spark: [Using Iceberg in Spark 3](https://iceberg.apache.org/docs/latest/spark-getting-started/#using-iceberg-in-spark-3) + +Let's now create a Paimon append only table with Iceberg compatibility enabled and insert some data. + +{{< tabs "create-paimon-append-only-table" >}} + +{{< tab "Flink SQL" >}} +```sql +CREATE CATALOG paimon_catalog WITH ( + 'type' = 'paimon', + 'warehouse' = '' +); + +CREATE TABLE paimon_catalog.`default`.cities ( + country STRING, + name STRING +) WITH ( + 'metadata.iceberg.storage' = 'hadoop-catalog' +); + +INSERT INTO paimon_catalog.`default`.cities VALUES ('usa', 'new york'), ('germany', 'berlin'), ('usa', 'chicago'), ('germany', 'hamburg'); +``` +{{< /tab >}} + +{{< tab "Spark SQL" >}} +Start `spark-sql` with the following command line. + +```bash +spark-sql --jars \ + --conf spark.sql.catalog.paimon_catalog=org.apache.paimon.spark.SparkCatalog \ + --conf spark.sql.catalog.paimon_catalog.warehouse= \ + --packages org.apache.iceberg:iceberg-spark-runtime- \ + --conf spark.sql.catalog.iceberg_catalog=org.apache.iceberg.spark.SparkCatalog \ + --conf spark.sql.catalog.iceberg_catalog.type=hadoop \ + --conf spark.sql.catalog.iceberg_catalog.warehouse=/iceberg \ + --conf spark.sql.catalog.iceberg_catalog.cache-enabled=false \ # disable iceberg catalog caching to quickly see the result + --conf spark.sql.extensions=org.apache.paimon.spark.extensions.PaimonSparkSessionExtensions,org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions +``` + +Run the following Spark SQL to create Paimon table and insert data. + +```sql +CREATE TABLE paimon_catalog.`default`.cities ( + country STRING, + name STRING +) TBLPROPERTIES ( + 'metadata.iceberg.storage' = 'hadoop-catalog' +); + +INSERT INTO paimon_catalog.`default`.cities VALUES ('usa', 'new york'), ('germany', 'berlin'), ('usa', 'chicago'), ('germany', 'hamburg'); +``` +{{< /tab >}} + +{{< /tabs >}} + +Now let's query this Paimon table with Iceberg connector. + +{{< tabs "query-paimon-append-only-table" >}} + +{{< tab "Flink SQL" >}} +```sql +CREATE CATALOG iceberg_catalog WITH ( + 'type' = 'iceberg', + 'catalog-type' = 'hadoop', + 'warehouse' = '/iceberg', + 'cache-enabled' = 'false' -- disable iceberg catalog caching to quickly see the result +); + +SELECT * FROM iceberg_catalog.`default`.cities WHERE country = 'germany'; +/* ++----+--------------------------------+--------------------------------+ +| op | country | name | ++----+--------------------------------+--------------------------------+ +| +I | germany | berlin | +| +I | germany | hamburg | ++----+--------------------------------+--------------------------------+ +*/ +``` +{{< /tab >}} + +{{< tab "Spark SQL" >}} +```sql +SELECT * FROM iceberg_catalog.`default`.cities WHERE country = 'germany'; +/* +germany berlin +germany hamburg +*/ +``` +{{< /tab >}} + +{{< /tabs >}} + +Let's insert more data and query again. + +{{< tabs "query-paimon-append-only-table-again" >}} + +{{< tab "Flink SQL" >}} +```sql +INSERT INTO paimon_catalog.`default`.cities VALUES ('usa', 'houston'), ('germany', 'munich'); + +SELECT * FROM iceberg_catalog.`default`.cities WHERE country = 'germany'; +/* ++----+--------------------------------+--------------------------------+ +| op | country | name | ++----+--------------------------------+--------------------------------+ +| +I | germany | munich | +| +I | germany | berlin | +| +I | germany | hamburg | ++----+--------------------------------+--------------------------------+ +*/ +``` +{{< /tab >}} + +{{< tab "Spark SQL" >}} +```sql +INSERT INTO paimon_catalog.`default`.cities VALUES ('usa', 'houston'), ('germany', 'munich'); + +SELECT * FROM iceberg_catalog.`default`.cities WHERE country = 'germany'; +/* +germany munich +germany berlin +germany hamburg +*/ +``` +{{< /tab >}} + +{{< /tabs >}} + +## Primary Key Tables + +{{< tabs "paimon-primary-key-table" >}} + +{{< tab "Flink SQL" >}} +```sql +CREATE CATALOG paimon_catalog WITH ( + 'type' = 'paimon', + 'warehouse' = '' +); + +CREATE TABLE paimon_catalog.`default`.orders ( + order_id BIGINT, + status STRING, + payment DOUBLE, + PRIMARY KEY (order_id) NOT ENFORCED +) WITH ( + 'metadata.iceberg.storage' = 'hadoop-catalog', + 'compaction.optimization-interval' = '1ms' -- ATTENTION: this option is only for testing, see "timeliness" section below for more information +); + +INSERT INTO paimon_catalog.`default`.orders VALUES (1, 'SUBMITTED', CAST(NULL AS DOUBLE)), (2, 'COMPLETED', 200.0), (3, 'SUBMITTED', CAST(NULL AS DOUBLE)); + +CREATE CATALOG iceberg_catalog WITH ( + 'type' = 'iceberg', + 'catalog-type' = 'hadoop', + 'warehouse' = '/iceberg', + 'cache-enabled' = 'false' -- disable iceberg catalog caching to quickly see the result +); + +SELECT * FROM iceberg_catalog.`default`.orders WHERE status = 'COMPLETED'; +/* ++----+----------------------+--------------------------------+--------------------------------+ +| op | order_id | status | payment | ++----+----------------------+--------------------------------+--------------------------------+ +| +I | 2 | COMPLETED | 200.0 | ++----+----------------------+--------------------------------+--------------------------------+ +*/ + +INSERT INTO paimon_catalog.`default`.orders VALUES (1, 'COMPLETED', 100.0); + +SELECT * FROM iceberg_catalog.`default`.orders WHERE status = 'COMPLETED'; +/* ++----+----------------------+--------------------------------+--------------------------------+ +| op | order_id | status | payment | ++----+----------------------+--------------------------------+--------------------------------+ +| +I | 1 | COMPLETED | 100.0 | +| +I | 2 | COMPLETED | 200.0 | ++----+----------------------+--------------------------------+--------------------------------+ +*/ +``` +{{< /tab >}} + +{{< tab "Spark SQL" >}} +Start `spark-sql` with the following command line. + +```bash +spark-sql --jars \ + --conf spark.sql.catalog.paimon_catalog=org.apache.paimon.spark.SparkCatalog \ + --conf spark.sql.catalog.paimon_catalog.warehouse= \ + --packages org.apache.iceberg:iceberg-spark-runtime- \ + --conf spark.sql.catalog.iceberg_catalog=org.apache.iceberg.spark.SparkCatalog \ + --conf spark.sql.catalog.iceberg_catalog.type=hadoop \ + --conf spark.sql.catalog.iceberg_catalog.warehouse=/iceberg \ + --conf spark.sql.catalog.iceberg_catalog.cache-enabled=false \ # disable iceberg catalog caching to quickly see the result + --conf spark.sql.extensions=org.apache.paimon.spark.extensions.PaimonSparkSessionExtensions,org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions +``` + +Run the following Spark SQL to create Paimon table, insert/update data, and query with Iceberg catalog. + +```sql +CREATE TABLE paimon_catalog.`default`.orders ( + order_id BIGINT, + status STRING, + payment DOUBLE +) TBLPROPERTIES ( + 'primary-key' = 'order_id', + 'metadata.iceberg.storage' = 'hadoop-catalog', + 'compaction.optimization-interval' = '1ms' -- ATTENTION: this option is only for testing, see "timeliness" section below for more information +); + +INSERT INTO paimon_catalog.`default`.orders VALUES (1, 'SUBMITTED', CAST(NULL AS DOUBLE)), (2, 'COMPLETED', 200.0), (3, 'SUBMITTED', CAST(NULL AS DOUBLE)); + +SELECT * FROM iceberg_catalog.`default`.orders WHERE status = 'COMPLETED'; +/* +2 COMPLETED 200.0 +*/ + +INSERT INTO paimon_catalog.`default`.orders VALUES (1, 'COMPLETED', 100.0); + +SELECT * FROM iceberg_catalog.`default`.orders WHERE status = 'COMPLETED'; +/* +2 COMPLETED 200.0 +1 COMPLETED 100.0 +*/ +``` +{{< /tab >}} + +{{< /tabs >}} + +Paimon primary key tables organize data files as LSM trees, so data files must be merged in memory before querying. +However, Iceberg readers are not able to merge data files, so they can only query data files on the highest level of LSM trees. +Data files on the highest level are produced by the full compaction process. +So **to conclude, for primary key tables, Iceberg readers can only query data after full compaction**. + +By default, there is no guarantee on how frequently Paimon will perform full compaction. +You can configure the following table option, so that Paimon is forced to perform full compaction after several commits. + + + + + + + + + + + + + + + + + + + + + + + + +
    OptionDefaultTypeDescription
    compaction.optimization-interval
    		(none)DurationFull compaction will be constantly triggered per time interval. First compaction after the job starts will always be full compaction.
    full-compaction.delta-commits
    		(none)IntegerFull compaction will be constantly triggered after delta commits. Only implemented in Flink.
    + +Note that full compaction is a resource-consuming process, so the value of this table option should not be too small. +We recommend full compaction to be performed once or twice per hour. + +## Hive Catalog + +When creating Paimon table, set `'metadata.iceberg.storage' = 'hive-catalog'`. +This option value not only store Iceberg metadata like hadoop-catalog, but also create Iceberg external table in Hive. +This Paimon table can be accessed from Iceberg Hive catalog later. + +To provide information about Hive metastore, +you also need to set some (or all) of the following table options when creating Paimon table. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    OptionDefaultTypeDescription
    metadata.iceberg.uri
    		StringHive metastore uri for Iceberg Hive catalog.
    metadata.iceberg.hive-conf-dir
    		Stringhive-conf-dir for Iceberg Hive catalog.
    metadata.iceberg.hadoop-conf-dir
    		Stringhadoop-conf-dir for Iceberg Hive catalog.
    metadata.iceberg.manifest-compression
    		snappyStringCompression for Iceberg manifest files.
    metadata.iceberg.manifest-legacy-version
    		falseBooleanShould use the legacy manifest version to generate Iceberg's 1.4 manifest files.
    metadata.iceberg.hive-client-class
    		org.apache.hadoop.hive.metastore.HiveMetaStoreClientStringHive client class name for Iceberg Hive Catalog.
    + +## AWS Glue Catalog + +You can use Hive Catalog to connect AWS Glue metastore, you can use set `'metadata.iceberg.hive-client-class'` to +`'com.amazonaws.glue.catalog.metastore.AWSCatalogMetastoreClient'`. + +> **Note:** You can use this [repo](https://github.com/promotedai/aws-glue-data-catalog-client-for-apache-hive-metastore) to build the required jar, include it in your path and configure the AWSCatalogMetastoreClient. +## AWS Athena + +AWS Athena may use old manifest reader to read Iceberg manifest by names, we should let Paimon producing legacy Iceberg +manifest list file, you can enable: `'metadata.iceberg.manifest-legacy-version'`. + +## DuckDB + +Duckdb may rely on files placed in the `root/data` directory, while Paimon is usually placed directly in the `root` +directory, so you can configure this parameter for the table to achieve compatibility: +`'data-file.path-directory' = 'data'`. + +## Trino Iceberg + +In this example, we use Trino Iceberg connector to access Paimon table through Iceberg Hive catalog. +Before trying out this example, make sure that you have configured Trino Iceberg connector. +See [Trino's document](https://trino.io/docs/current/connector/iceberg.html#general-configuration) for more information. + +Let's first create a Paimon table with Iceberg compatibility enabled. + +{{< tabs "paimon-append-only-table-trino-1" >}} + +{{< tab "Flink SQL" >}} +```sql +CREATE CATALOG paimon_catalog WITH ( + 'type' = 'paimon', + 'warehouse' = '' +); + +CREATE TABLE paimon_catalog.`default`.animals ( + kind STRING, + name STRING +) WITH ( + 'metadata.iceberg.storage' = 'hive-catalog', + 'metadata.iceberg.uri' = 'thrift://:' +); + +INSERT INTO paimon_catalog.`default`.animals VALUES ('mammal', 'cat'), ('mammal', 'dog'), ('reptile', 'snake'), ('reptile', 'lizard'); +``` +{{< /tab >}} + +{{< tab "Spark SQL" >}} +Start `spark-sql` with the following command line. + +```bash +spark-sql --jars \ + --conf spark.sql.catalog.paimon_catalog=org.apache.paimon.spark.SparkCatalog \ + --conf spark.sql.catalog.paimon_catalog.warehouse= \ + --packages org.apache.iceberg:iceberg-spark-runtime- \ + --conf spark.sql.catalog.iceberg_catalog=org.apache.iceberg.spark.SparkCatalog \ + --conf spark.sql.catalog.iceberg_catalog.type=hadoop \ + --conf spark.sql.catalog.iceberg_catalog.warehouse=/iceberg \ + --conf spark.sql.catalog.iceberg_catalog.cache-enabled=false \ # disable iceberg catalog caching to quickly see the result + --conf spark.sql.extensions=org.apache.paimon.spark.extensions.PaimonSparkSessionExtensions,org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions +``` + +Run the following Spark SQL to create Paimon table, insert/update data, and query with Iceberg catalog. + +```sql +CREATE TABLE paimon_catalog.`default`.animals ( + kind STRING, + name STRING +) TBLPROPERTIES ( + 'metadata.iceberg.storage' = 'hive-catalog', + 'metadata.iceberg.uri' = 'thrift://:' +); + +INSERT INTO paimon_catalog.`default`.animals VALUES ('mammal', 'cat'), ('mammal', 'dog'), ('reptile', 'snake'), ('reptile', 'lizard'); +``` +{{< /tab >}} + +{{< /tabs >}} + +Start Trino using Iceberg catalog and query from Paimon table. + +```sql +SELECT * FROM animals WHERE class = 'mammal'; +/* + kind | name +--------+------ + mammal | cat + mammal | dog +*/ +``` + +## Supported Types + +Paimon Iceberg compatibility currently supports the following data types. + +| Paimon Data Type | Iceberg Data Type | +|------------------|-------------------| +| `BOOLEAN` | `boolean` | +| `INT` | `int` | +| `BIGINT` | `long` | +| `FLOAT` | `float` | +| `DOUBLE` | `double` | +| `DECIMAL` | `decimal` | +| `CHAR` | `string` | +| `VARCHAR` | `string` | +| `BINARY` | `binary` | +| `VARBINARY` | `binary` | +| `DATE` | `date` | +| `TIMESTAMP`* | `timestamp` | +| `TIMESTAMP_LTZ`* | `timestamptz` | +| `ARRAY` | `list` | +| `MAP` | `map` | +| `ROW` | `struct` | + +*: `TIMESTAMP` and `TIMESTAMP_LTZ` type only support precision from 4 to 6 + +## Table Options + + + + + + + + + + + + + + + + + + + + + + + + +
    OptionDefaultTypeDescription
    metadata.iceberg.compaction.min.file-num
    		10IntegerMinimum number of Iceberg metadata files to trigger metadata compaction.
    metadata.iceberg.compaction.max.file-num
    		50IntegerIf number of small Iceberg metadata files exceeds this limit, always trigger metadata compaction regardless of their total size.
    diff --git a/docs/content/migration/upsert-to-partitioned.md b/docs/content/migration/upsert-to-partitioned.md index b82bc942a626..d4fd00ae630b 100644 --- a/docs/content/migration/upsert-to-partitioned.md +++ b/docs/content/migration/upsert-to-partitioned.md @@ -26,6 +26,10 @@ under the License. # Upsert To Partitioned +{{< hint warning >}} +__Note:__ Only Hive Engine can be used to query these upsert-to-partitioned tables. +{{< /hint >}} + The [Tag Management]({{< ref "maintenance/manage-tags" >}}) will maintain the manifests and data files of the snapshot. A typical usage is creating tags daily, then you can maintain the historical data of each day for batch reading. diff --git a/docs/content/primary-key-table/changelog-producer.md b/docs/content/primary-key-table/changelog-producer.md index bf7a23fae2a5..a9364ee9f07c 100644 --- a/docs/content/primary-key-table/changelog-producer.md +++ b/docs/content/primary-key-table/changelog-producer.md @@ -58,9 +58,11 @@ By specifying `'changelog-producer' = 'input'`, Paimon writers rely on their inp ## Lookup -If your input can’t produce a complete changelog but you still want to get rid of the costly normalized operator, you may consider using the `'lookup'` changelog producer. +If your input can’t produce a complete changelog but you still want to get rid of the costly normalized operator, you +may consider using the `'lookup'` changelog producer. -By specifying `'changelog-producer' = 'lookup'`, Paimon will generate changelog through `'lookup'` before committing the data writing. +By specifying `'changelog-producer' = 'lookup'`, Paimon will generate changelog through `'lookup'` before committing +the data writing (You can also enable [Async Compaction]({{< ref "primary-key-table/compaction#asynchronous-compaction" >}})). {{< img src="/img/changelog-producer-lookup.png">}} @@ -105,23 +107,37 @@ important for performance). ## Full Compaction -If you think the resource consumption of 'lookup' is too large, you can consider using 'full-compaction' changelog producer, -which can decouple data writing and changelog generation, and is more suitable for scenarios with high latency (For example, 10 minutes). +You can also consider using 'full-compaction' changelog producer to generate changelog, and is more suitable for scenarios +with large latency (For example, 30 minutes). -By specifying `'changelog-producer' = 'full-compaction'`, Paimon will compare the results between full compactions and produce the differences as changelog. The latency of changelog is affected by the frequency of full compactions. +1. By specifying `'changelog-producer' = 'full-compaction'`, Paimon will compare the results between full compactions and +produce the differences as changelog. The latency of changelog is affected by the frequency of full compactions. +2. By specifying `full-compaction.delta-commits` table property, full compaction will be constantly triggered after delta +commits (checkpoints). This is set to 1 by default, so each checkpoint will have a full compression and generate a +changelog. -By specifying `full-compaction.delta-commits` table property, full compaction will be constantly triggered after delta commits (checkpoints). This is set to 1 by default, so each checkpoint will have a full compression and generate a change log. +Generally speaking, the cost and consumption of full compaction are high, so we recommend using `'lookup'` changelog +producer. {{< img src="/img/changelog-producer-full-compaction.png">}} {{< hint info >}} -Full compaction changelog producer can produce complete changelog for any type of source. However it is not as efficient as the input changelog producer and the latency to produce changelog might be high. +Full compaction changelog producer can produce complete changelog for any type of source. However it is not as +efficient as the input changelog producer and the latency to produce changelog might be high. {{< /hint >}} Full-compaction changelog-producer supports `changelog-producer.row-deduplicate` to avoid generating -U, +U changelog for the same record. -(Note: Please increase `'execution.checkpointing.max-concurrent-checkpoints'` Flink configuration, this is very -important for performance). +## Changelog Merging + +For `input`, `lookup`, `full-compaction` 'changelog-producer'. + +If Flink's checkpoint interval is short (for example, 30 seconds) and the number of buckets is large, each snapshot may +produce lots of small changelog files. Too many files may put a burden on the distributed storage cluster. + +In order to compact small changelog files into large ones, you can set the table option `changelog.precommit-compact = true`. +Default value of this option is false, if true, it will add a compact coordinator and worker operator after the writer +operator, which copies changelog files into large ones. diff --git a/docs/content/primary-key-table/compaction.md b/docs/content/primary-key-table/compaction.md index ada7e0289b35..bee8c16e46e9 100644 --- a/docs/content/primary-key-table/compaction.md +++ b/docs/content/primary-key-table/compaction.md @@ -76,7 +76,6 @@ In compaction, you can configure record-Level expire time to expire records, you 1. `'record-level.expire-time'`: time retain for records. 2. `'record-level.time-field'`: time field for record level expire. -3. `'record-level.time-field-type'`: time field type for record level expire, it can be seconds-int,seconds-long or millis-long. Expiration happens in compaction, and there is no strong guarantee to expire records in time. diff --git a/docs/content/primary-key-table/merge-engine/aggregation.md b/docs/content/primary-key-table/merge-engine/aggregation.md index a9b9b5a38cb0..0cc6507f2b4c 100644 --- a/docs/content/primary-key-table/merge-engine/aggregation.md +++ b/docs/content/primary-key-table/merge-engine/aggregation.md @@ -3,7 +3,7 @@ title: "Aggregation" weight: 3 type: docs aliases: -- /cdc-ingestion/merge-engin/aggregation.html +- /primary-key-table/merge-engin/aggregation.html --- + +# Query Performance + +## Table Mode + +The table schema has the greatest impact on query performance. See [Table Mode]({{< ref "primary-key-table/table-mode" >}}). + +For Merge On Read table, the most important thing you should pay attention to is the number of buckets, which will limit +the concurrency of reading data. + +For MOW (Deletion Vectors) or COW table or [Read Optimized]({{< ref "concepts/system-tables#read-optimized-table" >}}) table, +There is no limit to the concurrency of reading data, and they can also utilize some filtering conditions for non-primary-key columns. + +## Data Skipping By Primary Key Filter + +For a regular bucketed table (For example, bucket = 5), the filtering conditions of the primary key will greatly +accelerate queries and reduce the reading of a large number of files. + +## Data Skipping By File Index + +You can use file index to table with Deletion Vectors enabled, it filters files by index on the read side. + +```sql +CREATE TABLE WITH ( + 'deletion-vectors' = 'true', + 'file-index.bloom-filter.columns' = 'c1,c2', + 'file-index.bloom-filter.c1.items' = '200' +); +``` + +Supported filter types: + +`Bloom Filter`: +* `file-index.bloom-filter.columns`: specify the columns that need bloom filter index. +* `file-index.bloom-filter..fpp` to config false positive probability. +* `file-index.bloom-filter..items` to config the expected distinct items in one data file. + +`Bitmap`: +* `file-index.bitmap.columns`: specify the columns that need bitmap index. + +`Bit-Slice Index Bitmap` +* `file-index.bsi.columns`: specify the columns that need bsi index. + +More filter types will be supported... + +If you want to add file index to existing table, without any rewrite, you can use `rewrite_file_index` procedure. Before +we use the procedure, you should config appropriate configurations in target table. You can use ALTER clause to config +`file-index..columns` to the table. + +How to invoke: see [flink procedures]({{< ref "flink/procedures#procedures" >}}) diff --git a/docs/content/primary-key-table/table-mode.md b/docs/content/primary-key-table/table-mode.md index d7bc2efb9109..c8cce7c8bce5 100644 --- a/docs/content/primary-key-table/table-mode.md +++ b/docs/content/primary-key-table/table-mode.md @@ -110,7 +110,7 @@ If you don't want to use Deletion Vectors mode, you want to query fast enough in older data, you can also: 1. Configure 'compaction.optimization-interval' when writing data. -2. Query from [read-optimized system table]({{< ref "maintenance/system-tables#read-optimized-table" >}}). Reading from +2. Query from [read-optimized system table]({{< ref "concepts/system-tables#read-optimized-table" >}}). Reading from results of optimized files avoids merging records with the same key, thus improving reading performance. You can flexibly balance query performance and data latency when reading. diff --git a/docs/content/program-api/catalog-api.md b/docs/content/program-api/catalog-api.md index d016cfa7b204..570577437d86 100644 --- a/docs/content/program-api/catalog-api.md +++ b/docs/content/program-api/catalog-api.md @@ -1,6 +1,6 @@ --- title: "Catalog API" -weight: 4 +weight: 3 type: docs aliases: - /api/catalog-api.html diff --git a/docs/content/program-api/flink-api.md b/docs/content/program-api/flink-api.md index 7cf9d1932ce5..6ecac3909ced 100644 --- a/docs/content/program-api/flink-api.md +++ b/docs/content/program-api/flink-api.md @@ -26,12 +26,8 @@ under the License. # Flink API -{{< hint warning >}} -We do not recommend using programming API. Paimon is designed for SQL first, unless you are a professional Flink developer, even if you do, it can be very difficult. - -We strongly recommend that you use Flink SQL or Spark SQL, or simply use SQL APIs in programs. - -The following documents are not detailed and are for reference only. +{{< hint info >}} +If possible, recommend using Flink SQL or Spark SQL, or simply use SQL APIs in programs. {{< /hint >}} ## Dependency diff --git a/docs/content/program-api/java-api.md b/docs/content/program-api/java-api.md index 97f4ac5ae3dd..1bf0ce82f6b0 100644 --- a/docs/content/program-api/java-api.md +++ b/docs/content/program-api/java-api.md @@ -26,12 +26,8 @@ under the License. # Java API -{{< hint warning >}} -We do not recommend using the Paimon API naked, unless you are a professional downstream ecosystem developer, and even if you do, there will be significant difficulties. - -If you are only using Paimon, we strongly recommend using computing engines such as Flink SQL or Spark SQL. - -The following documents are not detailed and are for reference only. +{{< hint info >}} +If possible, recommend using computing engines such as Flink SQL or Spark SQL. {{< /hint >}} ## Dependency diff --git a/docs/content/program-api/python-api.md b/docs/content/program-api/python-api.md index fbaa6181124d..079170760b25 100644 --- a/docs/content/program-api/python-api.md +++ b/docs/content/program-api/python-api.md @@ -1,6 +1,6 @@ --- title: "Python API" -weight: 3 +weight: 4 type: docs aliases: - /api/python-api.html @@ -34,9 +34,9 @@ Java-based implementation will launch a JVM and use `py4j` to execute Java code ### SDK Installing -SDK is published at [paimon-python](https://pypi.org/project/paimon-python/). You can install by +SDK is published at [pypaimon](https://pypi.org/project/pypaimon/). You can install by ```shell -pip install paimon-python +pip install pypaimon ``` ### Java Runtime Environment @@ -67,7 +67,7 @@ classpath via one of the following ways: ```python import os -from paimon_python_java import constants +from pypaimon.py4j import constants os.environ[constants.PYPAIMON_JAVA_CLASSPATH] = '/path/to/jars/*' ``` @@ -81,7 +81,7 @@ You can set JVM args via one of the following ways: ```python import os -from paimon_python_java import constants +from pypaimon.py4j import constants os.environ[constants.PYPAIMON_JVM_ARGS] = 'arg1 arg2 ...' ``` @@ -98,7 +98,7 @@ Otherwise, you should set hadoop classpath via one of the following ways: ```python import os -from paimon_python_java import constants +from pypaimon.py4j import constants os.environ[constants.PYPAIMON_HADOOP_CLASSPATH] = '/path/to/jars/*' ``` @@ -111,7 +111,7 @@ If you just want to test codes in local, we recommend to use [Flink Pre-bundled Before coming into contact with the Table, you need to create a Catalog. ```python -from paimon_python_java import Catalog +from pypaimon.py4j import Catalog # Note that keys and values are all string catalog_options = { @@ -121,6 +121,94 @@ catalog_options = { catalog = Catalog.create(catalog_options) ``` +## Create Database & Table + +You can use the catalog to create table for writing data. + +### Create Database (optional) +Table is located in a database. If you want to create table in a new database, you should create it. + +```python +catalog.create_database( + name='database_name', + ignore_if_exists=True, # If you want to raise error if the database exists, set False + properties={'key': 'value'} # optional database properties +) +``` + +### Create Schema + +Table schema contains fields definition, partition keys, primary keys, table options and comment. +The field definition is described by `pyarrow.Schema`. All arguments except fields definition are optional. + +Generally, there are two ways to build `pyarrow.Schema`. + +First, you can use `pyarrow.schema` method directly, for example: + +```python +import pyarrow as pa + +from pypaimon import Schema + +pa_schema = pa.schema([ + ('dt', pa.string()), + ('hh', pa.string()), + ('pk', pa.int64()), + ('value', pa.string()) +]) + +schema = Schema( + pa_schema=pa_schema, + partition_keys=['dt', 'hh'], + primary_keys=['dt', 'hh', 'pk'], + options={'bucket': '2'}, + comment='my test table' +) +``` + +See [Data Types]({{< ref "python-api#data-types" >}}) for all supported `pyarrow-to-paimon` data types mapping. + +Second, if you have some Pandas data, the `pa_schema` can be extracted from `DataFrame`: + +```python +import pandas as pd +import pyarrow as pa + +from pypaimon import Schema + +# Example DataFrame data +data = { + 'dt': ['2024-01-01', '2024-01-01', '2024-01-02'], + 'hh': ['12', '15', '20'], + 'pk': [1, 2, 3], + 'value': ['a', 'b', 'c'], +} +dataframe = pd.DataFrame(data) + +# Get Paimon Schema +record_batch = pa.RecordBatch.from_pandas(dataframe) +schema = Schema( + pa_schema=record_batch.schema, + partition_keys=['dt', 'hh'], + primary_keys=['dt', 'hh', 'pk'], + options={'bucket': '2'}, + comment='my test table' +) +``` + +### Create Table + +After building table schema, you can create corresponding table: + +```python +schema = ... +catalog.create_table( + identifier='database_name.table_name', + schema=schema, + ignore_if_exists=True # If you want to raise error if the table exists, set False +) +``` + ## Get Table The Table interface provides tools to read and write table. @@ -131,35 +219,164 @@ table = catalog.get_table('database_name.table_name') ## Batch Read -TableRead interface provides parallelly reading for multiple splits. You can set `'max-workers': 'N'` in `catalog_options` -to set thread numbers when reading splits. `max-workers` is 1 by default, that means TableRead will read splits sequentially +### Set Read Parallelism + +TableRead interface provides parallelly reading for multiple splits. You can set `'max-workers': 'N'` in `catalog_options` +to set thread numbers for reading splits. `max-workers` is 1 by default, that means TableRead will read splits sequentially if you doesn't set `max-workers`. +### Get ReadBuilder and Perform pushdown + +A `ReadBuilder` is used to build reading utils and perform filter and projection pushdown. + ```python table = catalog.get_table('database_name.table_name') - -# 1. Create table scan and read read_builder = table.new_read_builder() +``` + +You can use `PredicateBuilder` to build filters and pushdown them by `ReadBuilder`: + +```python +# Example filter: ('f0' < 3 OR 'f1' > 6) AND 'f3' = 'A' + +predicate_builder = read_builder.new_predicate_builder() + +predicate1 = predicate_builder.less_than('f0', 3) +predicate2 = predicate_builder.greater_than('f1', 6) +predicate3 = predicate_builder.or_predicates([predicate1, predicate2]) + +predicate4 = predicate_builder.equal('f3', 'A') +predicate_5 = predicate_builder.and_predicates([predicate3, predicate4]) + +read_builder = read_builder.with_filter(predicate_5) +``` + +See [Predicate]({{< ref "python-api#predicate" >}}) for all supported filters and building methods. + +You can also pushdown projection by `ReadBuilder`: + +```python +# select f3 and f2 columns +read_builder = read_builder.with_projection(['f3', 'f2']) +``` + +### Scan Plan + +Then you can step into Scan Plan stage to get `splits`: + +```python table_scan = read_builder.new_scan() -table_read = read_builder.new_read() +splits = table_scan.splits() +``` + +### Read Splits -# 2. Get splits -splits = table_scan.plan().splits() +Finally, you can read data from the `splits` to various data format. -# 3. Read splits. Support 3 methods: -# 3.1 Read as pandas.DataFrame -dataframe = table_read.to_pandas(splits) +#### Apache Arrow -# 3.2 Read as pyarrow.Table +This requires `pyarrow` to be installed. + +You can read all the data into a `pyarrow.Table`: + +```python +table_read = read_builder.new_read() pa_table = table_read.to_arrow(splits) +print(pa_table) + +# pyarrow.Table +# f0: int32 +# f1: string +# ---- +# f0: [[1,2,3],[4,5,6],...] +# f1: [["a","b","c"],["d","e","f"],...] +``` + +You can also read data into a `pyarrow.RecordBatchReader` and iterate record batches: -# 3.3 Read as pyarrow.RecordBatchReader -record_batch_reader = table_read.to_arrow_batch_reader(splits) +```python +table_read = read_builder.new_read() +for batch in table_read.to_arrow_batch_reader(splits): + print(batch) + +# pyarrow.RecordBatch +# f0: int32 +# f1: string +# ---- +# f0: [1,2,3] +# f1: ["a","b","c"] +``` + +#### Pandas + +This requires `pandas` to be installed. + +You can read all the data into a `pandas.DataFrame`: + +```python +table_read = read_builder.new_read() +df = table_read.to_pandas(splits) +print(df) + +# f0 f1 +# 0 1 a +# 1 2 b +# 2 3 c +# 3 4 d +# ... +``` + +#### DuckDB + +This requires `duckdb` to be installed. + +You can convert the splits into an in-memory DuckDB table and query it: + +```python +table_read = read_builder.new_read() +duckdb_con = table_read.to_duckdb(splits, 'duckdb_table') + +print(duckdb_con.query("SELECT * FROM duckdb_table").fetchdf()) +# f0 f1 +# 0 1 a +# 1 2 b +# 2 3 c +# 3 4 d +# ... + +print(duckdb_con.query("SELECT * FROM duckdb_table WHERE f0 = 1").fetchdf()) +# f0 f1 +# 0 1 a +``` + +#### Ray + +This requires `ray` to be installed. + +You can convert the splits into a Ray dataset and handle it by Ray API: + +```python +table_read = read_builder.new_read() +ray_dataset = table_read.to_ray(splits) + +print(ray_dataset) +# MaterializedDataset(num_blocks=1, num_rows=9, schema={f0: int32, f1: string}) + +print(ray_dataset.take(3)) +# [{'f0': 1, 'f1': 'a'}, {'f0': 2, 'f1': 'b'}, {'f0': 3, 'f1': 'c'}] + +print(ray_dataset.to_pandas()) +# f0 f1 +# 0 1 a +# 1 2 b +# 2 3 c +# 3 4 d +# ... ``` ## Batch Write -Paimon table write is Two-Phase Commit, you can write many times, but once committed, no more data can be write. +Paimon table write is Two-Phase Commit, you can write many times, but once committed, no more data can be written. {{< hint warning >}} Currently, Python SDK doesn't support writing primary key table with `bucket=-1`. @@ -170,12 +387,6 @@ table = catalog.get_table('database_name.table_name') # 1. Create table write and commit write_builder = table.new_batch_write_builder() -# By default, write data will be appended to table. -# If you want to overwrite table: -# write_builder.overwrite() -# If you want to overwrite partition 'dt=2024-01-01': -# write_builder.overwrite({'dt': '2024-01-01'}) - table_write = write_builder.new_write() table_commit = write_builder.new_commit() @@ -199,7 +410,16 @@ table_commit.commit(commit_messages) # 4. Close resources table_write.close() table_commit.close() +``` +By default, the data will be appended to table. If you want to overwrite table, you should use `TableWrite#overwrite` API: + +```python +# overwrite whole table +write_builder.overwrite() + +# overwrite partition 'dt=2024-01-01' +write_builder.overwrite({'dt': '2024-01-01'}) ``` ## Data Types @@ -214,3 +434,25 @@ table_commit.close() | pyarrow.float64() | DOUBLE | | pyarrow.string() | STRING | | pyarrow.boolean() | BOOLEAN | + +## Predicate + +| Predicate kind | Predicate method | +|:----------------------|:----------------------------------------------| +| p1 and p2 | PredicateBuilder.and_predicates([p1, p2]) | +| p1 or p2 | PredicateBuilder.or_predicates([p1, p2]) | +| f = literal | PredicateBuilder.equal(f, literal) | +| f != literal | PredicateBuilder.not_equal(f, literal) | +| f < literal | PredicateBuilder.less_than(f, literal) | +| f <= literal | PredicateBuilder.less_or_equal(f, literal) | +| f > literal | PredicateBuilder.greater_than(f, literal) | +| f >= literal | PredicateBuilder.greater_or_equal(f, literal) | +| f is null | PredicateBuilder.is_null(f) | +| f is not null | PredicateBuilder.is_not_null(f) | +| f.startswith(literal) | PredicateBuilder.startswith(f, literal) | +| f.endswith(literal) | PredicateBuilder.endswith(f, literal) | +| f.contains(literal) | PredicateBuilder.contains(f, literal) | +| f is in [l1, l2] | PredicateBuilder.is_in(f, [l1, l2]) | +| f is not in [l1, l2] | PredicateBuilder.is_not_in(f, [l1, l2]) | +| lower <= f <= upper | PredicateBuilder.between(f, lower, upper) | + diff --git a/docs/content/project/download.md b/docs/content/project/download.md index 5e49811076a6..23d0112b09a2 100644 --- a/docs/content/project/download.md +++ b/docs/content/project/download.md @@ -49,13 +49,8 @@ This documentation is a guide for downloading Paimon Jars. | Hive 2.3 | [paimon-hive-connector-2.3-{{< version >}}.jar](https://repository.apache.org/snapshots/org/apache/paimon/paimon-hive-connector-2.3/{{< version >}}/) | | Hive 2.2 | [paimon-hive-connector-2.2-{{< version >}}.jar](https://repository.apache.org/snapshots/org/apache/paimon/paimon-hive-connector-2.2/{{< version >}}/) | | Hive 2.1 | [paimon-hive-connector-2.1-{{< version >}}.jar](https://repository.apache.org/snapshots/org/apache/paimon/paimon-hive-connector-2.1/{{< version >}}/) | -| Hive 2.1-cdh-6.3 | [paimon-hive-connector-2.1-cdh-6.3-{{< version >}}.jar](https://repository.apache.org/snapshots/org/apache/paimon/paimon-hive-connector-2.1-cdh-6.3/{{< version >}}/) | -| Presto 0.236 | [paimon-presto-0.236-{{< version >}}.jar](https://repository.apache.org/snapshots/org/apache/paimon/paimon-presto-0.236/{{< version >}}/) | -| Presto 0.268 | [paimon-presto-0.268-{{< version >}}.jar](https://repository.apache.org/snapshots/org/apache/paimon/paimon-presto-0.268/{{< version >}}/) | -| Presto 0.273 | [paimon-presto-0.273-{{< version >}}.jar](https://repository.apache.org/snapshots/org/apache/paimon/paimon-presto-0.273/{{< version >}}/) | -| Presto SQL 332 | [paimon-prestosql-332-{{< version >}}.jar](https://repository.apache.org/snapshots/org/apache/paimon/paimon-prestosql-332/{{< version >}}/) | -| Trino 420 | [paimon-trino-420-{{< version >}}.jar](https://repository.apache.org/snapshots/org/apache/paimon/paimon-trino-420/{{< version >}}/) | -| Trino 427 | [paimon-trino-427-{{< version >}}.jar](https://repository.apache.org/snapshots/org/apache/paimon/paimon-trino-427/{{< version >}}/) | +| Hive 2.1-cdh-6.3 | [paimon-hive-connector-2.1-cdh-6.3-{{< version >}}.jar](https://repository.apache.org/snapshots/org/apache/paimon/paimon-hive-connector-2.1-cdh-6.3/{{< version >}}/) | | +| Trino 440 | [paimon-trino-440-{{< version >}}-plugin.tar.gz](https://repository.apache.org/content/repositories/snapshots/org/apache/paimon/paimon-trino-440/{{< version >}}/) | {{< /unstable >}} @@ -79,7 +74,6 @@ This documentation is a guide for downloading Paimon Jars. | Hive 2.2 | [paimon-hive-connector-2.2-{{< version >}}.jar](https://repo.maven.apache.org/maven2/org/apache/paimon/paimon-hive-connector-2.2/{{< version >}}/paimon-hive-connector-2.2-{{< version >}}.jar) | | Hive 2.1 | [paimon-hive-connector-2.1-{{< version >}}.jar](https://repo.maven.apache.org/maven2/org/apache/paimon/paimon-hive-connector-2.1/{{< version >}}/paimon-hive-connector-2.1-{{< version >}}.jar) | | Hive 2.1-cdh-6.3 | [paimon-hive-connector-2.1-cdh-6.3-{{< version >}}.jar](https://repo.maven.apache.org/maven2/org/apache/paimon/paimon-hive-connector-2.1-cdh-6.3/{{< version >}}/paimon-hive-connector-2.1-cdh-6.3-{{< version >}}.jar) | -| Presto | [Download from master](https://paimon.apache.org/docs/master/project/download/) | | Trino | [Download from master](https://paimon.apache.org/docs/master/project/download/) | {{< /stable >}} diff --git a/docs/content/project/roadmap.md b/docs/content/project/roadmap.md index 2f6b63af00a1..34628e28c80f 100644 --- a/docs/content/project/roadmap.md +++ b/docs/content/project/roadmap.md @@ -26,16 +26,6 @@ under the License. # Roadmap -## Native Format IO - -Integrate native Parquet & ORC reader & writer. - -## Deletion Vectors (Merge On Write) - -1. Primary Key Table Deletion Vectors Mode supports async compaction. -2. Append Table supports DELETE & UPDATE with Deletion Vectors Mode. (Now only Spark SQL) -3. Optimize lookup performance for HDD disk. - ## Flink Lookup Join Support Flink Custom Data Distribution Lookup Join to reach large-scale data lookup join. @@ -44,51 +34,24 @@ Support Flink Custom Data Distribution Lookup Join to reach large-scale data loo Introduce a mode to produce Iceberg snapshots. -## Branch - -Branch production ready. - -## Changelog life cycle decouple - -Changelog life cycle decouple supports none changelog-producer. - -## Partition Mark Done - -Support partition mark done. - -## Default File Format - -- Default compression is ZSTD with level 1. -- Parquet supports filter push down. -- Parquet supports arrow with row type element. -- Parquet becomes default file format. - ## Variant Type Support Variant Type with Spark 4.0 and Flink 2.0. Unlocking support for semi-structured data. -## Bucketed Join - -Support Bucketed Join with Spark SQL to reduce shuffler in Join. - ## File Index Add more index: -1. Bitmap -2. Inverse -## Column Family +1. Inverse + +## Vector Compaction -Support Column Family for super Wide Table. +Support Vector Compaction for super Wide Table. -## View & Function support +## Function support -Paimon Catalog supports views and functions. +Paimon Catalog supports functions. ## Files Schema Evolution Ingestion Introduce a files Ingestion with Schema Evolution. - -## Foreign Key Join - -Explore Foreign Key Join solution. diff --git a/docs/content/spark/_index.md b/docs/content/spark/_index.md index 24661e56f25a..07128574b0e7 100644 --- a/docs/content/spark/_index.md +++ b/docs/content/spark/_index.md @@ -3,7 +3,7 @@ title: Engine Spark icon: bold: true bookCollapseSection: true -weight: 5 +weight: 6 --- +*/}} + + + + + + + + + + + + + + + + + +
    KeyDefaultTypeDescription
    field-delimiter
    		","StringOptional field delimiter character for CSV (',' by default).
    diff --git a/docs/layouts/shortcodes/generated/hive_catalog_configuration.html b/docs/layouts/shortcodes/generated/hive_catalog_configuration.html index 076a46232c3c..7b6242616f35 100644 --- a/docs/layouts/shortcodes/generated/hive_catalog_configuration.html +++ b/docs/layouts/shortcodes/generated/hive_catalog_configuration.html @@ -39,12 +39,6 @@ String Specify client cache key, multiple elements separated by commas.
    • "ugi": the Hadoop UserGroupInformation instance that represents the current user using the cache.
    • "user_name" similar to UGI but only includes the user's name determined by UserGroupInformation#getUserName.
    • "conf": name of an arbitrary configuration. The value of the configuration will be extracted from catalog properties and added to the cache key. A conf element should start with a "conf:" prefix which is followed by the configuration name. E.g. specifying "conf:a.b.c" will add "a.b.c" to the key, and so that configurations with different default catalog wouldn't share the same client pool. Multiple conf elements can be specified.
    - -
    format-table.enabled
    - false - Boolean - Whether to support format tables, format table corresponds to a regular Hive table, allowing read and write operations. However, during these processes, it does not connect to the metastore; hence, newly added partitions will not be reflected in the metastore and need to be manually added as separate partition operations. -
    hadoop-conf-dir
    (none) @@ -71,5 +65,12 @@ you can set this option to true. + +
    metastore.client.class
    + "org.apache.hadoop.hive.metastore.HiveMetaStoreClient" + String + Class name of Hive metastore client. +NOTE: This class must directly implements org.apache.hadoop.hive.metastore.IMetaStoreClient. + diff --git a/docs/layouts/shortcodes/generated/kafka_sync_database.html b/docs/layouts/shortcodes/generated/kafka_sync_database.html index 888901991d69..3664128a26ca 100644 --- a/docs/layouts/shortcodes/generated/kafka_sync_database.html +++ b/docs/layouts/shortcodes/generated/kafka_sync_database.html @@ -37,13 +37,25 @@
    --ignore_incompatible
    It is default false, in this case, if MySQL table name exists in Paimon and their schema is incompatible,an exception will be thrown. You can specify it to true explicitly to ignore the incompatible tables and exception. + +
    --table_mapping
    + The table name mapping between source database and Paimon. For example, if you want to synchronize a source table named "test" to a Paimon table named "paimon_test", you can specify "--table_mapping test=paimon_test". Multiple mappings could be specified with multiple "--table_mapping" options. "--table_mapping" has higher priority than "--table_prefix" and "--table_suffix". + + +
    --table_prefix_db
    + The prefix of the Paimon tables to be synchronized from the specified db. For example, if you want to prefix the tables from db1 with "ods_db1_", you can specify "--table_prefix_db db1=ods_db1_". "--table_prefix_db" has higher priority than "--table_prefix". +
    --table_prefix
    - The prefix of all Paimon tables to be synchronized. For example, if you want all synchronized tables to have "ods_" as prefix, you can specify "--table_prefix ods_". + The prefix of all Paimon tables to be synchronized except those specified by "--table_mapping" or "--table_prefix_db". For example, if you want all synchronized tables to have "ods_" as prefix, you can specify "--table_prefix ods_". + + +
    --table_suffix_db
    + The suffix of the Paimon tables to be synchronized from the specified db. The usage is same as "--table_prefix_db".
    --table_suffix
    - The suffix of all Paimon tables to be synchronized. The usage is same as "--table_prefix". + The suffix of all Paimon tables to be synchronized except those specified by "--table_mapping" or "--table_suffix_db". The usage is same as "--table_prefix".
    --including_tables
    diff --git a/docs/static/img/unaware-bucket-topo.png b/docs/static/img/unaware-bucket-topo.png index 73bc862053fd..f530fc4a225c 100644 Binary files a/docs/static/img/unaware-bucket-topo.png and b/docs/static/img/unaware-bucket-topo.png differ diff --git a/paimon-arrow/src/main/java/org/apache/paimon/arrow/ArrowUtils.java b/paimon-arrow/src/main/java/org/apache/paimon/arrow/ArrowUtils.java index 0cf40ad9faae..b3925a0a769e 100644 --- a/paimon-arrow/src/main/java/org/apache/paimon/arrow/ArrowUtils.java +++ b/paimon-arrow/src/main/java/org/apache/paimon/arrow/ArrowUtils.java @@ -22,6 +22,7 @@ import org.apache.paimon.arrow.writer.ArrowFieldWriter; import org.apache.paimon.arrow.writer.ArrowFieldWriterFactoryVisitor; import org.apache.paimon.data.Timestamp; +import org.apache.paimon.table.SpecialFields; import org.apache.paimon.types.ArrayType; import org.apache.paimon.types.DataField; import org.apache.paimon.types.DataType; @@ -48,6 +49,7 @@ import java.io.OutputStream; import java.time.Instant; import java.time.ZoneId; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -56,6 +58,8 @@ /** Utilities for creating Arrow objects. */ public class ArrowUtils { + static final String PARQUET_FIELD_ID = "PARQUET:field_id"; + public static VectorSchemaRoot createVectorSchemaRoot( RowType rowType, BufferAllocator allocator) { return createVectorSchemaRoot(rowType, allocator, true); @@ -69,7 +73,9 @@ public static VectorSchemaRoot createVectorSchemaRoot( f -> toArrowField( allowUpperCase ? f.name() : f.name().toLowerCase(), - f.type())) + f.id(), + f.type(), + 0)) .collect(Collectors.toList()); return VectorSchemaRoot.create(new Schema(fields), allocator); } @@ -78,40 +84,105 @@ public static FieldVector createVector( DataField dataField, BufferAllocator allocator, boolean allowUpperCase) { return toArrowField( allowUpperCase ? dataField.name() : dataField.name().toLowerCase(), - dataField.type()) + dataField.id(), + dataField.type(), + 0) .createVector(allocator); } - public static Field toArrowField(String fieldName, DataType dataType) { + public static Field toArrowField(String fieldName, int fieldId, DataType dataType, int depth) { FieldType fieldType = dataType.accept(ArrowFieldTypeConversion.ARROW_FIELD_TYPE_VISITOR); + fieldType = + new FieldType( + fieldType.isNullable(), + fieldType.getType(), + fieldType.getDictionary(), + Collections.singletonMap(PARQUET_FIELD_ID, String.valueOf(fieldId))); List children = null; if (dataType instanceof ArrayType) { - children = - Collections.singletonList( - toArrowField( - ListVector.DATA_VECTOR_NAME, - ((ArrayType) dataType).getElementType())); + Field field = + toArrowField( + ListVector.DATA_VECTOR_NAME, + fieldId, + ((ArrayType) dataType).getElementType(), + depth + 1); + FieldType typeInner = field.getFieldType(); + field = + new Field( + field.getName(), + new FieldType( + typeInner.isNullable(), + typeInner.getType(), + typeInner.getDictionary(), + Collections.singletonMap( + PARQUET_FIELD_ID, + String.valueOf( + SpecialFields.getArrayElementFieldId( + fieldId, depth + 1)))), + field.getChildren()); + children = Collections.singletonList(field); } else if (dataType instanceof MapType) { MapType mapType = (MapType) dataType; - children = - Collections.singletonList( - new Field( - MapVector.DATA_VECTOR_NAME, - // data vector, key vector and value vector CANNOT be null - new FieldType(false, Types.MinorType.STRUCT.getType(), null), - Arrays.asList( - toArrowField( - MapVector.KEY_NAME, - mapType.getKeyType().notNull()), - toArrowField( - MapVector.VALUE_NAME, - mapType.getValueType().notNull())))); + + Field keyField = + toArrowField( + MapVector.KEY_NAME, fieldId, mapType.getKeyType().notNull(), depth + 1); + FieldType keyType = keyField.getFieldType(); + keyField = + new Field( + keyField.getName(), + new FieldType( + keyType.isNullable(), + keyType.getType(), + keyType.getDictionary(), + Collections.singletonMap( + PARQUET_FIELD_ID, + String.valueOf( + SpecialFields.getMapKeyFieldId( + fieldId, depth + 1)))), + keyField.getChildren()); + + Field valueField = + toArrowField( + MapVector.VALUE_NAME, + fieldId, + mapType.getValueType().notNull(), + depth + 1); + FieldType valueType = valueField.getFieldType(); + valueField = + new Field( + valueField.getName(), + new FieldType( + valueType.isNullable(), + valueType.getType(), + valueType.getDictionary(), + Collections.singletonMap( + PARQUET_FIELD_ID, + String.valueOf( + SpecialFields.getMapValueFieldId( + fieldId, depth + 1)))), + valueField.getChildren()); + + FieldType structType = + new FieldType( + false, + Types.MinorType.STRUCT.getType(), + null, + Collections.singletonMap(PARQUET_FIELD_ID, String.valueOf(fieldId))); + Field mapField = + new Field( + MapVector.DATA_VECTOR_NAME, + // data vector, key vector and value vector CANNOT be null + structType, + Arrays.asList(keyField, valueField)); + + children = Collections.singletonList(mapField); } else if (dataType instanceof RowType) { RowType rowType = (RowType) dataType; - children = - rowType.getFields().stream() - .map(f -> toArrowField(f.name(), f.type())) - .collect(Collectors.toList()); + children = new ArrayList<>(); + for (DataField field : rowType.getFields()) { + children.add(toArrowField(field.name(), field.id(), field.type(), 0)); + } } return new Field(fieldName, fieldType, children); } diff --git a/paimon-arrow/src/main/java/org/apache/paimon/arrow/writer/ArrowFieldWriterFactoryVisitor.java b/paimon-arrow/src/main/java/org/apache/paimon/arrow/writer/ArrowFieldWriterFactoryVisitor.java index 9c1a55ec33ea..eef53009cec3 100644 --- a/paimon-arrow/src/main/java/org/apache/paimon/arrow/writer/ArrowFieldWriterFactoryVisitor.java +++ b/paimon-arrow/src/main/java/org/apache/paimon/arrow/writer/ArrowFieldWriterFactoryVisitor.java @@ -158,6 +158,7 @@ public ArrowFieldWriterFactory visit(MapType mapType) { ArrowFieldWriterFactory valueWriterFactory = mapType.getValueType().accept(this); return fieldVector -> { MapVector mapVector = (MapVector) fieldVector; + mapVector.reAlloc(); List keyValueVectors = mapVector.getDataVector().getChildrenFromFields(); return new ArrowFieldWriters.MapWriter( fieldVector, diff --git a/paimon-arrow/src/test/java/org/apache/paimon/arrow/ArrowUtilsTest.java b/paimon-arrow/src/test/java/org/apache/paimon/arrow/ArrowUtilsTest.java new file mode 100644 index 000000000000..319df13ba10b --- /dev/null +++ b/paimon-arrow/src/test/java/org/apache/paimon/arrow/ArrowUtilsTest.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.arrow; + +import org.apache.paimon.schema.Schema; +import org.apache.paimon.types.DataTypes; +import org.apache.paimon.types.RowType; + +import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.vector.types.pojo.Field; +import org.assertj.core.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.Random; + +/** Test for {@link ArrowUtils}. */ +public class ArrowUtilsTest { + + private static final Random RANDOM = new Random(); + + @Test + public void testParquetFieldId() { + Schema.Builder schemaBuilder = Schema.newBuilder(); + schemaBuilder.column("f0", DataTypes.INT()); + schemaBuilder.column("f1", DataTypes.INT()); + schemaBuilder.column("f2", DataTypes.SMALLINT()); + schemaBuilder.column("f3", DataTypes.STRING()); + schemaBuilder.column("f4", DataTypes.DOUBLE()); + schemaBuilder.column("f5", DataTypes.STRING()); + schemaBuilder.column("F6", DataTypes.STRING()); + schemaBuilder.column("f7", DataTypes.BOOLEAN()); + schemaBuilder.column("f8", DataTypes.DATE()); + schemaBuilder.column("f10", DataTypes.TIMESTAMP(6)); + schemaBuilder.column("f11", DataTypes.DECIMAL(7, 2)); + schemaBuilder.column("f12", DataTypes.BYTES()); + schemaBuilder.column("f13", DataTypes.FLOAT()); + schemaBuilder.column("f14", DataTypes.BINARY(10)); + schemaBuilder.column("f15", DataTypes.VARBINARY(10)); + schemaBuilder.column( + "f16", + DataTypes.ARRAY( + DataTypes.ROW( + DataTypes.FIELD(0, "f0", DataTypes.INT()), + DataTypes.FIELD(1, "f1", DataTypes.SMALLINT()), + DataTypes.FIELD(2, "f2", DataTypes.STRING()), + DataTypes.FIELD(3, "f3", DataTypes.DOUBLE()), + DataTypes.FIELD(4, "f4", DataTypes.BOOLEAN()), + DataTypes.FIELD(5, "f5", DataTypes.DATE()), + DataTypes.FIELD(6, "f6", DataTypes.TIMESTAMP(6)), + DataTypes.FIELD(7, "f7", DataTypes.DECIMAL(7, 2)), + DataTypes.FIELD(8, "f8", DataTypes.BYTES()), + DataTypes.FIELD(9, "f9", DataTypes.FLOAT()), + DataTypes.FIELD(10, "f10", DataTypes.BINARY(10))))); + + RowType rowType = schemaBuilder.build().rowType(); + + List fields = + ArrowUtils.createVectorSchemaRoot(rowType, new RootAllocator()) + .getSchema() + .getFields(); + + for (int i = 0; i < 16; i++) { + Assertions.assertThat( + Integer.parseInt( + fields.get(i).getMetadata().get(ArrowUtils.PARQUET_FIELD_ID))) + .isEqualTo(i); + } + + fields = fields.get(15).getChildren().get(0).getChildren(); + for (int i = 16; i < 26; i++) { + Assertions.assertThat( + Integer.parseInt( + fields.get(i - 16) + .getMetadata() + .get(ArrowUtils.PARQUET_FIELD_ID))) + .isEqualTo(i); + } + } +} diff --git a/paimon-arrow/src/test/java/org/apache/paimon/arrow/converter/ArrowBatchConverterTest.java b/paimon-arrow/src/test/java/org/apache/paimon/arrow/converter/ArrowBatchConverterTest.java index c726283f0044..aef589d91242 100644 --- a/paimon-arrow/src/test/java/org/apache/paimon/arrow/converter/ArrowBatchConverterTest.java +++ b/paimon-arrow/src/test/java/org/apache/paimon/arrow/converter/ArrowBatchConverterTest.java @@ -910,8 +910,9 @@ private boolean isVectorizedWithDv(RecordReader.RecordIterator iter private Object[] randomRowValues(boolean[] nullable) { Object[] values = new Object[18]; - values[0] = BinaryString.fromString(StringUtils.getRandomString(RND, 10, 10)); - values[1] = BinaryString.fromString(StringUtils.getRandomString(RND, 1, 20)); + // The orc char reader will trim the string. See TreeReaderFactory.CharTreeReader + values[0] = BinaryString.fromString(StringUtils.getRandomString(RND, 9, 9) + "A"); + values[1] = BinaryString.fromString(StringUtils.getRandomString(RND, 1, 19) + "A"); values[2] = RND.nextBoolean(); values[3] = randomBytes(10, 10); values[4] = randomBytes(1, 20); diff --git a/paimon-benchmark/paimon-cluster-benchmark/src/main/java/org/apache/paimon/benchmark/QueryRunner.java b/paimon-benchmark/paimon-cluster-benchmark/src/main/java/org/apache/paimon/benchmark/QueryRunner.java index b07cdef8465e..8bfe4b6c9c03 100644 --- a/paimon-benchmark/paimon-cluster-benchmark/src/main/java/org/apache/paimon/benchmark/QueryRunner.java +++ b/paimon-benchmark/paimon-cluster-benchmark/src/main/java/org/apache/paimon/benchmark/QueryRunner.java @@ -77,7 +77,7 @@ public Result run() { String sinkPathConfig = BenchmarkGlobalConfiguration.loadConfiguration() - .getString(BenchmarkOptions.SINK_PATH); + .get(BenchmarkOptions.SINK_PATH); if (sinkPathConfig == null) { throw new IllegalArgumentException( BenchmarkOptions.SINK_PATH.key() + " must be set"); diff --git a/paimon-benchmark/paimon-cluster-benchmark/src/main/java/org/apache/paimon/benchmark/metric/cpu/SysInfoLinux.java b/paimon-benchmark/paimon-cluster-benchmark/src/main/java/org/apache/paimon/benchmark/metric/cpu/SysInfoLinux.java index e4a5cfa570c7..041637c2dd2f 100644 --- a/paimon-benchmark/paimon-cluster-benchmark/src/main/java/org/apache/paimon/benchmark/metric/cpu/SysInfoLinux.java +++ b/paimon-benchmark/paimon-cluster-benchmark/src/main/java/org/apache/paimon/benchmark/metric/cpu/SysInfoLinux.java @@ -18,7 +18,12 @@ package org.apache.paimon.benchmark.metric.cpu; -import java.io.*; +import java.io.BufferedReader; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.Reader; import java.math.BigInteger; import java.nio.charset.Charset; import java.util.HashMap; diff --git a/paimon-benchmark/paimon-micro-benchmarks/src/test/java/org/apache/paimon/benchmark/cache/CacheManagerBenchmark.java b/paimon-benchmark/paimon-micro-benchmarks/src/test/java/org/apache/paimon/benchmark/cache/CacheManagerBenchmark.java new file mode 100644 index 000000000000..9a64322e0bde --- /dev/null +++ b/paimon-benchmark/paimon-micro-benchmarks/src/test/java/org/apache/paimon/benchmark/cache/CacheManagerBenchmark.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.benchmark.cache; + +import org.apache.paimon.benchmark.Benchmark; +import org.apache.paimon.io.cache.Cache; +import org.apache.paimon.io.cache.CacheKey; +import org.apache.paimon.io.cache.CacheManager; +import org.apache.paimon.options.MemorySize; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.File; +import java.io.RandomAccessFile; +import java.nio.file.Path; + +import static org.assertj.core.api.Assertions.assertThat; + +/** Benchmark for measure the performance for cache. */ +public class CacheManagerBenchmark { + @TempDir Path tempDir; + + @Test + public void testCache() throws Exception { + Benchmark benchmark = + new Benchmark("cache-benchmark", 100) + .setNumWarmupIters(1) + .setOutputPerIteration(true); + File file1 = new File(tempDir.toFile(), "cache-benchmark1"); + assertThat(file1.createNewFile()).isTrue(); + CacheKey key1 = CacheKey.forPageIndex(new RandomAccessFile(file1, "r"), 0, 0); + + File file2 = new File(tempDir.toFile(), "cache-benchmark2"); + assertThat(file2.createNewFile()).isTrue(); + CacheKey key2 = CacheKey.forPageIndex(new RandomAccessFile(file2, "r"), 0, 0); + + for (Cache.CacheType cacheType : Cache.CacheType.values()) { + CacheManager cacheManager = new CacheManager(cacheType, MemorySize.ofBytes(10), 0.1); + benchmark.addCase( + String.format("cache-%s", cacheType.toString()), + 5, + () -> { + try { + final int count = 10; + for (int i = 0; i < count; i++) { + cacheManager.getPage( + i < count / 2 ? key1 : key2, + key -> { + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + return new byte[6]; + }, + key -> {}); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + } + benchmark.run(); + } +} diff --git a/paimon-benchmark/paimon-micro-benchmarks/src/test/java/org/apache/paimon/benchmark/lookup/AbstractLookupBenchmark.java b/paimon-benchmark/paimon-micro-benchmarks/src/test/java/org/apache/paimon/benchmark/lookup/AbstractLookupBenchmark.java index 430a207f5c36..653bfee6cc00 100644 --- a/paimon-benchmark/paimon-micro-benchmarks/src/test/java/org/apache/paimon/benchmark/lookup/AbstractLookupBenchmark.java +++ b/paimon-benchmark/paimon-micro-benchmarks/src/test/java/org/apache/paimon/benchmark/lookup/AbstractLookupBenchmark.java @@ -27,11 +27,13 @@ import org.apache.paimon.options.MemorySize; import org.apache.paimon.types.IntType; import org.apache.paimon.types.RowType; +import org.apache.paimon.utils.BloomFilter; import org.apache.paimon.utils.Pair; import java.io.File; import java.io.IOException; import java.nio.file.Path; +import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.UUID; @@ -47,6 +49,15 @@ abstract class AbstractLookupBenchmark { new RowCompactedSerializer(RowType.of(new IntType())); private final GenericRow reusedKey = new GenericRow(1); + protected static List> getCountBloomList() { + List> countBloomList = new ArrayList<>(); + for (Integer recordCount : RECORD_COUNT_LIST) { + countBloomList.add(Arrays.asList(recordCount, false)); + countBloomList.add(Arrays.asList(recordCount, true)); + } + return countBloomList; + } + protected byte[][] generateSequenceInputs(int start, int end) { int count = end - start; byte[][] result = new byte[count][6]; @@ -74,7 +85,12 @@ protected byte[] intToByteArray(int value) { } protected Pair writeData( - Path tempDir, CoreOptions options, byte[][] inputs, int valueLength, boolean sameValue) + Path tempDir, + CoreOptions options, + byte[][] inputs, + int valueLength, + boolean sameValue, + boolean bloomFilterEnabled) throws IOException { byte[] value1 = new byte[valueLength]; byte[] value2 = new byte[valueLength]; @@ -86,8 +102,11 @@ protected Pair writeData( new CacheManager(MemorySize.ofMebiBytes(10)), keySerializer.createSliceComparator()); - File file = new File(tempDir.toFile(), UUID.randomUUID().toString()); - LookupStoreWriter writer = factory.createWriter(file, null); + String name = + String.format( + "%s-%s-%s", options.lookupLocalFileType(), valueLength, bloomFilterEnabled); + File file = new File(tempDir.toFile(), UUID.randomUUID() + "-" + name); + LookupStoreWriter writer = factory.createWriter(file, createBloomFiler(bloomFilterEnabled)); int i = 0; for (byte[] input : inputs) { if (sameValue) { @@ -104,4 +123,11 @@ protected Pair writeData( LookupStoreFactory.Context context = writer.close(); return Pair.of(file.getAbsolutePath(), context); } + + private BloomFilter.Builder createBloomFiler(boolean enabled) { + if (!enabled) { + return null; + } + return BloomFilter.builder(5000000, 0.01); + } } diff --git a/paimon-benchmark/paimon-micro-benchmarks/src/test/java/org/apache/paimon/benchmark/lookup/LookupReaderBenchmark.java b/paimon-benchmark/paimon-micro-benchmarks/src/test/java/org/apache/paimon/benchmark/lookup/LookupReaderBenchmark.java index 6327d703afe0..2d8de84327d4 100644 --- a/paimon-benchmark/paimon-micro-benchmarks/src/test/java/org/apache/paimon/benchmark/lookup/LookupReaderBenchmark.java +++ b/paimon-benchmark/paimon-micro-benchmarks/src/test/java/org/apache/paimon/benchmark/lookup/LookupReaderBenchmark.java @@ -49,25 +49,38 @@ public class LookupReaderBenchmark extends AbstractLookupBenchmark { private static final int QUERY_KEY_COUNT = 10000; private final int recordCount; + private final boolean bloomFilterEnabled; @TempDir Path tempDir; - public LookupReaderBenchmark(int recordCount) { - this.recordCount = recordCount; + public LookupReaderBenchmark(List countBloomList) { + this.recordCount = (Integer) countBloomList.get(0); + this.bloomFilterEnabled = (Boolean) countBloomList.get(1); } - @Parameters(name = "record-count-{0}") - public static List getVarSeg() { - return RECORD_COUNT_LIST; + @Parameters(name = "countBloom-{0}") + public static List> getVarSeg() { + return getCountBloomList(); } + /** Query data based on some keys that are definitely stored. */ @TestTemplate void testLookupReader() throws IOException { readLookupDataBenchmark( generateSequenceInputs(0, recordCount), - generateRandomInputs(0, recordCount, QUERY_KEY_COUNT)); + generateRandomInputs(0, recordCount, QUERY_KEY_COUNT), + false); } - private void readLookupDataBenchmark(byte[][] inputs, byte[][] randomInputs) + /** Query data based on some keys that are definitely not stored. */ + @TestTemplate + void testLookupReaderMiss() throws IOException { + readLookupDataBenchmark( + generateSequenceInputs(0, recordCount), + generateRandomInputs(recordCount + 1, recordCount * 2, QUERY_KEY_COUNT), + true); + } + + private void readLookupDataBenchmark(byte[][] inputs, byte[][] randomInputs, boolean nullResult) throws IOException { Benchmark benchmark = new Benchmark("reader-" + randomInputs.length, randomInputs.length) @@ -81,7 +94,7 @@ private void readLookupDataBenchmark(byte[][] inputs, byte[][] randomInputs) Collections.singletonMap( LOOKUP_LOCAL_FILE_TYPE.key(), fileType.name())); Pair pair = - writeData(tempDir, options, inputs, valueLength, false); + writeData(tempDir, options, inputs, valueLength, false, bloomFilterEnabled); benchmark.addCase( String.format( "%s-read-%dB-value-%d-num", @@ -89,7 +102,12 @@ private void readLookupDataBenchmark(byte[][] inputs, byte[][] randomInputs) 5, () -> { try { - readData(options, randomInputs, pair.getLeft(), pair.getRight()); + readData( + options, + randomInputs, + pair.getLeft(), + pair.getRight(), + nullResult); } catch (IOException e) { throw new RuntimeException(e); } @@ -104,19 +122,24 @@ private void readData( CoreOptions options, byte[][] randomInputs, String filePath, - LookupStoreFactory.Context context) + LookupStoreFactory.Context context, + boolean nullResult) throws IOException { LookupStoreFactory factory = LookupStoreFactory.create( options, - new CacheManager(MemorySize.ofMebiBytes(10)), + new CacheManager(MemorySize.ofMebiBytes(20), 0.5), new RowCompactedSerializer(RowType.of(new IntType())) .createSliceComparator()); File file = new File(filePath); LookupStoreReader reader = factory.createReader(file, context); for (byte[] input : randomInputs) { - assertThat(reader.lookup(input)).isNotNull(); + if (nullResult) { + assertThat(reader.lookup(input)).isNull(); + } else { + assertThat(reader.lookup(input)).isNotNull(); + } } reader.close(); } diff --git a/paimon-benchmark/paimon-micro-benchmarks/src/test/java/org/apache/paimon/benchmark/lookup/LookupWriterBenchmark.java b/paimon-benchmark/paimon-micro-benchmarks/src/test/java/org/apache/paimon/benchmark/lookup/LookupWriterBenchmark.java index fe4f1cd2493d..a590866ef175 100644 --- a/paimon-benchmark/paimon-micro-benchmarks/src/test/java/org/apache/paimon/benchmark/lookup/LookupWriterBenchmark.java +++ b/paimon-benchmark/paimon-micro-benchmarks/src/test/java/org/apache/paimon/benchmark/lookup/LookupWriterBenchmark.java @@ -38,15 +38,17 @@ @ExtendWith(ParameterizedTestExtension.class) public class LookupWriterBenchmark extends AbstractLookupBenchmark { private final int recordCount; + private final boolean bloomFilterEnabled; @TempDir Path tempDir; - public LookupWriterBenchmark(int recordCount) { - this.recordCount = recordCount; + public LookupWriterBenchmark(List countBloomList) { + this.recordCount = (Integer) countBloomList.get(0); + this.bloomFilterEnabled = (Boolean) countBloomList.get(1); } - @Parameters(name = "record-count-{0}") - public static List getVarSeg() { - return RECORD_COUNT_LIST; + @Parameters(name = "countBloom-{0}") + public static List> getVarSeg() { + return getCountBloomList(); } @TestTemplate @@ -78,7 +80,13 @@ private void writeLookupDataBenchmark(byte[][] inputs, boolean sameValue) { 5, () -> { try { - writeData(tempDir, options, inputs, valueLength, sameValue); + writeData( + tempDir, + options, + inputs, + valueLength, + sameValue, + bloomFilterEnabled); } catch (IOException e) { throw new RuntimeException(e); } diff --git a/paimon-codegen/pom.xml b/paimon-codegen/pom.xml index 8a43f390990f..4a40d487dc25 100644 --- a/paimon-codegen/pom.xml +++ b/paimon-codegen/pom.xml @@ -41,19 +41,19 @@ under the License. org.scala-lang scala-library - ${scala.version} + ${codegen.scala.version} org.scala-lang scala-reflect - ${scala.version} + ${codegen.scala.version} org.scala-lang scala-compiler - ${scala.version} + ${codegen.scala.version} @@ -86,6 +86,10 @@ under the License. + + + ${codegen.scala.version} + diff --git a/paimon-codegen/src/main/java/org/apache/paimon/codegen/CodeGeneratorImpl.java b/paimon-codegen/src/main/java/org/apache/paimon/codegen/CodeGeneratorImpl.java index 29096e96b206..5a6a27738b50 100644 --- a/paimon-codegen/src/main/java/org/apache/paimon/codegen/CodeGeneratorImpl.java +++ b/paimon-codegen/src/main/java/org/apache/paimon/codegen/CodeGeneratorImpl.java @@ -39,17 +39,17 @@ public GeneratedClass generateNormalizedKeyComputer( List inputTypes, int[] sortFields) { return new SortCodeGenerator( RowType.builder().fields(inputTypes).build(), - getAscendingSortSpec(sortFields)) + getAscendingSortSpec(sortFields, true)) .generateNormalizedKeyComputer("NormalizedKeyComputer"); } @Override public GeneratedClass generateRecordComparator( - List inputTypes, int[] sortFields) { + List inputTypes, int[] sortFields, boolean isAscendingOrder) { return ComparatorCodeGenerator.gen( "RecordComparator", RowType.builder().fields(inputTypes).build(), - getAscendingSortSpec(sortFields)); + getAscendingSortSpec(sortFields, isAscendingOrder)); } @Override @@ -59,10 +59,10 @@ public GeneratedClass generateRecordEqualiser( .generateRecordEqualiser("RecordEqualiser"); } - private SortSpec getAscendingSortSpec(int[] sortFields) { + private SortSpec getAscendingSortSpec(int[] sortFields, boolean isAscendingOrder) { SortSpec.SortSpecBuilder builder = SortSpec.builder(); for (int sortField : sortFields) { - builder.addField(sortField, true, false); + builder.addField(sortField, isAscendingOrder, false); } return builder.build(); } diff --git a/paimon-common/src/main/java/org/apache/paimon/CoreOptions.java b/paimon-common/src/main/java/org/apache/paimon/CoreOptions.java index 81422ba681e8..dd5632c18b42 100644 --- a/paimon-common/src/main/java/org/apache/paimon/CoreOptions.java +++ b/paimon-common/src/main/java/org/apache/paimon/CoreOptions.java @@ -28,6 +28,7 @@ import org.apache.paimon.fs.Path; import org.apache.paimon.lookup.LookupStrategy; import org.apache.paimon.options.ConfigOption; +import org.apache.paimon.options.ConfigOptions; import org.apache.paimon.options.ExpireConfig; import org.apache.paimon.options.MemorySize; import org.apache.paimon.options.Options; @@ -183,12 +184,25 @@ public class CoreOptions implements Serializable { .defaultValue("data-") .withDescription("Specify the file name prefix of data files."); + public static final ConfigOption DATA_FILE_PATH_DIRECTORY = + key("data-file.path-directory") + .stringType() + .noDefaultValue() + .withDescription("Specify the path directory of data files."); + public static final ConfigOption CHANGELOG_FILE_PREFIX = key("changelog-file.prefix") .stringType() .defaultValue("changelog-") .withDescription("Specify the file name prefix of changelog files."); + public static final ConfigOption FILE_SUFFIX_INCLUDE_COMPRESSION = + key("file.suffix.include.compression") + .booleanType() + .defaultValue(false) + .withDescription( + "Whether to add file compression type in the file name of data file and changelog file."); + public static final ConfigOption FILE_BLOCK_SIZE = key("file.block-size") .memoryType() @@ -249,6 +263,14 @@ public class CoreOptions implements Serializable { "The default partition name in case the dynamic partition" + " column value is null/empty string."); + public static final ConfigOption PARTITION_GENERATE_LEGCY_NAME = + key("partition.legacy-name") + .booleanType() + .defaultValue(true) + .withDescription( + "The legacy partition name is using `toString` fpr all types. If false, using " + + "cast to string for all types."); + public static final ConfigOption SNAPSHOT_NUM_RETAINED_MIN = key("snapshot.num-retained.min") .intType() @@ -299,7 +321,7 @@ public class CoreOptions implements Serializable { public static final ConfigOption SNAPSHOT_EXPIRE_LIMIT = key("snapshot.expire.limit") .intType() - .defaultValue(10) + .defaultValue(50) .withDescription( "The maximum number of snapshots allowed to expire at a time."); @@ -511,6 +533,12 @@ public class CoreOptions implements Serializable { .defaultValue(false) .withDescription("Whether to force a compaction before commit."); + public static final ConfigOption COMMIT_TIMEOUT = + key("commit.timeout") + .durationType() + .noDefaultValue() + .withDescription("Timeout duration of retry when commit failed."); + public static final ConfigOption COMMIT_MAX_RETRIES = key("commit.max-retries") .intType() @@ -600,6 +628,13 @@ public class CoreOptions implements Serializable { "The field that generates the sequence number for primary key table," + " the sequence number determines which data is the most recent."); + @Immutable + public static final ConfigOption SEQUENCE_FIELD_SORT_ORDER = + key("sequence.field.sort-order") + .enumType(SortOrder.class) + .defaultValue(SortOrder.ASCENDING) + .withDescription("Specify the order of sequence.field."); + @Immutable public static final ConfigOption PARTIAL_UPDATE_REMOVE_RECORD_ON_DELETE = key("partial-update.remove-record-on-delete") @@ -608,6 +643,14 @@ public class CoreOptions implements Serializable { .withDescription( "Whether to remove the whole row in partial-update engine when -D records are received."); + @Immutable + public static final ConfigOption PARTIAL_UPDATE_REMOVE_RECORD_ON_SEQUENCE_GROUP = + key("partial-update.remove-record-on-sequence-group") + .stringType() + .noDefaultValue() + .withDescription( + "When -D records of the given sequence groups are received, remove the whole row."); + @Immutable public static final ConfigOption ROWKIND_FIELD = key("rowkind.field") @@ -778,6 +821,12 @@ public class CoreOptions implements Serializable { .defaultValue(Duration.ofHours(1)) .withDescription("The check interval of partition expiration."); + public static final ConfigOption PARTITION_EXPIRATION_MAX_NUM = + key("partition.expiration-max-num") + .intType() + .defaultValue(100) + .withDescription("The default deleted num of partition expiration."); + public static final ConfigOption PARTITION_TIMESTAMP_FORMATTER = key("partition.timestamp-formatter") .stringType() @@ -818,6 +867,13 @@ public class CoreOptions implements Serializable { + "$hour:00:00'.")) .build()); + public static final ConfigOption PARTITION_MARK_DONE_WHEN_END_INPUT = + ConfigOptions.key("partition.end-input-to-done") + .booleanType() + .defaultValue(false) + .withDescription( + "Whether mark the done status to indicate that the data is ready when end input."); + public static final ConfigOption SCAN_PLAN_SORT_PARTITION = key("scan.plan-sort-partition") .booleanType() @@ -853,7 +909,7 @@ public class CoreOptions implements Serializable { public static final ConfigOption LOOKUP_LOCAL_FILE_TYPE = key("lookup.local-file-type") .enumType(LookupLocalFileType.class) - .defaultValue(LookupLocalFileType.HASH) + .defaultValue(LookupLocalFileType.SORT) .withDescription("The local file type for lookup."); public static final ConfigOption LOOKUP_HASH_LOAD_FACTOR = @@ -892,6 +948,13 @@ public class CoreOptions implements Serializable { .defaultValue(MemorySize.parse("256 mb")) .withDescription("Max memory size for lookup cache."); + public static final ConfigOption LOOKUP_CACHE_HIGH_PRIO_POOL_RATIO = + key("lookup.cache.high-priority-pool-ratio") + .doubleType() + .defaultValue(0.25) + .withDescription( + "The fraction of cache memory that is reserved for high-priority data like index, filter."); + public static final ConfigOption LOOKUP_CACHE_BLOOM_FILTER_ENABLED = key("lookup.cache.bloom.filter.enabled") .booleanType() @@ -1026,7 +1089,7 @@ public class CoreOptions implements Serializable { public static final String STATS_MODE_SUFFIX = "stats-mode"; public static final ConfigOption METADATA_STATS_MODE = - key("metadata." + STATS_MODE_SUFFIX) + key("metadata.stats-mode") .stringType() .defaultValue("truncate(16)") .withDescription( @@ -1053,6 +1116,22 @@ public class CoreOptions implements Serializable { + STATS_MODE_SUFFIX)) .build()); + public static final ConfigOption METADATA_STATS_DENSE_STORE = + key("metadata.stats-dense-store") + .booleanType() + .defaultValue(true) + .withDescription( + Description.builder() + .text( + "Whether to store statistic densely in metadata (manifest files), which" + + " will significantly reduce the storage size of metadata when the" + + " none statistic mode is set.") + .linebreak() + .text( + "Note, when this mode is enabled with 'metadata.stats-mode:none', the Paimon sdk in" + + " reading engine requires at least version 0.9.1 or 1.0.0 or higher.") + .build()); + public static final ConfigOption COMMIT_CALLBACKS = key("commit.callbacks") .stringType() @@ -1305,14 +1384,8 @@ public class CoreOptions implements Serializable { key("record-level.time-field") .stringType() .noDefaultValue() - .withDescription("Time field for record level expire."); - - public static final ConfigOption RECORD_LEVEL_TIME_FIELD_TYPE = - key("record-level.time-field-type") - .enumType(TimeFieldType.class) - .defaultValue(TimeFieldType.SECONDS_INT) .withDescription( - "Time field type for record level expire, it can be seconds-int,seconds-long or millis-long."); + "Time field for record level expire. It supports the following types: `timestamps in seconds with INT`,`timestamps in seconds with BIGINT`, `timestamps in milliseconds with BIGINT` or `timestamp`."); public static final ConfigOption FIELDS_DEFAULT_AGG_FUNC = key(FIELDS_PREFIX + "." + DEFAULT_AGG_FUNCTION) @@ -1342,14 +1415,6 @@ public class CoreOptions implements Serializable { .withDescription( "When need to lookup, commit will wait for compaction by lookup."); - public static final ConfigOption METADATA_ICEBERG_COMPATIBLE = - key("metadata.iceberg-compatible") - .booleanType() - .defaultValue(false) - .withDescription( - "When set to true, produce Iceberg metadata after a snapshot is committed, " - + "so that Iceberg readers can read Paimon's raw files."); - public static final ConfigOption DELETE_FILE_THREAD_NUM = key("delete-file.thread-num") .intType() @@ -1373,12 +1438,26 @@ public class CoreOptions implements Serializable { .withDescription( "Whether to enable asynchronous IO writing when writing files."); - @ExcludeFromDocumentation("Only used internally to support materialized table") - public static final ConfigOption MATERIALIZED_TABLE_SNAPSHOT = - key("materialized-table.snapshot") - .longType() + public static final ConfigOption OBJECT_LOCATION = + key("object-location") + .stringType() .noDefaultValue() - .withDescription("The snapshot specified for the materialized table"); + .withDescription("The object location for object table."); + + public static final ConfigOption MANIFEST_DELETE_FILE_DROP_STATS = + key("manifest.delete-file-drop-stats") + .booleanType() + .defaultValue(false) + .withDescription( + "For DELETE manifest entry in manifest file, drop stats to reduce memory and storage." + + " Default value is false only for compatibility of old reader."); + + public static final ConfigOption DATA_FILE_THIN_MODE = + key("data-file.thin-mode") + .booleanType() + .defaultValue(false) + .withDescription( + "Enable data file thin mode to avoid duplicate columns storage."); @ExcludeFromDocumentation("Only used internally to support materialized table") public static final ConfigOption MATERIALIZED_TABLE_DEFINITION_QUERY = @@ -1491,6 +1570,10 @@ public static Path path(Options options) { return new Path(options.get(PATH)); } + public TableType type() { + return options.get(TYPE); + } + public String formatType() { return normalizeFileFormat(options.get(FILE_FORMAT)); } @@ -1523,6 +1606,10 @@ public String partitionDefaultName() { return options.get(PARTITION_DEFAULT_NAME); } + public boolean legacyPartitionName() { + return options.get(PARTITION_GENERATE_LEGCY_NAME); + } + public boolean sortBySize() { return options.get(SORT_RANG_STRATEGY) == RangeStrategy.SIZE; } @@ -1536,6 +1623,11 @@ public static FileFormat createFileFormat(Options options, ConfigOption return FileFormat.fromIdentifier(formatIdentifier, options); } + public String objectLocation() { + checkArgument(type() == TableType.OBJECT_TABLE, "Only object table has object location!"); + return options.get(OBJECT_LOCATION); + } + public Map fileCompressionPerLevel() { Map levelCompressions = options.get(FILE_COMPRESSION_PER_LEVEL); return levelCompressions.entrySet().stream() @@ -1559,10 +1651,19 @@ public String dataFilePrefix() { return options.get(DATA_FILE_PREFIX); } + @Nullable + public String dataFilePathDirectory() { + return options.get(DATA_FILE_PATH_DIRECTORY); + } + public String changelogFilePrefix() { return options.get(CHANGELOG_FILE_PREFIX); } + public boolean fileSuffixIncludeCompression() { + return options.get(FILE_SUFFIX_INCLUDE_COMPRESSION); + } + public String fieldsDefaultFunc() { return options.get(FIELDS_DEFAULT_AGG_FUNC); } @@ -1574,6 +1675,10 @@ public static String createCommitUser(Options options) { : commitUserPrefix + "_" + UUID.randomUUID(); } + public String createCommitUser() { + return createCommitUser(options); + } + public boolean definedAggFunc() { if (options.contains(FIELDS_DEFAULT_AGG_FUNC)) { return true; @@ -1800,6 +1905,10 @@ public MemorySize lookupCacheMaxMemory() { return options.get(LOOKUP_CACHE_MAX_MEMORY_SIZE); } + public double lookupCacheHighPrioPoolRatio() { + return options.get(LOOKUP_CACHE_HIGH_PRIO_POOL_RATIO); + } + public long targetFileSize(boolean hasPrimaryKey) { return options.getOptional(TARGET_FILE_SIZE) .orElse(hasPrimaryKey ? VALUE_128_MB : VALUE_256_MB) @@ -1844,6 +1953,12 @@ public boolean commitForceCompact() { return options.get(COMMIT_FORCE_COMPACT); } + public long commitTimeout() { + return options.get(COMMIT_TIMEOUT) == null + ? Long.MAX_VALUE + : options.get(COMMIT_TIMEOUT).toMillis(); + } + public int commitMaxRetries() { return options.get(COMMIT_MAX_RETRIES); } @@ -1876,6 +1991,10 @@ public boolean needLookup() { return lookupStrategy().needLookup; } + public boolean manifestDeleteFileDropStats() { + return options.get(MANIFEST_DELETE_FILE_DROP_STATS); + } + public LookupStrategy lookupStrategy() { return LookupStrategy.from( mergeEngine().equals(MergeEngine.FIRST_ROW), @@ -2011,6 +2130,10 @@ public List sequenceField() { .orElse(Collections.emptyList()); } + public boolean sequenceFieldSortOrderIsAscending() { + return options.get(SEQUENCE_FIELD_SORT_ORDER) == SortOrder.ASCENDING; + } + public boolean partialUpdateRemoveRecordOnDelete() { return options.get(PARTIAL_UPDATE_REMOVE_RECORD_ON_DELETE); } @@ -2039,6 +2162,10 @@ public Duration partitionExpireCheckInterval() { return options.get(PARTITION_EXPIRATION_CHECK_INTERVAL); } + public int partitionExpireMaxNum() { + return options.get(PARTITION_EXPIRATION_MAX_NUM); + } + public PartitionExpireStrategy partitionExpireStrategy() { return options.get(PARTITION_EXPIRATION_STRATEGY); } @@ -2152,7 +2279,7 @@ private Map callbacks( Map result = new HashMap<>(); for (String className : options.get(callbacks).split(",")) { className = className.trim(); - if (className.length() == 0) { + if (className.isEmpty()) { continue; } @@ -2220,11 +2347,6 @@ public String recordLevelTimeField() { return options.get(RECORD_LEVEL_TIME_FIELD); } - @Nullable - public TimeFieldType recordLevelTimeFieldType() { - return options.get(RECORD_LEVEL_TIME_FIELD_TYPE); - } - public boolean prepareCommitWaitCompaction() { if (!needLookup()) { return false; @@ -2237,8 +2359,12 @@ public boolean asyncFileWrite() { return options.get(ASYNC_FILE_WRITE); } - public boolean metadataIcebergCompatible() { - return options.get(METADATA_ICEBERG_COMPATIBLE); + public boolean statsDenseStore() { + return options.get(METADATA_STATS_DENSE_STORE); + } + + public boolean dataFileThinMode() { + return options.get(DATA_FILE_THIN_MODE); } /** Specifies the merge engine for table with primary key. */ @@ -2350,6 +2476,31 @@ public InlineElement getDescription() { } } + /** Specifies the sort order for field sequence id. */ + public enum SortOrder implements DescribedEnum { + ASCENDING("ascending", "specifies sequence.field sort order is ascending."), + + DESCENDING("descending", "specifies sequence.field sort order is descending."); + + private final String value; + private final String description; + + SortOrder(String value, String description) { + this.value = value; + this.description = description; + } + + @Override + public String toString() { + return value; + } + + @Override + public InlineElement getDescription() { + return text(description); + } + } + /** Specifies the log consistency mode for table. */ public enum LogConsistency implements DescribedEnum { TRANSACTIONAL( @@ -2848,33 +2999,6 @@ public InlineElement getDescription() { } } - /** Time field type for record level expire. */ - public enum TimeFieldType implements DescribedEnum { - SECONDS_INT("seconds-int", "Timestamps in seconds with INT field type."), - - SECONDS_LONG("seconds-long", "Timestamps in seconds with BIGINT field type."), - - MILLIS_LONG("millis-long", "Timestamps in milliseconds with BIGINT field type."); - - private final String value; - private final String description; - - TimeFieldType(String value, String description) { - this.value = value; - this.description = description; - } - - @Override - public String toString() { - return value; - } - - @Override - public InlineElement getDescription() { - return text(description); - } - } - /** The time unit of materialized table freshness. */ public enum MaterializedTableIntervalFreshnessTimeUnit { SECOND, diff --git a/paimon-common/src/main/java/org/apache/paimon/TableType.java b/paimon-common/src/main/java/org/apache/paimon/TableType.java index d690d5db3700..d9ac020f793c 100644 --- a/paimon-common/src/main/java/org/apache/paimon/TableType.java +++ b/paimon-common/src/main/java/org/apache/paimon/TableType.java @@ -29,7 +29,12 @@ public enum TableType implements DescribedEnum { FORMAT_TABLE( "format-table", "A file format table refers to a directory that contains multiple files of the same format."), - MATERIALIZED_TABLE("materialized-table", "A materialized table."); + MATERIALIZED_TABLE( + "materialized-table", + "A materialized table combines normal Paimon table and materialized SQL."), + OBJECT_TABLE( + "object-table", "A object table combines normal Paimon table and object location."); + private final String value; private final String description; diff --git a/paimon-core/src/main/java/org/apache/paimon/casting/AbstractCastRule.java b/paimon-common/src/main/java/org/apache/paimon/casting/AbstractCastRule.java similarity index 100% rename from paimon-core/src/main/java/org/apache/paimon/casting/AbstractCastRule.java rename to paimon-common/src/main/java/org/apache/paimon/casting/AbstractCastRule.java diff --git a/paimon-core/src/main/java/org/apache/paimon/casting/BinaryToBinaryCastRule.java b/paimon-common/src/main/java/org/apache/paimon/casting/BinaryToBinaryCastRule.java similarity index 100% rename from paimon-core/src/main/java/org/apache/paimon/casting/BinaryToBinaryCastRule.java rename to paimon-common/src/main/java/org/apache/paimon/casting/BinaryToBinaryCastRule.java diff --git a/paimon-core/src/main/java/org/apache/paimon/casting/BinaryToStringCastRule.java b/paimon-common/src/main/java/org/apache/paimon/casting/BinaryToStringCastRule.java similarity index 100% rename from paimon-core/src/main/java/org/apache/paimon/casting/BinaryToStringCastRule.java rename to paimon-common/src/main/java/org/apache/paimon/casting/BinaryToStringCastRule.java diff --git a/paimon-core/src/main/java/org/apache/paimon/casting/BooleanToNumericCastRule.java b/paimon-common/src/main/java/org/apache/paimon/casting/BooleanToNumericCastRule.java similarity index 100% rename from paimon-core/src/main/java/org/apache/paimon/casting/BooleanToNumericCastRule.java rename to paimon-common/src/main/java/org/apache/paimon/casting/BooleanToNumericCastRule.java diff --git a/paimon-core/src/main/java/org/apache/paimon/casting/BooleanToStringCastRule.java b/paimon-common/src/main/java/org/apache/paimon/casting/BooleanToStringCastRule.java similarity index 100% rename from paimon-core/src/main/java/org/apache/paimon/casting/BooleanToStringCastRule.java rename to paimon-common/src/main/java/org/apache/paimon/casting/BooleanToStringCastRule.java diff --git a/paimon-common/src/main/java/org/apache/paimon/casting/CastElementGetter.java b/paimon-common/src/main/java/org/apache/paimon/casting/CastElementGetter.java new file mode 100644 index 000000000000..b8a91f572a35 --- /dev/null +++ b/paimon-common/src/main/java/org/apache/paimon/casting/CastElementGetter.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.casting; + +import org.apache.paimon.data.InternalArray; + +/** Get element from array and cast it according to specific {@link CastExecutor}. */ +public class CastElementGetter { + + private final InternalArray.ElementGetter elementGetter; + private final CastExecutor castExecutor; + + @SuppressWarnings("unchecked") + public CastElementGetter( + InternalArray.ElementGetter elementGetter, CastExecutor castExecutor) { + this.elementGetter = elementGetter; + this.castExecutor = (CastExecutor) castExecutor; + } + + @SuppressWarnings("unchecked") + public V getElementOrNull(InternalArray array, int pos) { + Object value = elementGetter.getElementOrNull(array, pos); + return value == null ? null : (V) castExecutor.cast(value); + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/casting/CastExecutor.java b/paimon-common/src/main/java/org/apache/paimon/casting/CastExecutor.java similarity index 100% rename from paimon-core/src/main/java/org/apache/paimon/casting/CastExecutor.java rename to paimon-common/src/main/java/org/apache/paimon/casting/CastExecutor.java diff --git a/paimon-core/src/main/java/org/apache/paimon/casting/CastExecutors.java b/paimon-common/src/main/java/org/apache/paimon/casting/CastExecutors.java similarity index 100% rename from paimon-core/src/main/java/org/apache/paimon/casting/CastExecutors.java rename to paimon-common/src/main/java/org/apache/paimon/casting/CastExecutors.java diff --git a/paimon-core/src/main/java/org/apache/paimon/casting/CastFieldGetter.java b/paimon-common/src/main/java/org/apache/paimon/casting/CastFieldGetter.java similarity index 95% rename from paimon-core/src/main/java/org/apache/paimon/casting/CastFieldGetter.java rename to paimon-common/src/main/java/org/apache/paimon/casting/CastFieldGetter.java index 02168300a842..208ef5f30f5b 100644 --- a/paimon-core/src/main/java/org/apache/paimon/casting/CastFieldGetter.java +++ b/paimon-common/src/main/java/org/apache/paimon/casting/CastFieldGetter.java @@ -24,14 +24,17 @@ * Get field value from row with given pos and cast it according to specific {@link CastExecutor}. */ public class CastFieldGetter { + private final InternalRow.FieldGetter fieldGetter; private final CastExecutor castExecutor; + @SuppressWarnings("unchecked") public CastFieldGetter(InternalRow.FieldGetter fieldGetter, CastExecutor castExecutor) { this.fieldGetter = fieldGetter; this.castExecutor = (CastExecutor) castExecutor; } + @SuppressWarnings("unchecked") public V getFieldOrNull(InternalRow row) { Object value = fieldGetter.getFieldOrNull(row); return value == null ? null : (V) castExecutor.cast(value); diff --git a/paimon-core/src/main/java/org/apache/paimon/casting/CastRule.java b/paimon-common/src/main/java/org/apache/paimon/casting/CastRule.java similarity index 100% rename from paimon-core/src/main/java/org/apache/paimon/casting/CastRule.java rename to paimon-common/src/main/java/org/apache/paimon/casting/CastRule.java diff --git a/paimon-core/src/main/java/org/apache/paimon/casting/CastRulePredicate.java b/paimon-common/src/main/java/org/apache/paimon/casting/CastRulePredicate.java similarity index 100% rename from paimon-core/src/main/java/org/apache/paimon/casting/CastRulePredicate.java rename to paimon-common/src/main/java/org/apache/paimon/casting/CastRulePredicate.java diff --git a/paimon-common/src/main/java/org/apache/paimon/casting/CastedArray.java b/paimon-common/src/main/java/org/apache/paimon/casting/CastedArray.java new file mode 100644 index 000000000000..778b11d1f887 --- /dev/null +++ b/paimon-common/src/main/java/org/apache/paimon/casting/CastedArray.java @@ -0,0 +1,201 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.casting; + +import org.apache.paimon.data.BinaryString; +import org.apache.paimon.data.Decimal; +import org.apache.paimon.data.InternalArray; +import org.apache.paimon.data.InternalMap; +import org.apache.paimon.data.InternalRow; +import org.apache.paimon.data.Timestamp; + +/** + * An implementation of {@link InternalArray} which provides a casted view of the underlying {@link + * InternalArray}. + * + *

    It reads data from underlying {@link InternalArray} according to source logical type and casts + * it with specific {@link CastExecutor}. + */ +public class CastedArray implements InternalArray { + + private final CastElementGetter castElementGetter; + private InternalArray array; + + protected CastedArray(CastElementGetter castElementGetter) { + this.castElementGetter = castElementGetter; + } + + /** + * Replaces the underlying {@link InternalArray} backing this {@link CastedArray}. + * + *

    This method replaces the array in place and does not return a new object. This is done for + * performance reasons. + */ + public static CastedArray from(CastElementGetter castElementGetter) { + return new CastedArray(castElementGetter); + } + + public CastedArray replaceArray(InternalArray array) { + this.array = array; + return this; + } + + @Override + public int size() { + return array.size(); + } + + @Override + public boolean[] toBooleanArray() { + boolean[] result = new boolean[size()]; + for (int i = 0; i < result.length; i++) { + result[i] = castElementGetter.getElementOrNull(array, i); + } + return result; + } + + @Override + public byte[] toByteArray() { + byte[] result = new byte[size()]; + for (int i = 0; i < result.length; i++) { + result[i] = castElementGetter.getElementOrNull(array, i); + } + return result; + } + + @Override + public short[] toShortArray() { + short[] result = new short[size()]; + for (int i = 0; i < result.length; i++) { + result[i] = castElementGetter.getElementOrNull(array, i); + } + return result; + } + + @Override + public int[] toIntArray() { + int[] result = new int[size()]; + for (int i = 0; i < result.length; i++) { + result[i] = castElementGetter.getElementOrNull(array, i); + } + return result; + } + + @Override + public long[] toLongArray() { + long[] result = new long[size()]; + for (int i = 0; i < result.length; i++) { + result[i] = castElementGetter.getElementOrNull(array, i); + } + return result; + } + + @Override + public float[] toFloatArray() { + float[] result = new float[size()]; + for (int i = 0; i < result.length; i++) { + result[i] = castElementGetter.getElementOrNull(array, i); + } + return result; + } + + @Override + public double[] toDoubleArray() { + double[] result = new double[size()]; + for (int i = 0; i < result.length; i++) { + result[i] = castElementGetter.getElementOrNull(array, i); + } + return result; + } + + @Override + public boolean isNullAt(int pos) { + return castElementGetter.getElementOrNull(array, pos) == null; + } + + @Override + public boolean getBoolean(int pos) { + return castElementGetter.getElementOrNull(array, pos); + } + + @Override + public byte getByte(int pos) { + return castElementGetter.getElementOrNull(array, pos); + } + + @Override + public short getShort(int pos) { + return castElementGetter.getElementOrNull(array, pos); + } + + @Override + public int getInt(int pos) { + return castElementGetter.getElementOrNull(array, pos); + } + + @Override + public long getLong(int pos) { + return castElementGetter.getElementOrNull(array, pos); + } + + @Override + public float getFloat(int pos) { + return castElementGetter.getElementOrNull(array, pos); + } + + @Override + public double getDouble(int pos) { + return castElementGetter.getElementOrNull(array, pos); + } + + @Override + public BinaryString getString(int pos) { + return castElementGetter.getElementOrNull(array, pos); + } + + @Override + public Decimal getDecimal(int pos, int precision, int scale) { + return castElementGetter.getElementOrNull(array, pos); + } + + @Override + public Timestamp getTimestamp(int pos, int precision) { + return castElementGetter.getElementOrNull(array, pos); + } + + @Override + public byte[] getBinary(int pos) { + return castElementGetter.getElementOrNull(array, pos); + } + + @Override + public InternalArray getArray(int pos) { + return castElementGetter.getElementOrNull(array, pos); + } + + @Override + public InternalMap getMap(int pos) { + return castElementGetter.getElementOrNull(array, pos); + } + + @Override + public InternalRow getRow(int pos, int numFields) { + return castElementGetter.getElementOrNull(array, pos); + } +} diff --git a/paimon-common/src/main/java/org/apache/paimon/casting/CastedMap.java b/paimon-common/src/main/java/org/apache/paimon/casting/CastedMap.java new file mode 100644 index 000000000000..4068407ca71c --- /dev/null +++ b/paimon-common/src/main/java/org/apache/paimon/casting/CastedMap.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.casting; + +import org.apache.paimon.data.InternalArray; +import org.apache.paimon.data.InternalMap; + +/** + * An implementation of {@link InternalMap} which provides a casted view of the underlying {@link + * InternalMap}. + * + *

    It reads data from underlying {@link InternalMap} according to source logical type and casts + * it with specific {@link CastExecutor}. + */ +public class CastedMap implements InternalMap { + + private final CastedArray castedValueArray; + private InternalMap map; + + protected CastedMap(CastElementGetter castValueGetter) { + this.castedValueArray = CastedArray.from(castValueGetter); + } + + /** + * Replaces the underlying {@link InternalMap} backing this {@link CastedMap}. + * + *

    This method replaces the map in place and does not return a new object. This is done for + * performance reasons. + */ + public static CastedMap from(CastElementGetter castValueGetter) { + return new CastedMap(castValueGetter); + } + + public CastedMap replaceMap(InternalMap map) { + this.castedValueArray.replaceArray(map.valueArray()); + this.map = map; + return this; + } + + @Override + public int size() { + return map.size(); + } + + @Override + public InternalArray keyArray() { + return map.keyArray(); + } + + @Override + public InternalArray valueArray() { + return castedValueArray; + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/casting/CastedRow.java b/paimon-common/src/main/java/org/apache/paimon/casting/CastedRow.java similarity index 98% rename from paimon-core/src/main/java/org/apache/paimon/casting/CastedRow.java rename to paimon-common/src/main/java/org/apache/paimon/casting/CastedRow.java index 25c5744255ef..f9216d10b3a8 100644 --- a/paimon-core/src/main/java/org/apache/paimon/casting/CastedRow.java +++ b/paimon-common/src/main/java/org/apache/paimon/casting/CastedRow.java @@ -34,8 +34,6 @@ * *

    It reads data from underlying {@link InternalRow} according to source logical type and casts * it with specific {@link CastExecutor}. - * - *

    Note: This class supports only top-level castings, not nested castings. */ public class CastedRow implements InternalRow { diff --git a/paimon-core/src/main/java/org/apache/paimon/casting/DateToStringCastRule.java b/paimon-common/src/main/java/org/apache/paimon/casting/DateToStringCastRule.java similarity index 100% rename from paimon-core/src/main/java/org/apache/paimon/casting/DateToStringCastRule.java rename to paimon-common/src/main/java/org/apache/paimon/casting/DateToStringCastRule.java diff --git a/paimon-core/src/main/java/org/apache/paimon/casting/DateToTimestampCastRule.java b/paimon-common/src/main/java/org/apache/paimon/casting/DateToTimestampCastRule.java similarity index 100% rename from paimon-core/src/main/java/org/apache/paimon/casting/DateToTimestampCastRule.java rename to paimon-common/src/main/java/org/apache/paimon/casting/DateToTimestampCastRule.java diff --git a/paimon-core/src/main/java/org/apache/paimon/casting/DecimalToDecimalCastRule.java b/paimon-common/src/main/java/org/apache/paimon/casting/DecimalToDecimalCastRule.java similarity index 100% rename from paimon-core/src/main/java/org/apache/paimon/casting/DecimalToDecimalCastRule.java rename to paimon-common/src/main/java/org/apache/paimon/casting/DecimalToDecimalCastRule.java diff --git a/paimon-core/src/main/java/org/apache/paimon/casting/DecimalToNumericPrimitiveCastRule.java b/paimon-common/src/main/java/org/apache/paimon/casting/DecimalToNumericPrimitiveCastRule.java similarity index 100% rename from paimon-core/src/main/java/org/apache/paimon/casting/DecimalToNumericPrimitiveCastRule.java rename to paimon-common/src/main/java/org/apache/paimon/casting/DecimalToNumericPrimitiveCastRule.java diff --git a/paimon-core/src/main/java/org/apache/paimon/casting/DefaultValueRow.java b/paimon-common/src/main/java/org/apache/paimon/casting/DefaultValueRow.java similarity index 100% rename from paimon-core/src/main/java/org/apache/paimon/casting/DefaultValueRow.java rename to paimon-common/src/main/java/org/apache/paimon/casting/DefaultValueRow.java diff --git a/paimon-core/src/main/java/org/apache/paimon/casting/NumericPrimitiveCastRule.java b/paimon-common/src/main/java/org/apache/paimon/casting/NumericPrimitiveCastRule.java similarity index 100% rename from paimon-core/src/main/java/org/apache/paimon/casting/NumericPrimitiveCastRule.java rename to paimon-common/src/main/java/org/apache/paimon/casting/NumericPrimitiveCastRule.java diff --git a/paimon-core/src/main/java/org/apache/paimon/casting/NumericPrimitiveToDecimalCastRule.java b/paimon-common/src/main/java/org/apache/paimon/casting/NumericPrimitiveToDecimalCastRule.java similarity index 100% rename from paimon-core/src/main/java/org/apache/paimon/casting/NumericPrimitiveToDecimalCastRule.java rename to paimon-common/src/main/java/org/apache/paimon/casting/NumericPrimitiveToDecimalCastRule.java diff --git a/paimon-core/src/main/java/org/apache/paimon/casting/NumericPrimitiveToTimestamp.java b/paimon-common/src/main/java/org/apache/paimon/casting/NumericPrimitiveToTimestamp.java similarity index 100% rename from paimon-core/src/main/java/org/apache/paimon/casting/NumericPrimitiveToTimestamp.java rename to paimon-common/src/main/java/org/apache/paimon/casting/NumericPrimitiveToTimestamp.java diff --git a/paimon-core/src/main/java/org/apache/paimon/casting/NumericToBooleanCastRule.java b/paimon-common/src/main/java/org/apache/paimon/casting/NumericToBooleanCastRule.java similarity index 97% rename from paimon-core/src/main/java/org/apache/paimon/casting/NumericToBooleanCastRule.java rename to paimon-common/src/main/java/org/apache/paimon/casting/NumericToBooleanCastRule.java index 06fa89fe3599..5b47741e6d35 100644 --- a/paimon-core/src/main/java/org/apache/paimon/casting/NumericToBooleanCastRule.java +++ b/paimon-common/src/main/java/org/apache/paimon/casting/NumericToBooleanCastRule.java @@ -37,6 +37,6 @@ private NumericToBooleanCastRule() { @Override public CastExecutor create(DataType inputType, DataType targetType) { - return value -> value.intValue() != 0; + return value -> value.longValue() != 0; } } diff --git a/paimon-core/src/main/java/org/apache/paimon/casting/NumericToStringCastRule.java b/paimon-common/src/main/java/org/apache/paimon/casting/NumericToStringCastRule.java similarity index 100% rename from paimon-core/src/main/java/org/apache/paimon/casting/NumericToStringCastRule.java rename to paimon-common/src/main/java/org/apache/paimon/casting/NumericToStringCastRule.java diff --git a/paimon-core/src/main/java/org/apache/paimon/casting/StringToBinaryCastRule.java b/paimon-common/src/main/java/org/apache/paimon/casting/StringToBinaryCastRule.java similarity index 100% rename from paimon-core/src/main/java/org/apache/paimon/casting/StringToBinaryCastRule.java rename to paimon-common/src/main/java/org/apache/paimon/casting/StringToBinaryCastRule.java diff --git a/paimon-core/src/main/java/org/apache/paimon/casting/StringToBooleanCastRule.java b/paimon-common/src/main/java/org/apache/paimon/casting/StringToBooleanCastRule.java similarity index 100% rename from paimon-core/src/main/java/org/apache/paimon/casting/StringToBooleanCastRule.java rename to paimon-common/src/main/java/org/apache/paimon/casting/StringToBooleanCastRule.java diff --git a/paimon-core/src/main/java/org/apache/paimon/casting/StringToDateCastRule.java b/paimon-common/src/main/java/org/apache/paimon/casting/StringToDateCastRule.java similarity index 100% rename from paimon-core/src/main/java/org/apache/paimon/casting/StringToDateCastRule.java rename to paimon-common/src/main/java/org/apache/paimon/casting/StringToDateCastRule.java diff --git a/paimon-core/src/main/java/org/apache/paimon/casting/StringToDecimalCastRule.java b/paimon-common/src/main/java/org/apache/paimon/casting/StringToDecimalCastRule.java similarity index 100% rename from paimon-core/src/main/java/org/apache/paimon/casting/StringToDecimalCastRule.java rename to paimon-common/src/main/java/org/apache/paimon/casting/StringToDecimalCastRule.java diff --git a/paimon-core/src/main/java/org/apache/paimon/casting/StringToNumericPrimitiveCastRule.java b/paimon-common/src/main/java/org/apache/paimon/casting/StringToNumericPrimitiveCastRule.java similarity index 100% rename from paimon-core/src/main/java/org/apache/paimon/casting/StringToNumericPrimitiveCastRule.java rename to paimon-common/src/main/java/org/apache/paimon/casting/StringToNumericPrimitiveCastRule.java diff --git a/paimon-core/src/main/java/org/apache/paimon/casting/StringToStringCastRule.java b/paimon-common/src/main/java/org/apache/paimon/casting/StringToStringCastRule.java similarity index 100% rename from paimon-core/src/main/java/org/apache/paimon/casting/StringToStringCastRule.java rename to paimon-common/src/main/java/org/apache/paimon/casting/StringToStringCastRule.java diff --git a/paimon-core/src/main/java/org/apache/paimon/casting/StringToTimeCastRule.java b/paimon-common/src/main/java/org/apache/paimon/casting/StringToTimeCastRule.java similarity index 100% rename from paimon-core/src/main/java/org/apache/paimon/casting/StringToTimeCastRule.java rename to paimon-common/src/main/java/org/apache/paimon/casting/StringToTimeCastRule.java diff --git a/paimon-core/src/main/java/org/apache/paimon/casting/StringToTimestampCastRule.java b/paimon-common/src/main/java/org/apache/paimon/casting/StringToTimestampCastRule.java similarity index 100% rename from paimon-core/src/main/java/org/apache/paimon/casting/StringToTimestampCastRule.java rename to paimon-common/src/main/java/org/apache/paimon/casting/StringToTimestampCastRule.java diff --git a/paimon-core/src/main/java/org/apache/paimon/casting/TimeToStringCastRule.java b/paimon-common/src/main/java/org/apache/paimon/casting/TimeToStringCastRule.java similarity index 100% rename from paimon-core/src/main/java/org/apache/paimon/casting/TimeToStringCastRule.java rename to paimon-common/src/main/java/org/apache/paimon/casting/TimeToStringCastRule.java diff --git a/paimon-core/src/main/java/org/apache/paimon/casting/TimeToTimestampCastRule.java b/paimon-common/src/main/java/org/apache/paimon/casting/TimeToTimestampCastRule.java similarity index 100% rename from paimon-core/src/main/java/org/apache/paimon/casting/TimeToTimestampCastRule.java rename to paimon-common/src/main/java/org/apache/paimon/casting/TimeToTimestampCastRule.java diff --git a/paimon-core/src/main/java/org/apache/paimon/casting/TimestampToDateCastRule.java b/paimon-common/src/main/java/org/apache/paimon/casting/TimestampToDateCastRule.java similarity index 100% rename from paimon-core/src/main/java/org/apache/paimon/casting/TimestampToDateCastRule.java rename to paimon-common/src/main/java/org/apache/paimon/casting/TimestampToDateCastRule.java diff --git a/paimon-core/src/main/java/org/apache/paimon/casting/TimestampToNumericPrimitiveCastRule.java b/paimon-common/src/main/java/org/apache/paimon/casting/TimestampToNumericPrimitiveCastRule.java similarity index 100% rename from paimon-core/src/main/java/org/apache/paimon/casting/TimestampToNumericPrimitiveCastRule.java rename to paimon-common/src/main/java/org/apache/paimon/casting/TimestampToNumericPrimitiveCastRule.java diff --git a/paimon-core/src/main/java/org/apache/paimon/casting/TimestampToStringCastRule.java b/paimon-common/src/main/java/org/apache/paimon/casting/TimestampToStringCastRule.java similarity index 100% rename from paimon-core/src/main/java/org/apache/paimon/casting/TimestampToStringCastRule.java rename to paimon-common/src/main/java/org/apache/paimon/casting/TimestampToStringCastRule.java diff --git a/paimon-core/src/main/java/org/apache/paimon/casting/TimestampToTimeCastRule.java b/paimon-common/src/main/java/org/apache/paimon/casting/TimestampToTimeCastRule.java similarity index 100% rename from paimon-core/src/main/java/org/apache/paimon/casting/TimestampToTimeCastRule.java rename to paimon-common/src/main/java/org/apache/paimon/casting/TimestampToTimeCastRule.java diff --git a/paimon-core/src/main/java/org/apache/paimon/casting/TimestampToTimestampCastRule.java b/paimon-common/src/main/java/org/apache/paimon/casting/TimestampToTimestampCastRule.java similarity index 100% rename from paimon-core/src/main/java/org/apache/paimon/casting/TimestampToTimestampCastRule.java rename to paimon-common/src/main/java/org/apache/paimon/casting/TimestampToTimestampCastRule.java diff --git a/paimon-common/src/main/java/org/apache/paimon/codegen/CodeGenerator.java b/paimon-common/src/main/java/org/apache/paimon/codegen/CodeGenerator.java index e137619143a3..324a8d726fb2 100644 --- a/paimon-common/src/main/java/org/apache/paimon/codegen/CodeGenerator.java +++ b/paimon-common/src/main/java/org/apache/paimon/codegen/CodeGenerator.java @@ -44,9 +44,10 @@ GeneratedClass generateNormalizedKeyComputer( * @param inputTypes input types. * @param sortFields the sort key fields. Records are compared by the first field, then the * second field, then the third field and so on. All fields are compared in ascending order. + * @param isAscendingOrder decide the sort key fields order whether is ascending */ GeneratedClass generateRecordComparator( - List inputTypes, int[] sortFields); + List inputTypes, int[] sortFields, boolean isAscendingOrder); /** Generate a {@link RecordEqualiser} with fields. */ GeneratedClass generateRecordEqualiser( diff --git a/paimon-common/src/main/java/org/apache/paimon/data/BinaryRow.java b/paimon-common/src/main/java/org/apache/paimon/data/BinaryRow.java index 7068e25e6e60..d08c580be5ff 100644 --- a/paimon-common/src/main/java/org/apache/paimon/data/BinaryRow.java +++ b/paimon-common/src/main/java/org/apache/paimon/data/BinaryRow.java @@ -21,7 +21,11 @@ import org.apache.paimon.annotation.Public; import org.apache.paimon.memory.MemorySegment; import org.apache.paimon.memory.MemorySegmentUtils; +import org.apache.paimon.types.DataType; +import org.apache.paimon.types.DecimalType; +import org.apache.paimon.types.LocalZonedTimestampType; import org.apache.paimon.types.RowKind; +import org.apache.paimon.types.TimestampType; import javax.annotation.Nullable; @@ -442,4 +446,32 @@ public static BinaryRow singleColumn(@Nullable BinaryString string) { writer.complete(); return row; } + + /** + * If it is a fixed-length field, we can call this BinaryRowData's setXX method for in-place + * updates. If it is variable-length field, can't use this method, because the underlying data + * is stored continuously. + */ + public static boolean isInFixedLengthPart(DataType type) { + switch (type.getTypeRoot()) { + case BOOLEAN: + case TINYINT: + case SMALLINT: + case INTEGER: + case DATE: + case TIME_WITHOUT_TIME_ZONE: + case BIGINT: + case FLOAT: + case DOUBLE: + return true; + case DECIMAL: + return Decimal.isCompact(((DecimalType) type).getPrecision()); + case TIMESTAMP_WITHOUT_TIME_ZONE: + return Timestamp.isCompact(((TimestampType) type).getPrecision()); + case TIMESTAMP_WITH_LOCAL_TIME_ZONE: + return Timestamp.isCompact(((LocalZonedTimestampType) type).getPrecision()); + default: + return false; + } + } } diff --git a/paimon-common/src/main/java/org/apache/paimon/data/InternalRow.java b/paimon-common/src/main/java/org/apache/paimon/data/InternalRow.java index 4c4f3f978d56..a83cbaec7396 100644 --- a/paimon-common/src/main/java/org/apache/paimon/data/InternalRow.java +++ b/paimon-common/src/main/java/org/apache/paimon/data/InternalRow.java @@ -244,4 +244,98 @@ interface FieldGetter extends Serializable { @Nullable Object getFieldOrNull(InternalRow row); } + + /** + * Creates a {@link FieldSetter} for setting elements to a row from a row at the given position. + * + * @param fieldType the element type of the row + * @param fieldPos the element position of the row + */ + static FieldSetter createFieldSetter(DataType fieldType, int fieldPos) { + final FieldSetter fieldSetter; + // ordered by type root definition + switch (fieldType.getTypeRoot()) { + case BOOLEAN: + fieldSetter = (from, to) -> to.setBoolean(fieldPos, from.getBoolean(fieldPos)); + break; + case DECIMAL: + final int decimalPrecision = getPrecision(fieldType); + final int decimalScale = getScale(fieldType); + fieldSetter = + (from, to) -> + to.setDecimal( + fieldPos, + from.getDecimal(fieldPos, decimalPrecision, decimalScale), + decimalPrecision); + if (fieldType.isNullable() && !Decimal.isCompact(decimalPrecision)) { + return (from, to) -> { + if (from.isNullAt(fieldPos)) { + to.setNullAt(fieldPos); + to.setDecimal(fieldPos, null, decimalPrecision); + } else { + fieldSetter.setFieldFrom(from, to); + } + }; + } + break; + case TINYINT: + fieldSetter = (from, to) -> to.setByte(fieldPos, from.getByte(fieldPos)); + break; + case SMALLINT: + fieldSetter = (from, to) -> to.setShort(fieldPos, from.getShort(fieldPos)); + break; + case INTEGER: + case DATE: + case TIME_WITHOUT_TIME_ZONE: + fieldSetter = (from, to) -> to.setInt(fieldPos, from.getInt(fieldPos)); + break; + case BIGINT: + fieldSetter = (from, to) -> to.setLong(fieldPos, from.getLong(fieldPos)); + break; + case FLOAT: + fieldSetter = (from, to) -> to.setFloat(fieldPos, from.getFloat(fieldPos)); + break; + case DOUBLE: + fieldSetter = (from, to) -> to.setDouble(fieldPos, from.getDouble(fieldPos)); + break; + case TIMESTAMP_WITHOUT_TIME_ZONE: + case TIMESTAMP_WITH_LOCAL_TIME_ZONE: + final int timestampPrecision = getPrecision(fieldType); + fieldSetter = + (from, to) -> + to.setTimestamp( + fieldPos, + from.getTimestamp(fieldPos, timestampPrecision), + timestampPrecision); + if (fieldType.isNullable() && !Timestamp.isCompact(timestampPrecision)) { + return (from, to) -> { + if (from.isNullAt(fieldPos)) { + to.setNullAt(fieldPos); + to.setTimestamp(fieldPos, null, timestampPrecision); + } else { + fieldSetter.setFieldFrom(from, to); + } + }; + } + break; + default: + throw new IllegalArgumentException( + String.format("type %s not support for setting", fieldType)); + } + if (!fieldType.isNullable()) { + return fieldSetter; + } + return (from, to) -> { + if (from.isNullAt(fieldPos)) { + to.setNullAt(fieldPos); + } else { + fieldSetter.setFieldFrom(from, to); + } + }; + } + + /** Accessor for setting the field of a row during runtime. */ + interface FieldSetter extends Serializable { + void setFieldFrom(DataGetters from, DataSetters to); + } } diff --git a/paimon-common/src/main/java/org/apache/paimon/data/columnar/ColumnarRowIterator.java b/paimon-common/src/main/java/org/apache/paimon/data/columnar/ColumnarRowIterator.java index 27e3d1c1ddad..874c22134864 100644 --- a/paimon-common/src/main/java/org/apache/paimon/data/columnar/ColumnarRowIterator.java +++ b/paimon-common/src/main/java/org/apache/paimon/data/columnar/ColumnarRowIterator.java @@ -95,7 +95,7 @@ public ColumnarRowIterator mapping( vectors = VectorMappingUtils.createPartitionMappedVectors(partitionInfo, vectors); } if (indexMapping != null) { - vectors = VectorMappingUtils.createIndexMappedVectors(indexMapping, vectors); + vectors = VectorMappingUtils.createMappedVectors(indexMapping, vectors); } return copy(vectors); } diff --git a/paimon-common/src/main/java/org/apache/paimon/data/columnar/heap/AbstractHeapVector.java b/paimon-common/src/main/java/org/apache/paimon/data/columnar/heap/AbstractHeapVector.java index 702877642327..f0e82eac4fb1 100644 --- a/paimon-common/src/main/java/org/apache/paimon/data/columnar/heap/AbstractHeapVector.java +++ b/paimon-common/src/main/java/org/apache/paimon/data/columnar/heap/AbstractHeapVector.java @@ -25,7 +25,8 @@ import java.util.Arrays; /** Heap vector that nullable shared structure. */ -public abstract class AbstractHeapVector extends AbstractWritableVector { +public abstract class AbstractHeapVector extends AbstractWritableVector + implements ElementCountable { public static final boolean LITTLE_ENDIAN = ByteOrder.nativeOrder() == ByteOrder.LITTLE_ENDIAN; @@ -116,6 +117,7 @@ public HeapIntVector getDictionaryIds() { return dictionaryIds; } + @Override public int getLen() { return this.len; } diff --git a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/flink/streaming/runtime/streamrecord/RecordAttributes.java b/paimon-common/src/main/java/org/apache/paimon/data/columnar/heap/ElementCountable.java similarity index 79% rename from paimon-flink/paimon-flink-cdc/src/main/java/org/apache/flink/streaming/runtime/streamrecord/RecordAttributes.java rename to paimon-common/src/main/java/org/apache/paimon/data/columnar/heap/ElementCountable.java index 723c71dc565d..a32762d659fd 100644 --- a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/flink/streaming/runtime/streamrecord/RecordAttributes.java +++ b/paimon-common/src/main/java/org/apache/paimon/data/columnar/heap/ElementCountable.java @@ -16,7 +16,10 @@ * limitations under the License. */ -package org.apache.flink.streaming.runtime.streamrecord; +package org.apache.paimon.data.columnar.heap; -/** Placeholder class for new feature introduced since flink 1.19. Should never be used. */ -public class RecordAttributes extends StreamElement {} +/** Container with a known number of elements. */ +public interface ElementCountable { + + int getLen(); +} diff --git a/paimon-common/src/main/java/org/apache/paimon/data/serializer/BinaryRowSerializer.java b/paimon-common/src/main/java/org/apache/paimon/data/serializer/BinaryRowSerializer.java index e5f3c9e7d0f5..8773e734d891 100644 --- a/paimon-common/src/main/java/org/apache/paimon/data/serializer/BinaryRowSerializer.java +++ b/paimon-common/src/main/java/org/apache/paimon/data/serializer/BinaryRowSerializer.java @@ -107,6 +107,11 @@ public BinaryRow toBinaryRow(BinaryRow rowData) throws IOException { // ============================ Page related operations =================================== + @Override + public BinaryRow createReuseInstance() { + return new BinaryRow(numFields); + } + @Override public int serializeToPages(BinaryRow record, AbstractPagedOutputView headerLessView) throws IOException { diff --git a/paimon-common/src/main/java/org/apache/paimon/data/serializer/InternalRowSerializer.java b/paimon-common/src/main/java/org/apache/paimon/data/serializer/InternalRowSerializer.java index 8a32a222c1df..ac8cc34e0c01 100644 --- a/paimon-common/src/main/java/org/apache/paimon/data/serializer/InternalRowSerializer.java +++ b/paimon-common/src/main/java/org/apache/paimon/data/serializer/InternalRowSerializer.java @@ -167,6 +167,11 @@ public BinaryRow toBinaryRow(InternalRow row) { return reuseRow; } + @Override + public InternalRow createReuseInstance() { + return binarySerializer.createReuseInstance(); + } + @Override public int serializeToPages(InternalRow row, AbstractPagedOutputView target) throws IOException { diff --git a/paimon-common/src/main/java/org/apache/paimon/data/serializer/PagedTypeSerializer.java b/paimon-common/src/main/java/org/apache/paimon/data/serializer/PagedTypeSerializer.java index f916d01b5e20..ede6c9b103be 100644 --- a/paimon-common/src/main/java/org/apache/paimon/data/serializer/PagedTypeSerializer.java +++ b/paimon-common/src/main/java/org/apache/paimon/data/serializer/PagedTypeSerializer.java @@ -27,6 +27,9 @@ /** A type serializer which provides paged serialize and deserialize methods. */ public interface PagedTypeSerializer extends Serializer { + /** Creates a new instance for reusing. */ + T createReuseInstance(); + /** * Serializes the given record to the given target paged output view. Some implementations may * skip some bytes if current page does not have enough space left, .e.g {@link BinaryRow}. diff --git a/paimon-common/src/main/java/org/apache/paimon/factories/Factory.java b/paimon-common/src/main/java/org/apache/paimon/factories/Factory.java index b0f1ec84c170..74796879ef4b 100644 --- a/paimon-common/src/main/java/org/apache/paimon/factories/Factory.java +++ b/paimon-common/src/main/java/org/apache/paimon/factories/Factory.java @@ -20,7 +20,7 @@ /** * Base interface for all kind of factories that create object instances from a list of key-value - * pairs in Paimon's catalog, lineage. + * pairs in Paimon's catalog. * *

    A factory is uniquely identified by {@link Class} and {@link #identifier()}. * diff --git a/paimon-common/src/main/java/org/apache/paimon/factories/FactoryUtil.java b/paimon-common/src/main/java/org/apache/paimon/factories/FactoryUtil.java index 25fe559663fa..1213168e16ae 100644 --- a/paimon-common/src/main/java/org/apache/paimon/factories/FactoryUtil.java +++ b/paimon-common/src/main/java/org/apache/paimon/factories/FactoryUtil.java @@ -18,6 +18,9 @@ package org.apache.paimon.factories; +import org.apache.paimon.shade.caffeine2.com.github.benmanes.caffeine.cache.Cache; +import org.apache.paimon.shade.caffeine2.com.github.benmanes.caffeine.cache.Caffeine; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -29,13 +32,17 @@ /** Utility for working with {@link Factory}s. */ public class FactoryUtil { + private static final Logger LOG = LoggerFactory.getLogger(FactoryUtil.class); + private static final Cache> FACTORIES = + Caffeine.newBuilder().softValues().maximumSize(100).executor(Runnable::run).build(); + /** Discovers a factory using the given factory base class and identifier. */ @SuppressWarnings("unchecked") public static T discoverFactory( ClassLoader classLoader, Class factoryClass, String identifier) { - final List factories = discoverFactories(classLoader); + final List factories = getFactories(classLoader); final List foundFactories = factories.stream() @@ -85,7 +92,7 @@ public static T discoverFactory( public static List discoverIdentifiers( ClassLoader classLoader, Class factoryClass) { - final List factories = discoverFactories(classLoader); + final List factories = getFactories(classLoader); return factories.stream() .filter(f -> factoryClass.isAssignableFrom(f.getClass())) @@ -93,6 +100,10 @@ public static List discoverIdentifiers( .collect(Collectors.toList()); } + private static List getFactories(ClassLoader classLoader) { + return FACTORIES.get(classLoader, FactoryUtil::discoverFactories); + } + private static List discoverFactories(ClassLoader classLoader) { final Iterator serviceLoaderIterator = ServiceLoader.load(Factory.class, classLoader).iterator(); @@ -110,9 +121,8 @@ private static List discoverFactories(ClassLoader classLoader) { } catch (Throwable t) { if (t instanceof NoClassDefFoundError) { LOG.debug( - "NoClassDefFoundError when loading a " - + Factory.class.getCanonicalName() - + ". This is expected when trying to load factory but no implementation is loaded.", + "NoClassDefFoundError when loading a {}. This is expected when trying to load factory but no implementation is loaded.", + Factory.class.getCanonicalName(), t); } else { throw new RuntimeException( diff --git a/paimon-common/src/main/java/org/apache/paimon/fileindex/FileIndexPredicate.java b/paimon-common/src/main/java/org/apache/paimon/fileindex/FileIndexPredicate.java index 1d19dfbb90a7..8f5485dbe66d 100644 --- a/paimon-common/src/main/java/org/apache/paimon/fileindex/FileIndexPredicate.java +++ b/paimon-common/src/main/java/org/apache/paimon/fileindex/FileIndexPredicate.java @@ -67,22 +67,20 @@ public FileIndexPredicate(SeekableInputStream inputStream, RowType fileRowType) this.reader = FileIndexFormat.createReader(inputStream, fileRowType); } - public boolean testPredicate(@Nullable Predicate filePredicate) { - if (filePredicate == null) { - return true; + public FileIndexResult evaluate(@Nullable Predicate predicate) { + if (predicate == null) { + return REMAIN; } - - Set requredFieldNames = getRequiredNames(filePredicate); - + Set requiredFieldNames = getRequiredNames(predicate); Map> indexReaders = new HashMap<>(); - requredFieldNames.forEach(name -> indexReaders.put(name, reader.readColumnIndex(name))); - if (!new FileIndexPredicateTest(indexReaders).test(filePredicate).remain()) { + requiredFieldNames.forEach(name -> indexReaders.put(name, reader.readColumnIndex(name))); + FileIndexResult result = new FileIndexPredicateTest(indexReaders).test(predicate); + if (!result.remain()) { LOG.debug( "One file has been filtered: " + (path == null ? "in scan stage" : path.toString())); - return false; } - return true; + return result; } private Set getRequiredNames(Predicate filePredicate) { diff --git a/paimon-common/src/main/java/org/apache/paimon/fileindex/bitmap/ApplyBitmapIndexFileRecordIterator.java b/paimon-common/src/main/java/org/apache/paimon/fileindex/bitmap/ApplyBitmapIndexFileRecordIterator.java new file mode 100644 index 000000000000..eec931d3e98f --- /dev/null +++ b/paimon-common/src/main/java/org/apache/paimon/fileindex/bitmap/ApplyBitmapIndexFileRecordIterator.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.fileindex.bitmap; + +import org.apache.paimon.data.InternalRow; +import org.apache.paimon.fs.Path; +import org.apache.paimon.reader.FileRecordIterator; +import org.apache.paimon.utils.RoaringBitmap32; + +import javax.annotation.Nullable; + +import java.io.IOException; + +/** + * A {@link FileRecordIterator} wraps a {@link FileRecordIterator} and {@link BitmapIndexResult}. + */ +public class ApplyBitmapIndexFileRecordIterator implements FileRecordIterator { + + private final FileRecordIterator iterator; + private final RoaringBitmap32 bitmap; + private final int last; + + public ApplyBitmapIndexFileRecordIterator( + FileRecordIterator iterator, BitmapIndexResult fileIndexResult) { + this.iterator = iterator; + this.bitmap = fileIndexResult.get(); + this.last = bitmap.last(); + } + + @Override + public long returnedPosition() { + return iterator.returnedPosition(); + } + + @Override + public Path filePath() { + return iterator.filePath(); + } + + @Nullable + @Override + public InternalRow next() throws IOException { + while (true) { + InternalRow next = iterator.next(); + if (next == null) { + return null; + } + int position = (int) returnedPosition(); + if (position > last) { + return null; + } + if (bitmap.contains(position)) { + return next; + } + } + } + + @Override + public void releaseBatch() { + iterator.releaseBatch(); + } +} diff --git a/paimon-common/src/main/java/org/apache/paimon/fileindex/bitmap/ApplyBitmapIndexRecordReader.java b/paimon-common/src/main/java/org/apache/paimon/fileindex/bitmap/ApplyBitmapIndexRecordReader.java new file mode 100644 index 000000000000..3b1207c8bd6e --- /dev/null +++ b/paimon-common/src/main/java/org/apache/paimon/fileindex/bitmap/ApplyBitmapIndexRecordReader.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.fileindex.bitmap; + +import org.apache.paimon.data.InternalRow; +import org.apache.paimon.reader.FileRecordIterator; +import org.apache.paimon.reader.FileRecordReader; +import org.apache.paimon.reader.RecordReader; + +import javax.annotation.Nullable; + +import java.io.IOException; + +/** A {@link RecordReader} which apply {@link BitmapIndexResult} to filter record. */ +public class ApplyBitmapIndexRecordReader implements FileRecordReader { + + private final FileRecordReader reader; + + private final BitmapIndexResult fileIndexResult; + + public ApplyBitmapIndexRecordReader( + FileRecordReader reader, BitmapIndexResult fileIndexResult) { + this.reader = reader; + this.fileIndexResult = fileIndexResult; + } + + @Nullable + @Override + public FileRecordIterator readBatch() throws IOException { + FileRecordIterator batch = reader.readBatch(); + if (batch == null) { + return null; + } + + return new ApplyBitmapIndexFileRecordIterator(batch, fileIndexResult); + } + + @Override + public void close() throws IOException { + reader.close(); + } +} diff --git a/paimon-common/src/main/java/org/apache/paimon/fileindex/bitmap/BitmapFileIndex.java b/paimon-common/src/main/java/org/apache/paimon/fileindex/bitmap/BitmapFileIndex.java index 22375e763f79..4020302c565c 100644 --- a/paimon-common/src/main/java/org/apache/paimon/fileindex/bitmap/BitmapFileIndex.java +++ b/paimon-common/src/main/java/org/apache/paimon/fileindex/bitmap/BitmapFileIndex.java @@ -188,7 +188,7 @@ public FileIndexResult visitNotEqual(FieldRef fieldRef, Object literal) { @Override public FileIndexResult visitIn(FieldRef fieldRef, List literals) { - return new BitmapIndexResultLazy( + return new BitmapIndexResult( () -> { readInternalMeta(fieldRef.type()); return getInListResultBitmap(literals); @@ -197,7 +197,7 @@ public FileIndexResult visitIn(FieldRef fieldRef, List literals) { @Override public FileIndexResult visitNotIn(FieldRef fieldRef, List literals) { - return new BitmapIndexResultLazy( + return new BitmapIndexResult( () -> { readInternalMeta(fieldRef.type()); RoaringBitmap32 bitmap = getInListResultBitmap(literals); diff --git a/paimon-common/src/main/java/org/apache/paimon/fileindex/bitmap/BitmapIndexResultLazy.java b/paimon-common/src/main/java/org/apache/paimon/fileindex/bitmap/BitmapIndexResult.java similarity index 66% rename from paimon-common/src/main/java/org/apache/paimon/fileindex/bitmap/BitmapIndexResultLazy.java rename to paimon-common/src/main/java/org/apache/paimon/fileindex/bitmap/BitmapIndexResult.java index 15210e856627..8d572ff254fc 100644 --- a/paimon-common/src/main/java/org/apache/paimon/fileindex/bitmap/BitmapIndexResultLazy.java +++ b/paimon-common/src/main/java/org/apache/paimon/fileindex/bitmap/BitmapIndexResult.java @@ -25,32 +25,31 @@ import java.util.function.Supplier; /** bitmap file index result. */ -public class BitmapIndexResultLazy extends LazyField implements FileIndexResult { +public class BitmapIndexResult extends LazyField implements FileIndexResult { - public BitmapIndexResultLazy(Supplier supplier) { + public BitmapIndexResult(Supplier supplier) { super(supplier); } + @Override public boolean remain() { return !get().isEmpty(); } + @Override public FileIndexResult and(FileIndexResult fileIndexResult) { - if (fileIndexResult instanceof BitmapIndexResultLazy) { - return new BitmapIndexResultLazy( - () -> - RoaringBitmap32.and( - get(), ((BitmapIndexResultLazy) fileIndexResult).get())); + if (fileIndexResult instanceof BitmapIndexResult) { + return new BitmapIndexResult( + () -> RoaringBitmap32.and(get(), ((BitmapIndexResult) fileIndexResult).get())); } return FileIndexResult.super.and(fileIndexResult); } + @Override public FileIndexResult or(FileIndexResult fileIndexResult) { - if (fileIndexResult instanceof BitmapIndexResultLazy) { - return new BitmapIndexResultLazy( - () -> - RoaringBitmap32.or( - get(), ((BitmapIndexResultLazy) fileIndexResult).get())); + if (fileIndexResult instanceof BitmapIndexResult) { + return new BitmapIndexResult( + () -> RoaringBitmap32.or(get(), ((BitmapIndexResult) fileIndexResult).get())); } return FileIndexResult.super.and(fileIndexResult); } diff --git a/paimon-common/src/main/java/org/apache/paimon/fileindex/bsi/BitSliceIndexBitmapFileIndex.java b/paimon-common/src/main/java/org/apache/paimon/fileindex/bsi/BitSliceIndexBitmapFileIndex.java new file mode 100644 index 000000000000..df6b8d897ca1 --- /dev/null +++ b/paimon-common/src/main/java/org/apache/paimon/fileindex/bsi/BitSliceIndexBitmapFileIndex.java @@ -0,0 +1,403 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.fileindex.bsi; + +import org.apache.paimon.data.Decimal; +import org.apache.paimon.data.Timestamp; +import org.apache.paimon.fileindex.FileIndexReader; +import org.apache.paimon.fileindex.FileIndexResult; +import org.apache.paimon.fileindex.FileIndexWriter; +import org.apache.paimon.fileindex.FileIndexer; +import org.apache.paimon.fileindex.bitmap.BitmapIndexResult; +import org.apache.paimon.fs.SeekableInputStream; +import org.apache.paimon.options.Options; +import org.apache.paimon.predicate.FieldRef; +import org.apache.paimon.types.BigIntType; +import org.apache.paimon.types.DataType; +import org.apache.paimon.types.DataTypeDefaultVisitor; +import org.apache.paimon.types.DateType; +import org.apache.paimon.types.DecimalType; +import org.apache.paimon.types.IntType; +import org.apache.paimon.types.LocalZonedTimestampType; +import org.apache.paimon.types.SmallIntType; +import org.apache.paimon.types.TimeType; +import org.apache.paimon.types.TimestampType; +import org.apache.paimon.types.TinyIntType; +import org.apache.paimon.utils.BitSliceIndexRoaringBitmap; +import org.apache.paimon.utils.RoaringBitmap32; + +import java.io.ByteArrayOutputStream; +import java.io.DataInput; +import java.io.DataInputStream; +import java.io.DataOutput; +import java.io.DataOutputStream; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.function.Function; + +/** implementation of BSI file index. */ +public class BitSliceIndexBitmapFileIndex implements FileIndexer { + + public static final int VERSION_1 = 1; + + private final DataType dataType; + + public BitSliceIndexBitmapFileIndex(DataType dataType, Options options) { + this.dataType = dataType; + } + + @Override + public FileIndexWriter createWriter() { + return new Writer(dataType); + } + + @Override + public FileIndexReader createReader(SeekableInputStream inputStream, int start, int length) { + try { + inputStream.seek(start); + DataInput input = new DataInputStream(inputStream); + byte version = input.readByte(); + if (version > VERSION_1) { + throw new RuntimeException( + String.format( + "read bsi index file fail, " + + "your plugin version is lower than %d", + version)); + } + + int rowNumber = input.readInt(); + + boolean hasPositive = input.readBoolean(); + BitSliceIndexRoaringBitmap positive = + hasPositive + ? BitSliceIndexRoaringBitmap.map(input) + : BitSliceIndexRoaringBitmap.EMPTY; + + boolean hasNegative = input.readBoolean(); + BitSliceIndexRoaringBitmap negative = + hasNegative + ? BitSliceIndexRoaringBitmap.map(input) + : BitSliceIndexRoaringBitmap.EMPTY; + + return new Reader(dataType, rowNumber, positive, negative); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + private static class Writer extends FileIndexWriter { + + private final Function valueMapper; + private final StatsCollectList collector; + + public Writer(DataType dataType) { + this.valueMapper = getValueMapper(dataType); + this.collector = new StatsCollectList(); + } + + @Override + public void write(Object key) { + collector.add(valueMapper.apply(key)); + } + + @Override + public byte[] serializedBytes() { + try { + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + DataOutput out = new DataOutputStream(bos); + + BitSliceIndexRoaringBitmap.Appender positive = + new BitSliceIndexRoaringBitmap.Appender( + collector.positiveMin, collector.positiveMax); + BitSliceIndexRoaringBitmap.Appender negative = + new BitSliceIndexRoaringBitmap.Appender( + collector.negativeMin, collector.negativeMax); + + for (int i = 0; i < collector.values.size(); i++) { + Long value = collector.values.get(i); + if (value != null) { + if (value < 0) { + negative.append(i, Math.abs(value)); + } else { + positive.append(i, value); + } + } + } + + out.writeByte(VERSION_1); + out.writeInt(collector.values.size()); + + boolean hasPositive = positive.isNotEmpty(); + out.writeBoolean(hasPositive); + if (hasPositive) { + positive.serialize(out); + } + + boolean hasNegative = negative.isNotEmpty(); + out.writeBoolean(hasNegative); + if (hasNegative) { + negative.serialize(out); + } + return bos.toByteArray(); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + private static class StatsCollectList { + private long positiveMin; + private long positiveMax; + private long negativeMin; + private long negativeMax; + // todo: Find a way to reduce the risk of out-of-memory. + private final List values = new ArrayList<>(); + + public void add(Long value) { + values.add(value); + if (value != null) { + collect(value); + } + } + + private void collect(long value) { + if (value < 0) { + negativeMin = Math.min(negativeMin, Math.abs(value)); + negativeMax = Math.max(negativeMax, Math.abs(value)); + } else { + positiveMin = Math.min(positiveMin, value); + positiveMax = Math.max(positiveMax, value); + } + } + } + } + + private static class Reader extends FileIndexReader { + + private final int rowNumber; + private final BitSliceIndexRoaringBitmap positive; + private final BitSliceIndexRoaringBitmap negative; + private final Function valueMapper; + + public Reader( + DataType dataType, + int rowNumber, + BitSliceIndexRoaringBitmap positive, + BitSliceIndexRoaringBitmap negative) { + this.rowNumber = rowNumber; + this.positive = positive; + this.negative = negative; + this.valueMapper = getValueMapper(dataType); + } + + @Override + public FileIndexResult visitIsNull(FieldRef fieldRef) { + return new BitmapIndexResult( + () -> { + RoaringBitmap32 bitmap = + RoaringBitmap32.or(positive.isNotNull(), negative.isNotNull()); + bitmap.flip(0, rowNumber); + return bitmap; + }); + } + + @Override + public FileIndexResult visitIsNotNull(FieldRef fieldRef) { + return new BitmapIndexResult( + () -> RoaringBitmap32.or(positive.isNotNull(), negative.isNotNull())); + } + + @Override + public FileIndexResult visitEqual(FieldRef fieldRef, Object literal) { + return visitIn(fieldRef, Collections.singletonList(literal)); + } + + @Override + public FileIndexResult visitNotEqual(FieldRef fieldRef, Object literal) { + return visitNotIn(fieldRef, Collections.singletonList(literal)); + } + + @Override + public FileIndexResult visitIn(FieldRef fieldRef, List literals) { + return new BitmapIndexResult( + () -> + literals.stream() + .map(valueMapper) + .map( + value -> { + if (value < 0) { + return negative.eq(Math.abs(value)); + } else { + return positive.eq(value); + } + }) + .reduce( + new RoaringBitmap32(), + (x1, x2) -> RoaringBitmap32.or(x1, x2))); + } + + @Override + public FileIndexResult visitNotIn(FieldRef fieldRef, List literals) { + return new BitmapIndexResult( + () -> { + RoaringBitmap32 ebm = + RoaringBitmap32.or(positive.isNotNull(), negative.isNotNull()); + RoaringBitmap32 eq = + literals.stream() + .map(valueMapper) + .map( + value -> { + if (value < 0) { + return negative.eq(Math.abs(value)); + } else { + return positive.eq(value); + } + }) + .reduce( + new RoaringBitmap32(), + (x1, x2) -> RoaringBitmap32.or(x1, x2)); + return RoaringBitmap32.andNot(ebm, eq); + }); + } + + @Override + public FileIndexResult visitLessThan(FieldRef fieldRef, Object literal) { + return new BitmapIndexResult( + () -> { + Long value = valueMapper.apply(literal); + if (value < 0) { + return negative.gt(Math.abs(value)); + } else { + return RoaringBitmap32.or(positive.lt(value), negative.isNotNull()); + } + }); + } + + @Override + public FileIndexResult visitLessOrEqual(FieldRef fieldRef, Object literal) { + return new BitmapIndexResult( + () -> { + Long value = valueMapper.apply(literal); + if (value < 0) { + return negative.gte(Math.abs(value)); + } else { + return RoaringBitmap32.or(positive.lte(value), negative.isNotNull()); + } + }); + } + + @Override + public FileIndexResult visitGreaterThan(FieldRef fieldRef, Object literal) { + return new BitmapIndexResult( + () -> { + Long value = valueMapper.apply(literal); + if (value < 0) { + return RoaringBitmap32.or( + positive.isNotNull(), negative.lt(Math.abs(value))); + } else { + return positive.gt(value); + } + }); + } + + @Override + public FileIndexResult visitGreaterOrEqual(FieldRef fieldRef, Object literal) { + return new BitmapIndexResult( + () -> { + Long value = valueMapper.apply(literal); + if (value < 0) { + return RoaringBitmap32.or( + positive.isNotNull(), negative.lte(Math.abs(value))); + } else { + return positive.gte(value); + } + }); + } + } + + public static Function getValueMapper(DataType dataType) { + return dataType.accept( + new DataTypeDefaultVisitor>() { + @Override + public Function visit(DecimalType decimalType) { + return o -> o == null ? null : ((Decimal) o).toUnscaledLong(); + } + + @Override + public Function visit(TinyIntType tinyIntType) { + return o -> o == null ? null : ((Byte) o).longValue(); + } + + @Override + public Function visit(SmallIntType smallIntType) { + return o -> o == null ? null : ((Short) o).longValue(); + } + + @Override + public Function visit(IntType intType) { + return o -> o == null ? null : ((Integer) o).longValue(); + } + + @Override + public Function visit(BigIntType bigIntType) { + return o -> o == null ? null : (Long) o; + } + + @Override + public Function visit(DateType dateType) { + return o -> o == null ? null : ((Integer) o).longValue(); + } + + @Override + public Function visit(TimeType timeType) { + return o -> o == null ? null : ((Integer) o).longValue(); + } + + @Override + public Function visit(TimestampType timestampType) { + return getTimeStampMapper(timestampType.getPrecision()); + } + + @Override + public Function visit( + LocalZonedTimestampType localZonedTimestampType) { + return getTimeStampMapper(localZonedTimestampType.getPrecision()); + } + + @Override + protected Function defaultMethod(DataType dataType) { + throw new UnsupportedOperationException( + dataType.asSQLString() + + " type is not support to build bsi index yet."); + } + + private Function getTimeStampMapper(int precision) { + return o -> { + if (o == null) { + return null; + } else if (precision <= 3) { + return ((Timestamp) o).getMillisecond(); + } else { + return ((Timestamp) o).toMicros(); + } + }; + } + }); + } +} diff --git a/paimon-common/src/main/java/org/apache/paimon/fileindex/bsi/BitSliceIndexBitmapFileIndexFactory.java b/paimon-common/src/main/java/org/apache/paimon/fileindex/bsi/BitSliceIndexBitmapFileIndexFactory.java new file mode 100644 index 000000000000..aa7a92b78525 --- /dev/null +++ b/paimon-common/src/main/java/org/apache/paimon/fileindex/bsi/BitSliceIndexBitmapFileIndexFactory.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.fileindex.bsi; + +import org.apache.paimon.fileindex.FileIndexer; +import org.apache.paimon.fileindex.FileIndexerFactory; +import org.apache.paimon.options.Options; +import org.apache.paimon.types.DataType; + +/** Factory to create {@link BitSliceIndexBitmapFileIndex}. */ +public class BitSliceIndexBitmapFileIndexFactory implements FileIndexerFactory { + + public static final String BSI_INDEX = "bsi"; + + @Override + public String identifier() { + return BSI_INDEX; + } + + @Override + public FileIndexer create(DataType dataType, Options options) { + return new BitSliceIndexBitmapFileIndex(dataType, options); + } +} diff --git a/paimon-common/src/main/java/org/apache/paimon/format/FormatReaderContext.java b/paimon-common/src/main/java/org/apache/paimon/format/FormatReaderContext.java index 92a569e031de..cae6a977e615 100644 --- a/paimon-common/src/main/java/org/apache/paimon/format/FormatReaderContext.java +++ b/paimon-common/src/main/java/org/apache/paimon/format/FormatReaderContext.java @@ -18,21 +18,31 @@ package org.apache.paimon.format; +import org.apache.paimon.fileindex.FileIndexResult; import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.Path; import org.apache.paimon.reader.RecordReader; +import javax.annotation.Nullable; + /** the context for creating RecordReader {@link RecordReader}. */ public class FormatReaderContext implements FormatReaderFactory.Context { private final FileIO fileIO; private final Path file; private final long fileSize; + @Nullable private final FileIndexResult fileIndexResult; public FormatReaderContext(FileIO fileIO, Path file, long fileSize) { + this(fileIO, file, fileSize, null); + } + + public FormatReaderContext( + FileIO fileIO, Path file, long fileSize, @Nullable FileIndexResult fileIndexResult) { this.fileIO = fileIO; this.file = file; this.fileSize = fileSize; + this.fileIndexResult = fileIndexResult; } @Override @@ -49,4 +59,10 @@ public Path filePath() { public long fileSize() { return fileSize; } + + @Nullable + @Override + public FileIndexResult fileIndex() { + return fileIndexResult; + } } diff --git a/paimon-common/src/main/java/org/apache/paimon/format/FormatReaderFactory.java b/paimon-common/src/main/java/org/apache/paimon/format/FormatReaderFactory.java index d2fc91501636..5ef084ec4d34 100644 --- a/paimon-common/src/main/java/org/apache/paimon/format/FormatReaderFactory.java +++ b/paimon-common/src/main/java/org/apache/paimon/format/FormatReaderFactory.java @@ -19,16 +19,20 @@ package org.apache.paimon.format; import org.apache.paimon.data.InternalRow; +import org.apache.paimon.fileindex.FileIndexResult; import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.Path; +import org.apache.paimon.reader.FileRecordReader; import org.apache.paimon.reader.RecordReader; +import javax.annotation.Nullable; + import java.io.IOException; /** A factory to create {@link RecordReader} for file. */ public interface FormatReaderFactory { - RecordReader createReader(Context context) throws IOException; + FileRecordReader createReader(Context context) throws IOException; /** Context for creating reader. */ interface Context { @@ -38,5 +42,8 @@ interface Context { Path filePath(); long fileSize(); + + @Nullable + FileIndexResult fileIndex(); } } diff --git a/paimon-common/src/main/java/org/apache/paimon/format/SimpleColStats.java b/paimon-common/src/main/java/org/apache/paimon/format/SimpleColStats.java index 5d5891a8a3b2..0b0062b7568f 100644 --- a/paimon-common/src/main/java/org/apache/paimon/format/SimpleColStats.java +++ b/paimon-common/src/main/java/org/apache/paimon/format/SimpleColStats.java @@ -33,6 +33,8 @@ */ public class SimpleColStats { + public static final SimpleColStats NONE = new SimpleColStats(null, null, null); + @Nullable private final Object min; @Nullable private final Object max; private final Long nullCount; @@ -58,6 +60,10 @@ public Long nullCount() { return nullCount; } + public boolean isNone() { + return min == null && max == null && nullCount == null; + } + @Override public boolean equals(Object o) { if (!(o instanceof SimpleColStats)) { diff --git a/paimon-common/src/main/java/org/apache/paimon/fs/FileStatus.java b/paimon-common/src/main/java/org/apache/paimon/fs/FileStatus.java index 8308f5205d66..c3e6cde9cf0b 100644 --- a/paimon-common/src/main/java/org/apache/paimon/fs/FileStatus.java +++ b/paimon-common/src/main/java/org/apache/paimon/fs/FileStatus.java @@ -20,6 +20,8 @@ import org.apache.paimon.annotation.Public; +import javax.annotation.Nullable; + /** * Interface that represents the client side information for a file independent of the file system. * @@ -56,4 +58,24 @@ public interface FileStatus { * milliseconds since the epoch (UTC January 1, 1970). */ long getModificationTime(); + + /** + * Get the last access time of the file. + * + * @return A long value representing the time the file was last accessed, measured in + * milliseconds since the epoch (UTC January 1, 1970). + */ + default long getAccessTime() { + return 0; + } + + /** + * Returns the owner of this file. + * + * @return the owner of this file + */ + @Nullable + default String getOwner() { + return null; + } } diff --git a/paimon-common/src/main/java/org/apache/paimon/fs/hadoop/HadoopFileIO.java b/paimon-common/src/main/java/org/apache/paimon/fs/hadoop/HadoopFileIO.java index 70325ee69635..0a8d64a73b00 100644 --- a/paimon-common/src/main/java/org/apache/paimon/fs/hadoop/HadoopFileIO.java +++ b/paimon-common/src/main/java/org/apache/paimon/fs/hadoop/HadoopFileIO.java @@ -329,6 +329,16 @@ public Path getPath() { public long getModificationTime() { return status.getModificationTime(); } + + @Override + public long getAccessTime() { + return status.getAccessTime(); + } + + @Override + public String getOwner() { + return status.getOwner(); + } } // ============================== extra methods =================================== diff --git a/paimon-common/src/main/java/org/apache/paimon/io/CompressedPageFileInput.java b/paimon-common/src/main/java/org/apache/paimon/io/CompressedPageFileInput.java index 3d1a23892d7e..242e7431109b 100644 --- a/paimon-common/src/main/java/org/apache/paimon/io/CompressedPageFileInput.java +++ b/paimon-common/src/main/java/org/apache/paimon/io/CompressedPageFileInput.java @@ -30,7 +30,7 @@ public class CompressedPageFileInput implements PageFileInput { private final RandomAccessFile file; private final int pageSize; - private final long uncompressBytes; + private final long uncompressedBytes; private final long[] pagePositions; private final BlockDecompressor decompressor; @@ -44,11 +44,11 @@ public CompressedPageFileInput( RandomAccessFile file, int pageSize, BlockCompressionFactory compressionFactory, - long uncompressBytes, + long uncompressedBytes, long[] pagePositions) { this.file = file; this.pageSize = pageSize; - this.uncompressBytes = uncompressBytes; + this.uncompressedBytes = uncompressedBytes; this.pagePositions = pagePositions; this.uncompressedBuffer = new byte[pageSize]; @@ -67,7 +67,7 @@ public RandomAccessFile file() { @Override public long uncompressBytes() { - return uncompressBytes; + return uncompressedBytes; } @Override diff --git a/paimon-common/src/main/java/org/apache/paimon/io/cache/Cache.java b/paimon-common/src/main/java/org/apache/paimon/io/cache/Cache.java new file mode 100644 index 000000000000..4762d15f22dd --- /dev/null +++ b/paimon-common/src/main/java/org/apache/paimon/io/cache/Cache.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.io.cache; + +import org.apache.paimon.memory.MemorySegment; + +import javax.annotation.Nullable; + +import java.util.Map; +import java.util.function.Function; + +/** Cache interface in paimon which supports caffeine and guava caches. */ +public interface Cache { + @Nullable + CacheValue get(CacheKey key, Function supplier); + + void put(CacheKey key, CacheValue value); + + void invalidate(CacheKey key); + + void invalidateAll(); + + Map asMap(); + + /** Value for cache. */ + class CacheValue { + + final MemorySegment segment; + final CacheCallback callback; + + CacheValue(MemorySegment segment, CacheCallback callback) { + this.segment = segment; + this.callback = callback; + } + } + + /** Type for cache. */ + enum CacheType { + CAFFEINE, + GUAVA; + } +} diff --git a/paimon-common/src/main/java/org/apache/paimon/io/cache/CacheBuilder.java b/paimon-common/src/main/java/org/apache/paimon/io/cache/CacheBuilder.java new file mode 100644 index 000000000000..402f21f06264 --- /dev/null +++ b/paimon-common/src/main/java/org/apache/paimon/io/cache/CacheBuilder.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.io.cache; + +import org.apache.paimon.options.MemorySize; + +import org.apache.paimon.shade.caffeine2.com.github.benmanes.caffeine.cache.Caffeine; +import org.apache.paimon.shade.caffeine2.com.github.benmanes.caffeine.cache.RemovalCause; +import org.apache.paimon.shade.guava30.com.google.common.cache.RemovalNotification; + +/** Cache builder builds cache from cache type. */ +public abstract class CacheBuilder { + protected MemorySize memorySize; + + CacheBuilder maximumWeight(MemorySize memorySize) { + this.memorySize = memorySize; + return this; + } + + public abstract Cache build(); + + public static CacheBuilder newBuilder(Cache.CacheType type) { + switch (type) { + case CAFFEINE: + return new CaffeineCacheBuilder(); + case GUAVA: + return new GuavaCacheBuilder(); + default: + throw new UnsupportedOperationException("Unsupported CacheType: " + type); + } + } + + static class CaffeineCacheBuilder extends CacheBuilder { + @Override + public Cache build() { + return new CaffeineCache( + Caffeine.newBuilder() + .weigher(CacheBuilder::weigh) + .maximumWeight(memorySize.getBytes()) + .removalListener(this::onRemoval) + .executor(Runnable::run) + .build()); + } + + private void onRemoval(CacheKey key, Cache.CacheValue value, RemovalCause cause) { + if (value != null) { + value.callback.onRemoval(key); + } + } + } + + static class GuavaCacheBuilder extends CacheBuilder { + @Override + public Cache build() { + return new GuavaCache( + org.apache.paimon.shade.guava30.com.google.common.cache.CacheBuilder + .newBuilder() + .weigher(CacheBuilder::weigh) + // The concurrency level determines the number of segment caches in + // Guava,limiting the maximum block entries held in cache. Since we do + // not access this cache concurrently, it is set to 1. + .concurrencyLevel(1) + .maximumWeight(memorySize.getBytes()) + .removalListener(this::onRemoval) + .build()); + } + + private void onRemoval(RemovalNotification notification) { + if (notification.getValue() != null) { + notification.getValue().callback.onRemoval(notification.getKey()); + } + } + } + + private static int weigh(CacheKey cacheKey, Cache.CacheValue cacheValue) { + return cacheValue.segment.size(); + } +} diff --git a/paimon-common/src/main/java/org/apache/paimon/io/cache/CacheKey.java b/paimon-common/src/main/java/org/apache/paimon/io/cache/CacheKey.java index b313018d3589..11b8beb22c55 100644 --- a/paimon-common/src/main/java/org/apache/paimon/io/cache/CacheKey.java +++ b/paimon-common/src/main/java/org/apache/paimon/io/cache/CacheKey.java @@ -24,25 +24,31 @@ /** Key for cache manager. */ public interface CacheKey { - static CacheKey forPosition(RandomAccessFile file, long position, int length) { - return new PositionCacheKey(file, position, length); + static CacheKey forPosition(RandomAccessFile file, long position, int length, boolean isIndex) { + return new PositionCacheKey(file, position, length, isIndex); } static CacheKey forPageIndex(RandomAccessFile file, int pageSize, int pageIndex) { - return new PageIndexCacheKey(file, pageSize, pageIndex); + return new PageIndexCacheKey(file, pageSize, pageIndex, false); } + /** @return Whether this cache key is for index cache. */ + boolean isIndex(); + /** Key for file position and length. */ class PositionCacheKey implements CacheKey { private final RandomAccessFile file; private final long position; private final int length; + private final boolean isIndex; - private PositionCacheKey(RandomAccessFile file, long position, int length) { + private PositionCacheKey( + RandomAccessFile file, long position, int length, boolean isIndex) { this.file = file; this.position = position; this.length = length; + this.isIndex = isIndex; } @Override @@ -56,12 +62,18 @@ public boolean equals(Object o) { PositionCacheKey that = (PositionCacheKey) o; return position == that.position && length == that.length + && isIndex == that.isIndex && Objects.equals(file, that.file); } @Override public int hashCode() { - return Objects.hash(file, position, length); + return Objects.hash(file, position, length, isIndex); + } + + @Override + public boolean isIndex() { + return isIndex; } } @@ -71,17 +83,25 @@ class PageIndexCacheKey implements CacheKey { private final RandomAccessFile file; private final int pageSize; private final int pageIndex; + private final boolean isIndex; - private PageIndexCacheKey(RandomAccessFile file, int pageSize, int pageIndex) { + private PageIndexCacheKey( + RandomAccessFile file, int pageSize, int pageIndex, boolean isIndex) { this.file = file; this.pageSize = pageSize; this.pageIndex = pageIndex; + this.isIndex = isIndex; } public int pageIndex() { return pageIndex; } + @Override + public boolean isIndex() { + return isIndex; + } + @Override public boolean equals(Object o) { if (this == o) { @@ -93,12 +113,13 @@ public boolean equals(Object o) { PageIndexCacheKey that = (PageIndexCacheKey) o; return pageSize == that.pageSize && pageIndex == that.pageIndex + && isIndex == that.isIndex && Objects.equals(file, that.file); } @Override public int hashCode() { - return Objects.hash(file, pageSize, pageIndex); + return Objects.hash(file, pageSize, pageIndex, isIndex); } } } diff --git a/paimon-common/src/main/java/org/apache/paimon/io/cache/CacheManager.java b/paimon-common/src/main/java/org/apache/paimon/io/cache/CacheManager.java index b8f00205ed91..677d87d49909 100644 --- a/paimon-common/src/main/java/org/apache/paimon/io/cache/CacheManager.java +++ b/paimon-common/src/main/java/org/apache/paimon/io/cache/CacheManager.java @@ -21,68 +21,95 @@ import org.apache.paimon.annotation.VisibleForTesting; import org.apache.paimon.memory.MemorySegment; import org.apache.paimon.options.MemorySize; +import org.apache.paimon.utils.Preconditions; -import org.apache.paimon.shade.caffeine2.com.github.benmanes.caffeine.cache.Cache; -import org.apache.paimon.shade.caffeine2.com.github.benmanes.caffeine.cache.Caffeine; -import org.apache.paimon.shade.caffeine2.com.github.benmanes.caffeine.cache.RemovalCause; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.IOException; +import static org.apache.paimon.utils.Preconditions.checkNotNull; + /** Cache manager to cache bytes to paged {@link MemorySegment}s. */ public class CacheManager { + private static final Logger LOG = LoggerFactory.getLogger(CacheManager.class); + /** * Refreshing the cache comes with some costs, so not every time we visit the CacheManager, but * every 10 visits, refresh the LRU strategy. */ public static final int REFRESH_COUNT = 10; - private final Cache cache; + private final Cache dataCache; + private final Cache indexCache; private int fileReadCount; + @VisibleForTesting public CacheManager(MemorySize maxMemorySize) { - this.cache = - Caffeine.newBuilder() - .weigher(this::weigh) - .maximumWeight(maxMemorySize.getBytes()) - .removalListener(this::onRemoval) - .executor(Runnable::run) - .build(); - this.fileReadCount = 0; + this(Cache.CacheType.GUAVA, maxMemorySize, 0); } - @VisibleForTesting - public Cache cache() { - return cache; + public CacheManager(MemorySize dataMaxMemorySize, double highPriorityPoolRatio) { + this(Cache.CacheType.GUAVA, dataMaxMemorySize, highPriorityPoolRatio); } - public MemorySegment getPage(CacheKey key, CacheReader reader, CacheCallback callback) { - CacheValue value = cache.getIfPresent(key); - while (value == null || value.isClosed) { - try { - this.fileReadCount++; - value = new CacheValue(MemorySegment.wrap(reader.read(key)), callback); - } catch (IOException e) { - throw new RuntimeException(e); - } - cache.put(key, value); + public CacheManager( + Cache.CacheType cacheType, MemorySize maxMemorySize, double highPriorityPoolRatio) { + Preconditions.checkArgument( + highPriorityPoolRatio >= 0 && highPriorityPoolRatio < 1, + "The high priority pool ratio should in the range [0, 1)."); + MemorySize indexCacheSize = + MemorySize.ofBytes((long) (maxMemorySize.getBytes() * highPriorityPoolRatio)); + MemorySize dataCacheSize = + MemorySize.ofBytes((long) (maxMemorySize.getBytes() * (1 - highPriorityPoolRatio))); + this.dataCache = CacheBuilder.newBuilder(cacheType).maximumWeight(dataCacheSize).build(); + if (highPriorityPoolRatio == 0) { + this.indexCache = dataCache; + } else { + this.indexCache = + CacheBuilder.newBuilder(cacheType).maximumWeight(indexCacheSize).build(); } - return value.segment; + this.fileReadCount = 0; + LOG.info( + "Initialize cache manager with data cache of {} and index cache of {}.", + dataCacheSize, + indexCacheSize); } - public void invalidPage(CacheKey key) { - cache.invalidate(key); + @VisibleForTesting + public Cache dataCache() { + return dataCache; } - private int weigh(CacheKey cacheKey, CacheValue cacheValue) { - return cacheValue.segment.size(); + @VisibleForTesting + public Cache indexCache() { + return indexCache; } - private void onRemoval(CacheKey key, CacheValue value, RemovalCause cause) { - if (value != null) { - value.isClosed = true; - value.callback.onRemoval(key); + public MemorySegment getPage(CacheKey key, CacheReader reader, CacheCallback callback) { + Cache cache = key.isIndex() ? indexCache : dataCache; + Cache.CacheValue value = + cache.get( + key, + k -> { + this.fileReadCount++; + try { + return new Cache.CacheValue( + MemorySegment.wrap(reader.read(key)), callback); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + return checkNotNull(value, String.format("Cache result for key(%s) is null", key)).segment; + } + + public void invalidPage(CacheKey key) { + if (key.isIndex()) { + indexCache.invalidate(key); + } else { + dataCache.invalidate(key); } } @@ -90,16 +117,25 @@ public int fileReadCount() { return fileReadCount; } - private static class CacheValue { + /** The container for the segment. */ + public static class SegmentContainer { private final MemorySegment segment; - private final CacheCallback callback; - private boolean isClosed = false; + private int accessCount; - private CacheValue(MemorySegment segment, CacheCallback callback) { + public SegmentContainer(MemorySegment segment) { this.segment = segment; - this.callback = callback; + this.accessCount = 0; + } + + public MemorySegment access() { + this.accessCount++; + return segment; + } + + public int getAccessCount() { + return accessCount; } } } diff --git a/paimon-common/src/main/java/org/apache/paimon/io/cache/CaffeineCache.java b/paimon-common/src/main/java/org/apache/paimon/io/cache/CaffeineCache.java new file mode 100644 index 000000000000..8fda391fccfe --- /dev/null +++ b/paimon-common/src/main/java/org/apache/paimon/io/cache/CaffeineCache.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.io.cache; + +import javax.annotation.Nullable; + +import java.util.Map; +import java.util.function.Function; + +/** Caffeine cache implementation. */ +public class CaffeineCache implements Cache { + private final org.apache.paimon.shade.caffeine2.com.github.benmanes.caffeine.cache.Cache< + CacheKey, CacheValue> + cache; + + public CaffeineCache( + org.apache.paimon.shade.caffeine2.com.github.benmanes.caffeine.cache.Cache< + CacheKey, CacheValue> + cache) { + this.cache = cache; + } + + @Nullable + @Override + public CacheValue get(CacheKey key, Function supplier) { + return this.cache.get(key, supplier); + } + + @Override + public void put(CacheKey key, CacheValue value) { + this.cache.put(key, value); + } + + @Override + public void invalidate(CacheKey key) { + this.cache.invalidate(key); + } + + @Override + public void invalidateAll() { + this.cache.invalidateAll(); + } + + @Override + public Map asMap() { + return this.cache.asMap(); + } +} diff --git a/paimon-common/src/main/java/org/apache/paimon/io/cache/FileBasedRandomInputView.java b/paimon-common/src/main/java/org/apache/paimon/io/cache/FileBasedRandomInputView.java index b6fa559d9186..b27cab0b4c83 100644 --- a/paimon-common/src/main/java/org/apache/paimon/io/cache/FileBasedRandomInputView.java +++ b/paimon-common/src/main/java/org/apache/paimon/io/cache/FileBasedRandomInputView.java @@ -22,6 +22,7 @@ import org.apache.paimon.io.PageFileInput; import org.apache.paimon.io.SeekableDataInputView; import org.apache.paimon.io.cache.CacheKey.PageIndexCacheKey; +import org.apache.paimon.io.cache.CacheManager.SegmentContainer; import org.apache.paimon.memory.MemorySegment; import org.apache.paimon.utils.MathUtils; @@ -72,7 +73,7 @@ public void setReadPosition(long position) { private MemorySegment getCurrentPage() { SegmentContainer container = segments.get(currentSegmentIndex); - if (container == null || container.accessCount == REFRESH_COUNT) { + if (container == null || container.getAccessCount() == REFRESH_COUNT) { int pageIndex = currentSegmentIndex; MemorySegment segment = cacheManager.getPage( @@ -115,21 +116,4 @@ public void close() throws IOException { input.close(); } - - private static class SegmentContainer { - - private final MemorySegment segment; - - private int accessCount; - - private SegmentContainer(MemorySegment segment) { - this.segment = segment; - this.accessCount = 0; - } - - private MemorySegment access() { - this.accessCount++; - return segment; - } - } } diff --git a/paimon-common/src/main/java/org/apache/paimon/io/cache/GuavaCache.java b/paimon-common/src/main/java/org/apache/paimon/io/cache/GuavaCache.java new file mode 100644 index 000000000000..fab48973f57e --- /dev/null +++ b/paimon-common/src/main/java/org/apache/paimon/io/cache/GuavaCache.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.io.cache; + +import javax.annotation.Nullable; + +import java.util.Map; +import java.util.concurrent.ExecutionException; +import java.util.function.Function; + +/** Guava cache implementation. */ +public class GuavaCache implements Cache { + private final org.apache.paimon.shade.guava30.com.google.common.cache.Cache< + CacheKey, CacheValue> + cache; + + public GuavaCache( + org.apache.paimon.shade.guava30.com.google.common.cache.Cache + cache) { + this.cache = cache; + } + + @Nullable + @Override + public CacheValue get(CacheKey key, Function supplier) { + try { + return cache.get(key, () -> supplier.apply(key)); + } catch (ExecutionException e) { + throw new RuntimeException(e); + } + } + + @Override + public void put(CacheKey key, CacheValue value) { + this.cache.put(key, value); + } + + @Override + public void invalidate(CacheKey key) { + this.cache.invalidate(key); + } + + @Override + public void invalidateAll() { + this.cache.invalidateAll(); + } + + @Override + public Map asMap() { + return this.cache.asMap(); + } +} diff --git a/paimon-common/src/main/java/org/apache/paimon/lineage/LineageMeta.java b/paimon-common/src/main/java/org/apache/paimon/lineage/LineageMeta.java deleted file mode 100644 index 5d1c42daf6c8..000000000000 --- a/paimon-common/src/main/java/org/apache/paimon/lineage/LineageMeta.java +++ /dev/null @@ -1,102 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.paimon.lineage; - -import org.apache.paimon.predicate.Predicate; - -import javax.annotation.Nullable; - -import java.util.Iterator; - -/** Metadata store will manage table lineage and data lineage information for the catalog. */ -public interface LineageMeta extends AutoCloseable { - /** - * Save the source table and job lineage. - * - * @param entity the table lineage entity - */ - void saveSourceTableLineage(TableLineageEntity entity); - - /** - * Delete the source table lineage for given job. - * - * @param job the job for table lineage - */ - void deleteSourceTableLineage(String job); - - /** - * Get source table and job lineages. - * - * @param predicate the predicate for the table lineages - * @return the iterator for source table and job lineages - */ - Iterator sourceTableLineages(@Nullable Predicate predicate); - - /** - * Save the sink table and job lineage. - * - * @param entity the table lineage entity - */ - void saveSinkTableLineage(TableLineageEntity entity); - - /** - * Get sink table and job lineages. - * - * @param predicate the predicate for the table lineages - * @return the iterator for sink table and job lineages - */ - Iterator sinkTableLineages(@Nullable Predicate predicate); - - /** - * Delete the sink table lineage for given job. - * - * @param job the job for table lineage - */ - void deleteSinkTableLineage(String job); - - /** - * Save the source table and job lineage. - * - * @param entity the data lineage entity - */ - void saveSourceDataLineage(DataLineageEntity entity); - - /** - * Get source data and job lineages. - * - * @param predicate the predicate for the table lineages - * @return the iterator for source table and job lineages - */ - Iterator sourceDataLineages(@Nullable Predicate predicate); - - /** - * Save the sink table and job lineage. - * - * @param entity the data lineage entity - */ - void saveSinkDataLineage(DataLineageEntity entity); - - /** - * Get sink data and job lineages. - * - * @param predicate the predicate for the table lineages - * @return the iterator for sink table and job lineages - */ - Iterator sinkDataLineages(@Nullable Predicate predicate); -} diff --git a/paimon-common/src/main/java/org/apache/paimon/lookup/hash/HashLookupStoreReader.java b/paimon-common/src/main/java/org/apache/paimon/lookup/hash/HashLookupStoreReader.java index c984dafdd5c3..742cc1a60bbe 100644 --- a/paimon-common/src/main/java/org/apache/paimon/lookup/hash/HashLookupStoreReader.java +++ b/paimon-common/src/main/java/org/apache/paimon/lookup/hash/HashLookupStoreReader.java @@ -169,6 +169,9 @@ private byte[] getValue(long offset) throws IOException { @Override public void close() throws IOException { + if (bloomFilter != null) { + bloomFilter.close(); + } inputView.close(); inputView = null; } diff --git a/paimon-common/src/main/java/org/apache/paimon/lookup/sort/BlockCache.java b/paimon-common/src/main/java/org/apache/paimon/lookup/sort/BlockCache.java new file mode 100644 index 000000000000..0441a24f220e --- /dev/null +++ b/paimon-common/src/main/java/org/apache/paimon/lookup/sort/BlockCache.java @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.lookup.sort; + +import org.apache.paimon.io.cache.CacheKey; +import org.apache.paimon.io.cache.CacheManager; +import org.apache.paimon.io.cache.CacheManager.SegmentContainer; +import org.apache.paimon.memory.MemorySegment; + +import java.io.Closeable; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.function.Function; + +/** Cache for block reading. */ +public class BlockCache implements Closeable { + + private final RandomAccessFile file; + private final FileChannel channel; + private final CacheManager cacheManager; + private final Map blocks; + + public BlockCache(RandomAccessFile file, CacheManager cacheManager) { + this.file = file; + this.channel = this.file.getChannel(); + this.cacheManager = cacheManager; + this.blocks = new HashMap<>(); + } + + private byte[] readFrom(long offset, int length) throws IOException { + byte[] buffer = new byte[length]; + int read = channel.read(ByteBuffer.wrap(buffer), offset); + + if (read != length) { + throw new IOException("Could not read all the data"); + } + return buffer; + } + + public MemorySegment getBlock( + long position, int length, Function decompressFunc, boolean isIndex) { + + CacheKey cacheKey = CacheKey.forPosition(file, position, length, isIndex); + + SegmentContainer container = blocks.get(cacheKey); + if (container == null || container.getAccessCount() == CacheManager.REFRESH_COUNT) { + MemorySegment segment = + cacheManager.getPage( + cacheKey, + key -> { + byte[] bytes = readFrom(position, length); + return decompressFunc.apply(bytes); + }, + blocks::remove); + container = new SegmentContainer(segment); + blocks.put(cacheKey, container); + } + return container.access(); + } + + @Override + public void close() throws IOException { + Set sets = new HashSet<>(blocks.keySet()); + for (CacheKey key : sets) { + cacheManager.invalidPage(key); + } + } +} diff --git a/paimon-common/src/main/java/org/apache/paimon/lookup/sort/SortLookupStoreFactory.java b/paimon-common/src/main/java/org/apache/paimon/lookup/sort/SortLookupStoreFactory.java index 62c4131e541b..244d7f9dd7e4 100644 --- a/paimon-common/src/main/java/org/apache/paimon/lookup/sort/SortLookupStoreFactory.java +++ b/paimon-common/src/main/java/org/apache/paimon/lookup/sort/SortLookupStoreFactory.java @@ -52,7 +52,8 @@ public SortLookupStoreFactory( @Override public SortLookupStoreReader createReader(File file, Context context) throws IOException { - return new SortLookupStoreReader(comparator, file, (SortContext) context, cacheManager); + return new SortLookupStoreReader( + comparator, file, blockSize, (SortContext) context, cacheManager); } @Override diff --git a/paimon-common/src/main/java/org/apache/paimon/lookup/sort/SortLookupStoreReader.java b/paimon-common/src/main/java/org/apache/paimon/lookup/sort/SortLookupStoreReader.java index 5dec3c06ebcc..6dbfe130e3bb 100644 --- a/paimon-common/src/main/java/org/apache/paimon/lookup/sort/SortLookupStoreReader.java +++ b/paimon-common/src/main/java/org/apache/paimon/lookup/sort/SortLookupStoreReader.java @@ -20,73 +20,67 @@ import org.apache.paimon.compression.BlockCompressionFactory; import org.apache.paimon.compression.BlockDecompressor; +import org.apache.paimon.io.PageFileInput; import org.apache.paimon.io.cache.CacheManager; import org.apache.paimon.lookup.LookupStoreReader; import org.apache.paimon.memory.MemorySegment; import org.apache.paimon.memory.MemorySlice; import org.apache.paimon.memory.MemorySliceInput; -import org.apache.paimon.utils.BloomFilter; +import org.apache.paimon.utils.FileBasedBloomFilter; import org.apache.paimon.utils.MurmurHashUtils; import javax.annotation.Nullable; import java.io.File; -import java.io.FileInputStream; import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.channels.FileChannel; import java.util.Comparator; import static org.apache.paimon.lookup.sort.SortLookupStoreUtils.crc32c; import static org.apache.paimon.utils.Preconditions.checkArgument; -/** - * A {@link LookupStoreReader} for sort store. - * - *

    TODO add block cache support. - * - *

    TODO separate index cache and block cache. - */ +/** A {@link LookupStoreReader} for sort store. */ public class SortLookupStoreReader implements LookupStoreReader { private final Comparator comparator; - private final FileChannel fileChannel; private final String filePath; private final long fileSize; private final BlockIterator indexBlockIterator; - @Nullable private final BloomFilter bloomFilter; + @Nullable private FileBasedBloomFilter bloomFilter; + private final BlockCache blockCache; + private final PageFileInput fileInput; public SortLookupStoreReader( Comparator comparator, File file, + int blockSize, SortContext context, CacheManager cacheManager) throws IOException { this.comparator = comparator; - //noinspection resource - this.fileChannel = new FileInputStream(file).getChannel(); this.filePath = file.getAbsolutePath(); this.fileSize = context.fileSize(); + this.fileInput = PageFileInput.create(file, blockSize, null, fileSize, null); + this.blockCache = new BlockCache(fileInput.file(), cacheManager); Footer footer = readFooter(); - this.indexBlockIterator = readBlock(footer.getIndexBlockHandle()).iterator(); - this.bloomFilter = readBloomFilter(footer.getBloomFilterHandle()); - } - - private BloomFilter readBloomFilter(@Nullable BloomFilterHandle bloomFilterHandle) - throws IOException { - BloomFilter bloomFilter = null; - if (bloomFilterHandle != null) { - MemorySegment segment = read(bloomFilterHandle.offset(), bloomFilterHandle.size()); - bloomFilter = new BloomFilter(bloomFilterHandle.expectedEntries(), segment.size()); - bloomFilter.setMemorySegment(segment, 0); + this.indexBlockIterator = readBlock(footer.getIndexBlockHandle(), true).iterator(); + BloomFilterHandle handle = footer.getBloomFilterHandle(); + if (handle != null) { + this.bloomFilter = + new FileBasedBloomFilter( + fileInput, + cacheManager, + handle.expectedEntries(), + handle.offset(), + handle.size()); } - return bloomFilter; } private Footer readFooter() throws IOException { - MemorySegment footerData = read(fileSize - Footer.ENCODED_LENGTH, Footer.ENCODED_LENGTH); + MemorySegment footerData = + blockCache.getBlock( + fileSize - Footer.ENCODED_LENGTH, Footer.ENCODED_LENGTH, b -> b, true); return Footer.readFooter(MemorySlice.wrap(footerData).toInput()); } @@ -112,38 +106,42 @@ public byte[] lookup(byte[] key) throws IOException { return null; } - private BlockIterator getNextBlock() throws IOException { + private BlockIterator getNextBlock() { + // index block handle, point to the key, value position. MemorySlice blockHandle = indexBlockIterator.next().getValue(); - BlockReader dataBlock = openBlock(blockHandle); + BlockReader dataBlock = + readBlock(BlockHandle.readBlockHandle(blockHandle.toInput()), false); return dataBlock.iterator(); } - private BlockReader openBlock(MemorySlice blockEntry) throws IOException { - BlockHandle blockHandle = BlockHandle.readBlockHandle(blockEntry.toInput()); - return readBlock(blockHandle); - } - - private MemorySegment read(long offset, int length) throws IOException { - // TODO use cache - // TODO cache uncompressed block - // TODO separate index and data cache - byte[] buffer = new byte[length]; - int read = fileChannel.read(ByteBuffer.wrap(buffer), offset); - if (read != length) { - throw new IOException("Could not read all the data"); - } - return MemorySegment.wrap(buffer); - } - - private BlockReader readBlock(BlockHandle blockHandle) throws IOException { + /** + * @param blockHandle The block handle. + * @param index Whether read the block as an index. + * @return The reader of the target block. + */ + private BlockReader readBlock(BlockHandle blockHandle, boolean index) { // read block trailer MemorySegment trailerData = - read(blockHandle.offset() + blockHandle.size(), BlockTrailer.ENCODED_LENGTH); + blockCache.getBlock( + blockHandle.offset() + blockHandle.size(), + BlockTrailer.ENCODED_LENGTH, + b -> b, + true); BlockTrailer blockTrailer = BlockTrailer.readBlockTrailer(MemorySlice.wrap(trailerData).toInput()); - MemorySegment block = read(blockHandle.offset(), blockHandle.size()); - int crc32cCode = crc32c(block, blockTrailer.getCompressionType()); + MemorySegment unCompressedBlock = + blockCache.getBlock( + blockHandle.offset(), + blockHandle.size(), + bytes -> decompressBlock(bytes, blockTrailer), + index); + return new BlockReader(MemorySlice.wrap(unCompressedBlock), comparator); + } + + private byte[] decompressBlock(byte[] compressedBytes, BlockTrailer blockTrailer) { + MemorySegment compressed = MemorySegment.wrap(compressedBytes); + int crc32cCode = crc32c(compressed, blockTrailer.getCompressionType()); checkArgument( blockTrailer.getCrc32c() == crc32cCode, String.format( @@ -151,32 +149,32 @@ private BlockReader readBlock(BlockHandle blockHandle) throws IOException { blockTrailer.getCrc32c(), crc32cCode, filePath)); // decompress data - MemorySlice uncompressedData; BlockCompressionFactory compressionFactory = BlockCompressionFactory.create(blockTrailer.getCompressionType()); if (compressionFactory == null) { - uncompressedData = MemorySlice.wrap(block); + return compressedBytes; } else { - MemorySliceInput compressedInput = MemorySlice.wrap(block).toInput(); + MemorySliceInput compressedInput = MemorySlice.wrap(compressed).toInput(); byte[] uncompressed = new byte[compressedInput.readVarLenInt()]; BlockDecompressor decompressor = compressionFactory.getDecompressor(); int uncompressedLength = decompressor.decompress( - block.getHeapMemory(), + compressed.getHeapMemory(), compressedInput.position(), compressedInput.available(), uncompressed, 0); checkArgument(uncompressedLength == uncompressed.length); - uncompressedData = MemorySlice.wrap(uncompressed); + return uncompressed; } - - return new BlockReader(uncompressedData, comparator); } @Override public void close() throws IOException { - this.fileChannel.close(); - // TODO clear cache too + if (bloomFilter != null) { + bloomFilter.close(); + } + blockCache.close(); + fileInput.close(); } } diff --git a/paimon-common/src/main/java/org/apache/paimon/options/CatalogOptions.java b/paimon-common/src/main/java/org/apache/paimon/options/CatalogOptions.java index 081668675dd9..bb8cfae68284 100644 --- a/paimon-common/src/main/java/org/apache/paimon/options/CatalogOptions.java +++ b/paimon-common/src/main/java/org/apache/paimon/options/CatalogOptions.java @@ -18,8 +18,6 @@ package org.apache.paimon.options; -import org.apache.paimon.options.description.Description; -import org.apache.paimon.options.description.TextElement; import org.apache.paimon.table.CatalogTableType; import java.time.Duration; @@ -94,10 +92,17 @@ public class CatalogOptions { public static final ConfigOption CACHE_EXPIRATION_INTERVAL_MS = key("cache.expiration-interval") .durationType() - .defaultValue(Duration.ofSeconds(60)) + .defaultValue(Duration.ofMinutes(10)) .withDescription( "Controls the duration for which databases and tables in the catalog are cached."); + public static final ConfigOption CACHE_PARTITION_MAX_NUM = + key("cache.partition.max-num") + .longType() + .defaultValue(0L) + .withDescription( + "Controls the max number for which partitions in the catalog are cached."); + public static final ConfigOption CACHE_MANIFEST_SMALL_FILE_MEMORY = key("cache.manifest.small-file-memory") .memoryType() @@ -116,25 +121,12 @@ public class CatalogOptions { .noDefaultValue() .withDescription("Controls the maximum memory to cache manifest content."); - public static final ConfigOption LINEAGE_META = - key("lineage-meta") - .stringType() - .noDefaultValue() + public static final ConfigOption CACHE_SNAPSHOT_MAX_NUM_PER_TABLE = + key("cache.snapshot.max-num-per-table") + .intType() + .defaultValue(20) .withDescription( - Description.builder() - .text( - "The lineage meta to store table and data lineage information.") - .linebreak() - .linebreak() - .text("Possible values:") - .linebreak() - .list( - TextElement.text( - "\"jdbc\": Use standard jdbc to store table and data lineage information.")) - .list( - TextElement.text( - "\"custom\": You can implement LineageMetaFactory and LineageMeta to store lineage information in customized storage.")) - .build()); + "Controls the max number for snapshots per table in the catalog are cached."); public static final ConfigOption ALLOW_UPPER_CASE = ConfigOptions.key("allow-upper-case") @@ -149,4 +141,13 @@ public class CatalogOptions { .booleanType() .defaultValue(false) .withDescription("Sync all table properties to hive metastore"); + + public static final ConfigOption FORMAT_TABLE_ENABLED = + ConfigOptions.key("format-table.enabled") + .booleanType() + .defaultValue(true) + .withDescription( + "Whether to support format tables, format table corresponds to a regular csv, parquet or orc table, allowing read and write operations. " + + "However, during these processes, it does not connect to the metastore; hence, newly added partitions will not be reflected in" + + " the metastore and need to be manually added as separate partition operations."); } diff --git a/paimon-common/src/main/java/org/apache/paimon/options/OptionsUtils.java b/paimon-common/src/main/java/org/apache/paimon/options/OptionsUtils.java index 47eb45007197..a625454f3996 100644 --- a/paimon-common/src/main/java/org/apache/paimon/options/OptionsUtils.java +++ b/paimon-common/src/main/java/org/apache/paimon/options/OptionsUtils.java @@ -27,6 +27,8 @@ import java.util.Locale; import java.util.Map; import java.util.function.Predicate; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import java.util.stream.Collectors; import static org.apache.paimon.options.StructuredOptionsSplitter.escapeWithSingleQuote; @@ -302,6 +304,34 @@ public static Map convertToPropertiesPrefixKey( return properties; } + public static Map convertToDynamicTableProperties( + Map confData, + String globalOptionKeyPrefix, + Pattern tableOptionKeyPattern, + int keyGroup) { + Map globalOptions = new HashMap<>(); + Map tableOptions = new HashMap<>(); + + confData.keySet().stream() + .filter(k -> k.startsWith(globalOptionKeyPrefix)) + .forEach( + k -> { + Matcher matcher = tableOptionKeyPattern.matcher(k); + if (matcher.find()) { + tableOptions.put( + matcher.group(keyGroup), convertToString(confData.get(k))); + } else { + globalOptions.put( + k.substring(globalOptionKeyPrefix.length()), + convertToString(confData.get(k))); + } + }); + + // table options should override global options for the same key + globalOptions.putAll(tableOptions); + return globalOptions; + } + static boolean containsPrefixMap(Map confData, String key) { return confData.keySet().stream().anyMatch(candidate -> filterPrefixMapKey(key, candidate)); } diff --git a/paimon-common/src/main/java/org/apache/paimon/predicate/InPredicateVisitor.java b/paimon-common/src/main/java/org/apache/paimon/predicate/InPredicateVisitor.java new file mode 100644 index 000000000000..dbd31dd2d191 --- /dev/null +++ b/paimon-common/src/main/java/org/apache/paimon/predicate/InPredicateVisitor.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.predicate; + +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; + +/** A utils to handle {@link Predicate}. */ +public class InPredicateVisitor { + + /** + * Method for handling with In CompoundPredicate. + * + * @param predicate CompoundPredicate to traverse handle + * @param leafName LeafPredicate name + */ + public static Optional> extractInElements(Predicate predicate, String leafName) { + if (!(predicate instanceof CompoundPredicate)) { + return Optional.empty(); + } + + CompoundPredicate compoundPredicate = (CompoundPredicate) predicate; + List leafValues = new ArrayList<>(); + List children = compoundPredicate.children(); + for (Predicate leaf : children) { + if (leaf instanceof LeafPredicate + && (((LeafPredicate) leaf).function() instanceof Equal) + && leaf.visit(LeafPredicateExtractor.INSTANCE).get(leafName) != null) { + leafValues.add(((LeafPredicate) leaf).literals().get(0)); + } else { + return Optional.empty(); + } + } + return Optional.of(leafValues); + } +} diff --git a/paimon-common/src/main/java/org/apache/paimon/reader/EmptyFileRecordReader.java b/paimon-common/src/main/java/org/apache/paimon/reader/EmptyFileRecordReader.java new file mode 100644 index 000000000000..3fa25dce5c49 --- /dev/null +++ b/paimon-common/src/main/java/org/apache/paimon/reader/EmptyFileRecordReader.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.reader; + +import javax.annotation.Nullable; + +import java.io.IOException; + +/** An empty {@link FileRecordReader}. */ +public class EmptyFileRecordReader implements FileRecordReader { + + @Nullable + @Override + public FileRecordIterator readBatch() throws IOException { + return null; + } + + @Override + public void close() throws IOException {} +} diff --git a/paimon-common/src/main/java/org/apache/paimon/reader/FileRecordIterator.java b/paimon-common/src/main/java/org/apache/paimon/reader/FileRecordIterator.java index d22b27053f98..2d3c85f193dc 100644 --- a/paimon-common/src/main/java/org/apache/paimon/reader/FileRecordIterator.java +++ b/paimon-common/src/main/java/org/apache/paimon/reader/FileRecordIterator.java @@ -27,10 +27,8 @@ import java.util.function.Function; /** - * Wrap {@link RecordReader.RecordIterator} to support returning the record's row position and file + * A {@link RecordReader.RecordIterator} to support returning the record's row position and file * Path. - * - * @param The type of the record. */ public interface FileRecordIterator extends RecordReader.RecordIterator { diff --git a/paimon-common/src/main/java/org/apache/paimon/reader/FileRecordReader.java b/paimon-common/src/main/java/org/apache/paimon/reader/FileRecordReader.java new file mode 100644 index 000000000000..4d5356edf275 --- /dev/null +++ b/paimon-common/src/main/java/org/apache/paimon/reader/FileRecordReader.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.reader; + +import javax.annotation.Nullable; + +import java.io.IOException; + +/** A {@link RecordReader} to support returning {@link FileRecordIterator}. */ +public interface FileRecordReader extends RecordReader { + + @Override + @Nullable + FileRecordIterator readBatch() throws IOException; +} diff --git a/paimon-common/src/main/java/org/apache/paimon/reader/PackChangelogReader.java b/paimon-common/src/main/java/org/apache/paimon/reader/PackChangelogReader.java new file mode 100644 index 000000000000..a60780ff5e06 --- /dev/null +++ b/paimon-common/src/main/java/org/apache/paimon/reader/PackChangelogReader.java @@ -0,0 +1,131 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.reader; + +import org.apache.paimon.data.InternalRow; +import org.apache.paimon.data.serializer.InternalRowSerializer; +import org.apache.paimon.types.RowKind; +import org.apache.paimon.types.RowType; + +import javax.annotation.Nullable; + +import java.io.IOException; +import java.util.function.BiFunction; + +/** The reader which will pack the update before and update after message together. */ +public class PackChangelogReader implements RecordReader { + + private final RecordReader reader; + private final BiFunction function; + private final InternalRowSerializer serializer; + private boolean initialized = false; + + public PackChangelogReader( + RecordReader reader, + BiFunction function, + RowType rowType) { + this.reader = reader; + this.function = function; + this.serializer = new InternalRowSerializer(rowType); + } + + @Nullable + @Override + public RecordIterator readBatch() throws IOException { + if (!initialized) { + initialized = true; + return new InternRecordIterator(reader, function, serializer); + } + return null; + } + + @Override + public void close() throws IOException { + reader.close(); + } + + private static class InternRecordIterator implements RecordIterator { + + private RecordIterator currentBatch; + + private final BiFunction function; + private final RecordReader reader; + private final InternalRowSerializer serializer; + private boolean endOfData; + + public InternRecordIterator( + RecordReader reader, + BiFunction function, + InternalRowSerializer serializer) { + this.reader = reader; + this.function = function; + this.serializer = serializer; + this.endOfData = false; + } + + @Nullable + @Override + public InternalRow next() throws IOException { + InternalRow row1 = nextRow(); + if (row1 == null) { + return null; + } + InternalRow row2 = null; + if (row1.getRowKind() == RowKind.UPDATE_BEFORE) { + row1 = serializer.copy(row1); + row2 = nextRow(); + } + return function.apply(row1, row2); + } + + @Nullable + private InternalRow nextRow() throws IOException { + InternalRow row = null; + while (!endOfData && row == null) { + RecordIterator batch = nextBatch(); + if (batch == null) { + endOfData = true; + return null; + } + + row = batch.next(); + if (row == null) { + releaseBatch(); + } + } + return row; + } + + @Nullable + private RecordIterator nextBatch() throws IOException { + if (currentBatch == null) { + currentBatch = reader.readBatch(); + } + return currentBatch; + } + + @Override + public void releaseBatch() { + if (currentBatch != null) { + currentBatch.releaseBatch(); + currentBatch = null; + } + } + } +} diff --git a/paimon-common/src/main/java/org/apache/paimon/statistics/NoneSimpleColStatsCollector.java b/paimon-common/src/main/java/org/apache/paimon/statistics/NoneSimpleColStatsCollector.java index d4f645b49957..57369c9cd89e 100644 --- a/paimon-common/src/main/java/org/apache/paimon/statistics/NoneSimpleColStatsCollector.java +++ b/paimon-common/src/main/java/org/apache/paimon/statistics/NoneSimpleColStatsCollector.java @@ -29,11 +29,11 @@ public void collect(Object field, Serializer fieldSerializer) {} @Override public SimpleColStats result() { - return new SimpleColStats(null, null, null); + return SimpleColStats.NONE; } @Override public SimpleColStats convert(SimpleColStats source) { - return new SimpleColStats(null, null, null); + return SimpleColStats.NONE; } } diff --git a/paimon-common/src/main/java/org/apache/paimon/statistics/TruncateSimpleColStatsCollector.java b/paimon-common/src/main/java/org/apache/paimon/statistics/TruncateSimpleColStatsCollector.java index 926e287f3cc7..cba942c7905c 100644 --- a/paimon-common/src/main/java/org/apache/paimon/statistics/TruncateSimpleColStatsCollector.java +++ b/paimon-common/src/main/java/org/apache/paimon/statistics/TruncateSimpleColStatsCollector.java @@ -53,6 +53,11 @@ public void collect(Object field, Serializer fieldSerializer) { return; } + // fast fail since the result is not correct + if (failed) { + return; + } + // TODO use comparator for not comparable types and extract this logic to a util class if (!(field instanceof Comparable)) { return; @@ -66,7 +71,12 @@ public void collect(Object field, Serializer fieldSerializer) { Object max = truncateMax(field); // may fail if (max != null) { - maxValue = fieldSerializer.copy(truncateMax(field)); + if (max != field) { + // copied in `truncateMax` + maxValue = max; + } else { + maxValue = fieldSerializer.copy(max); + } } } } diff --git a/paimon-common/src/main/java/org/apache/paimon/table/SpecialFields.java b/paimon-common/src/main/java/org/apache/paimon/table/SpecialFields.java new file mode 100644 index 000000000000..3288276a1f64 --- /dev/null +++ b/paimon-common/src/main/java/org/apache/paimon/table/SpecialFields.java @@ -0,0 +1,130 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.table; + +import org.apache.paimon.types.DataField; +import org.apache.paimon.types.DataTypes; +import org.apache.paimon.types.VarCharType; + +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +/** + * Special fields in a {@link org.apache.paimon.types.RowType} with specific field ids. + * + *

    System fields: + * + *

      + *
    • _KEY_<key-field>: Keys of a key-value. ID = 1073741823 + + * (field-id) + * . + *
    • _SEQUENCE_NUMBER: Sequence number of a key-value. ID = 2147483646. + *
    • _VALUE_KIND: Type of a key-value. See {@link org.apache.paimon.types.RowKind}. + * ID = 2147483645. + *
    • _LEVEL: Which LSM tree level does this key-value stay in. ID = 2147483644. + *
    • rowkind: THw rowkind field in audit-log system tables. ID = 2147483643. + *
    + * + *

    Structured type fields: + * + *

    These ids are mainly used as field ids in parquet files, so compute engines can read a field + * directly by id. These ids are not stored in {@link org.apache.paimon.types.DataField}. + * + *

      + *
    • Array element field: ID = 536870911 + 1024 * (array-field-id) + depth. + *
    • Map key field: ID = 536870911 - 1024 * (array-field-id) - depth. + *
    • Map value field: ID = 536870911 + 1024 * (array-field-id) + depth. + *
    + * + *

    Examples: + * + *

      + *
    • ARRAY(MAP(INT, ARRAY(INT))) type, outer array has field id 10, then map (element of outer + * array) has field id 536870911 + 1024 * 10 + 1, map key (int) has field id 536870911 - 1024 + * * 10 - 2, map value (inner array) has field id 536870911 + 1024 * 10 + 2, inner array + * element (int) has field id 536870911 + 1024 * 10 + 3 + *
    + */ +public class SpecialFields { + + // ---------------------------------------------------------------------------------------- + // System fields + // ---------------------------------------------------------------------------------------- + + public static final int SYSTEM_FIELD_ID_START = Integer.MAX_VALUE / 2; + + public static final String KEY_FIELD_PREFIX = "_KEY_"; + public static final int KEY_FIELD_ID_START = SYSTEM_FIELD_ID_START; + + public static final DataField SEQUENCE_NUMBER = + new DataField(Integer.MAX_VALUE - 1, "_SEQUENCE_NUMBER", DataTypes.BIGINT().notNull()); + + public static final DataField VALUE_KIND = + new DataField(Integer.MAX_VALUE - 2, "_VALUE_KIND", DataTypes.TINYINT().notNull()); + + public static final DataField LEVEL = + new DataField(Integer.MAX_VALUE - 3, "_LEVEL", DataTypes.INT().notNull()); + + // only used by AuditLogTable + public static final DataField ROW_KIND = + new DataField( + Integer.MAX_VALUE - 4, "rowkind", new VarCharType(VarCharType.MAX_LENGTH)); + + public static final Set SYSTEM_FIELD_NAMES = + Stream.of(SEQUENCE_NUMBER.name(), VALUE_KIND.name(), LEVEL.name(), ROW_KIND.name()) + .collect(Collectors.toSet()); + + public static boolean isSystemField(int fieldId) { + return fieldId >= SYSTEM_FIELD_ID_START; + } + + public static boolean isSystemField(String field) { + return field.startsWith(KEY_FIELD_PREFIX) || SYSTEM_FIELD_NAMES.contains(field); + } + + public static boolean isKeyField(String field) { + return field.startsWith(KEY_FIELD_PREFIX); + } + + // ---------------------------------------------------------------------------------------- + // Structured type fields + // ---------------------------------------------------------------------------------------- + + public static final int STRUCTURED_TYPE_FIELD_ID_BASE = Integer.MAX_VALUE / 4; + public static final int STRUCTURED_TYPE_FIELD_DEPTH_LIMIT = 1 << 10; + + public static int getArrayElementFieldId(int arrayFieldId, int depth) { + return STRUCTURED_TYPE_FIELD_ID_BASE + + arrayFieldId * STRUCTURED_TYPE_FIELD_DEPTH_LIMIT + + depth; + } + + public static int getMapKeyFieldId(int mapFieldId, int depth) { + return STRUCTURED_TYPE_FIELD_ID_BASE + - mapFieldId * STRUCTURED_TYPE_FIELD_DEPTH_LIMIT + - depth; + } + + public static int getMapValueFieldId(int mapFieldId, int depth) { + return STRUCTURED_TYPE_FIELD_ID_BASE + + mapFieldId * STRUCTURED_TYPE_FIELD_DEPTH_LIMIT + + depth; + } +} diff --git a/paimon-common/src/main/java/org/apache/paimon/table/SystemFields.java b/paimon-common/src/main/java/org/apache/paimon/table/SystemFields.java deleted file mode 100644 index 4ed212f362e2..000000000000 --- a/paimon-common/src/main/java/org/apache/paimon/table/SystemFields.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.paimon.table; - -import org.apache.paimon.types.DataField; -import org.apache.paimon.types.DataTypes; -import org.apache.paimon.types.VarCharType; - -import java.util.Set; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -/** System fields. */ -public class SystemFields { - - public static final int SYSTEM_FIELD_ID_START = Integer.MAX_VALUE / 2; - - public static final String KEY_FIELD_PREFIX = "_KEY_"; - public static final int KEY_FIELD_ID_START = SYSTEM_FIELD_ID_START; - - public static final DataField SEQUENCE_NUMBER = - new DataField(Integer.MAX_VALUE - 1, "_SEQUENCE_NUMBER", DataTypes.BIGINT().notNull()); - - public static final DataField VALUE_KIND = - new DataField(Integer.MAX_VALUE - 2, "_VALUE_KIND", DataTypes.TINYINT().notNull()); - - public static final DataField LEVEL = - new DataField(Integer.MAX_VALUE - 3, "_LEVEL", DataTypes.INT().notNull()); - - // only used by AuditLogTable - public static final DataField ROW_KIND = - new DataField( - Integer.MAX_VALUE - 4, "rowkind", new VarCharType(VarCharType.MAX_LENGTH)); - - public static final Set SYSTEM_FIELD_NAMES = - Stream.of(SEQUENCE_NUMBER.name(), VALUE_KIND.name(), LEVEL.name(), ROW_KIND.name()) - .collect(Collectors.toSet()); - - public static boolean isSystemField(int fieldId) { - return fieldId >= SYSTEM_FIELD_ID_START; - } -} diff --git a/paimon-common/src/main/java/org/apache/paimon/types/DataTypes.java b/paimon-common/src/main/java/org/apache/paimon/types/DataTypes.java index 659212b064eb..b025b6a838e0 100644 --- a/paimon-common/src/main/java/org/apache/paimon/types/DataTypes.java +++ b/paimon-common/src/main/java/org/apache/paimon/types/DataTypes.java @@ -103,6 +103,10 @@ public static LocalZonedTimestampType TIMESTAMP_WITH_LOCAL_TIME_ZONE(int precisi return new LocalZonedTimestampType(precision); } + public static LocalZonedTimestampType TIMESTAMP_LTZ_MILLIS() { + return new LocalZonedTimestampType(3); + } + public static DecimalType DECIMAL(int precision, int scale) { return new DecimalType(precision, scale); } diff --git a/paimon-common/src/main/java/org/apache/paimon/types/FieldIdentifier.java b/paimon-common/src/main/java/org/apache/paimon/types/FieldIdentifier.java new file mode 100644 index 000000000000..7e9ced7cf95a --- /dev/null +++ b/paimon-common/src/main/java/org/apache/paimon/types/FieldIdentifier.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.types; + +import java.util.Objects; + +/** Used to indicate the uniqueness of a field. */ +public class FieldIdentifier { + private String name; + private DataType type; + private String description; + + public FieldIdentifier(DataField dataField) { + this.name = dataField.name(); + this.type = dataField.type(); + this.description = dataField.description(); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + FieldIdentifier field = (FieldIdentifier) o; + return Objects.equals(name, field.name) + && Objects.equals(type, field.type) + && Objects.equals(description, field.description); + } + + @Override + public int hashCode() { + return Objects.hash(name, type, description); + } +} diff --git a/paimon-common/src/main/java/org/apache/paimon/types/RowType.java b/paimon-common/src/main/java/org/apache/paimon/types/RowType.java index 9dc89117f331..f3fce0db6df1 100644 --- a/paimon-common/src/main/java/org/apache/paimon/types/RowType.java +++ b/paimon-common/src/main/java/org/apache/paimon/types/RowType.java @@ -20,7 +20,7 @@ import org.apache.paimon.annotation.Public; import org.apache.paimon.data.InternalRow; -import org.apache.paimon.table.SystemFields; +import org.apache.paimon.table.SpecialFields; import org.apache.paimon.utils.Preconditions; import org.apache.paimon.utils.StringUtils; @@ -169,11 +169,16 @@ public int defaultSize() { } @Override - public DataType copy(boolean isNullable) { + public RowType copy(boolean isNullable) { return new RowType( isNullable, fields.stream().map(DataField::copy).collect(Collectors.toList())); } + @Override + public RowType notNull() { + return copy(false); + } + @Override public String asSQLString() { return withNullability( @@ -332,7 +337,7 @@ public static int currentHighestFieldId(List fields) { Set fieldIds = new HashSet<>(); new RowType(fields).collectFieldIds(fieldIds); return fieldIds.stream() - .filter(i -> !SystemFields.isSystemField(i)) + .filter(i -> !SpecialFields.isSystemField(i)) .max(Integer::compareTo) .orElse(-1); } diff --git a/paimon-common/src/main/java/org/apache/paimon/utils/BitSliceIndexRoaringBitmap.java b/paimon-common/src/main/java/org/apache/paimon/utils/BitSliceIndexRoaringBitmap.java new file mode 100644 index 000000000000..c2dcf58e85ad --- /dev/null +++ b/paimon-common/src/main/java/org/apache/paimon/utils/BitSliceIndexRoaringBitmap.java @@ -0,0 +1,324 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.utils; + +import org.apache.paimon.annotation.VisibleForTesting; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.util.Arrays; +import java.util.Objects; +import java.util.Optional; +import java.util.function.Supplier; + +/* This file is based on source code from the RoaringBitmap Project (http://roaringbitmap.org/), licensed by the Apache + * Software Foundation (ASF) under the Apache License, Version 2.0. See the NOTICE file distributed with this work for + * additional information regarding copyright ownership. */ + +/** A bit slice index compressed bitmap. */ +public class BitSliceIndexRoaringBitmap { + + public static final byte VERSION_1 = 1; + + public static final BitSliceIndexRoaringBitmap EMPTY = + new BitSliceIndexRoaringBitmap(0, 0, new RoaringBitmap32(), new RoaringBitmap32[] {}); + + private final long min; + private final long max; + private final RoaringBitmap32 ebm; + private final RoaringBitmap32[] slices; + + private BitSliceIndexRoaringBitmap( + long min, long max, RoaringBitmap32 ebm, RoaringBitmap32[] slices) { + this.min = min; + this.max = max; + this.ebm = ebm; + this.slices = slices; + } + + public RoaringBitmap32 eq(long predicate) { + return compare(Operation.EQ, predicate, null); + } + + public RoaringBitmap32 lt(long predicate) { + return compare(Operation.LT, predicate, null); + } + + public RoaringBitmap32 lte(long predicate) { + return compare(Operation.LTE, predicate, null); + } + + public RoaringBitmap32 gt(long predicate) { + return compare(Operation.GT, predicate, null); + } + + public RoaringBitmap32 gte(long predicate) { + return compare(Operation.GTE, predicate, null); + } + + public RoaringBitmap32 isNotNull() { + return ebm.clone(); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + BitSliceIndexRoaringBitmap that = (BitSliceIndexRoaringBitmap) o; + return min == that.min + && Objects.equals(ebm, that.ebm) + && Arrays.equals(slices, that.slices); + } + + private RoaringBitmap32 compare(Operation operation, long predicate, RoaringBitmap32 foundSet) { + // using min/max to fast skip + return compareUsingMinMax(operation, predicate, foundSet) + .orElseGet(() -> oNeilCompare(operation, predicate - min, foundSet)); + } + + @VisibleForTesting + protected Optional compareUsingMinMax( + Operation operation, long predicate, RoaringBitmap32 foundSet) { + Supplier> empty = () -> Optional.of(new RoaringBitmap32()); + Supplier> all = + () -> { + if (foundSet == null) { + return Optional.of(isNotNull()); + } else { + return Optional.of(RoaringBitmap32.and(foundSet, ebm)); + } + }; + + switch (operation) { + case EQ: + { + if (min == max && min == predicate) { + return all.get(); + } else if (predicate < min || predicate > max) { + return empty.get(); + } + break; + } + case NEQ: + { + if (min == max && min == predicate) { + return empty.get(); + } else if (predicate < min || predicate > max) { + return all.get(); + } + break; + } + case GTE: + { + if (predicate <= min) { + return all.get(); + } else if (predicate > max) { + return empty.get(); + } + break; + } + case GT: + { + if (predicate < min) { + return all.get(); + } else if (predicate >= max) { + return empty.get(); + } + break; + } + case LTE: + { + if (predicate >= max) { + return all.get(); + } else if (predicate < min) { + return empty.get(); + } + break; + } + case LT: + { + if (predicate > max) { + return all.get(); + } else if (predicate <= min) { + return empty.get(); + } + break; + } + default: + throw new IllegalArgumentException("not support operation: " + operation); + } + return Optional.empty(); + } + + /** + * O'Neil bit-sliced index compare algorithm. + * + *

    See Improved query performance with + * variant indexes + * + * @param operation compare operation + * @param predicate the value we found filter + * @param foundSet rid set we want compare, using RoaringBitmap to express + * @return rid set we found in this bsi with giving conditions, using RoaringBitmap to express + */ + private RoaringBitmap32 oNeilCompare( + Operation operation, long predicate, RoaringBitmap32 foundSet) { + RoaringBitmap32 fixedFoundSet = foundSet == null ? ebm : foundSet; + RoaringBitmap32 gt = new RoaringBitmap32(); + RoaringBitmap32 lt = new RoaringBitmap32(); + RoaringBitmap32 eq = ebm; + + for (int i = slices.length - 1; i >= 0; i--) { + long bit = (predicate >> i) & 1; + if (bit == 1) { + lt = RoaringBitmap32.or(lt, RoaringBitmap32.andNot(eq, slices[i])); + eq = RoaringBitmap32.and(eq, slices[i]); + } else { + gt = RoaringBitmap32.or(gt, RoaringBitmap32.and(eq, slices[i])); + eq = RoaringBitmap32.andNot(eq, slices[i]); + } + } + + eq = RoaringBitmap32.and(fixedFoundSet, eq); + switch (operation) { + case EQ: + return eq; + case NEQ: + return RoaringBitmap32.andNot(fixedFoundSet, eq); + case GT: + return RoaringBitmap32.and(gt, fixedFoundSet); + case LT: + return RoaringBitmap32.and(lt, fixedFoundSet); + case LTE: + return RoaringBitmap32.and(RoaringBitmap32.or(lt, eq), fixedFoundSet); + case GTE: + return RoaringBitmap32.and(RoaringBitmap32.or(gt, eq), fixedFoundSet); + default: + throw new IllegalArgumentException("not support operation: " + operation); + } + } + + /** Specifies O'Neil compare algorithm operation. */ + @VisibleForTesting + protected enum Operation { + EQ, + NEQ, + LTE, + LT, + GTE, + GT + } + + public static BitSliceIndexRoaringBitmap map(DataInput in) throws IOException { + int version = in.readByte(); + if (version > VERSION_1) { + throw new RuntimeException( + String.format( + "deserialize bsi index fail, " + "your plugin version is lower than %d", + version)); + } + + // deserialize min & max + long min = in.readLong(); + long max = in.readLong(); + + // deserialize ebm + RoaringBitmap32 ebm = new RoaringBitmap32(); + ebm.deserialize(in); + + // deserialize slices + RoaringBitmap32[] slices = new RoaringBitmap32[in.readInt()]; + for (int i = 0; i < slices.length; i++) { + RoaringBitmap32 rb = new RoaringBitmap32(); + rb.deserialize(in); + slices[i] = rb; + } + + return new BitSliceIndexRoaringBitmap(min, max, ebm, slices); + } + + /** A Builder for {@link BitSliceIndexRoaringBitmap}. */ + public static class Appender { + private final long min; + private final long max; + private final RoaringBitmap32 ebm; + private final RoaringBitmap32[] slices; + + public Appender(long min, long max) { + if (min < 0) { + throw new IllegalArgumentException("values should be non-negative"); + } + if (min > max) { + throw new IllegalArgumentException("min should be less than max"); + } + + this.min = min; + this.max = max; + this.ebm = new RoaringBitmap32(); + this.slices = new RoaringBitmap32[64 - Long.numberOfLeadingZeros(max - min)]; + for (int i = 0; i < slices.length; i++) { + slices[i] = new RoaringBitmap32(); + } + } + + public void append(int rid, long value) { + if (value > max) { + throw new IllegalArgumentException(String.format("value %s is too large", value)); + } + + if (ebm.contains(rid)) { + throw new IllegalArgumentException(String.format("rid=%s is already exists", rid)); + } + + // reduce the number of slices + value = value - min; + + // only bit=1 need to set + while (value != 0) { + slices[Long.numberOfTrailingZeros(value)].add(rid); + value &= (value - 1); + } + ebm.add(rid); + } + + public boolean isNotEmpty() { + return !ebm.isEmpty(); + } + + public void serialize(DataOutput out) throws IOException { + out.writeByte(VERSION_1); + out.writeLong(min); + out.writeLong(max); + ebm.serialize(out); + out.writeInt(slices.length); + for (RoaringBitmap32 slice : slices) { + slice.serialize(out); + } + } + + public BitSliceIndexRoaringBitmap build() throws IOException { + return new BitSliceIndexRoaringBitmap(min, max, ebm, slices); + } + } +} diff --git a/paimon-common/src/main/java/org/apache/paimon/utils/FileBasedBloomFilter.java b/paimon-common/src/main/java/org/apache/paimon/utils/FileBasedBloomFilter.java index 7b885a5e6f7a..ede7a8e3cfe6 100644 --- a/paimon-common/src/main/java/org/apache/paimon/utils/FileBasedBloomFilter.java +++ b/paimon-common/src/main/java/org/apache/paimon/utils/FileBasedBloomFilter.java @@ -25,18 +25,21 @@ import org.apache.paimon.io.cache.CacheManager; import org.apache.paimon.memory.MemorySegment; +import java.io.Closeable; +import java.io.IOException; + import static org.apache.paimon.io.cache.CacheManager.REFRESH_COUNT; import static org.apache.paimon.utils.Preconditions.checkArgument; /** Util to apply a built bloom filter . */ -public class FileBasedBloomFilter { +public class FileBasedBloomFilter implements Closeable { private final PageFileInput input; private final CacheManager cacheManager; private final BloomFilter filter; private final long readOffset; private final int readLength; - + private final CacheKey cacheKey; private int accessCount; public FileBasedBloomFilter( @@ -52,6 +55,7 @@ public FileBasedBloomFilter( this.readOffset = readOffset; this.readLength = readLength; this.accessCount = 0; + this.cacheKey = CacheKey.forPosition(input.file(), readOffset, readLength, true); } public boolean testHash(int hash) { @@ -61,7 +65,7 @@ public boolean testHash(int hash) { if (accessCount == REFRESH_COUNT || filter.getMemorySegment() == null) { MemorySegment segment = cacheManager.getPage( - CacheKey.forPosition(input.file(), readOffset, readLength), + cacheKey, key -> input.readPosition(readOffset, readLength), new BloomFilterCallBack(filter)); filter.setMemorySegment(segment, 0); @@ -75,6 +79,11 @@ BloomFilter bloomFilter() { return filter; } + @Override + public void close() throws IOException { + cacheManager.invalidPage(cacheKey); + } + /** Call back for cache manager. */ private static class BloomFilterCallBack implements CacheCallback { diff --git a/paimon-common/src/main/java/org/apache/paimon/utils/Filter.java b/paimon-common/src/main/java/org/apache/paimon/utils/Filter.java index 2764bc77363a..4d9416252e0f 100644 --- a/paimon-common/src/main/java/org/apache/paimon/utils/Filter.java +++ b/paimon-common/src/main/java/org/apache/paimon/utils/Filter.java @@ -37,6 +37,13 @@ public interface Filter { */ boolean test(T t); + default Filter and(Filter other) { + if (other == null) { + return this; + } + return t -> test(t) && other.test(t); + } + @SuppressWarnings({"unchecked", "rawtypes"}) static Filter alwaysTrue() { return (Filter) ALWAYS_TRUE; diff --git a/paimon-common/src/main/java/org/apache/paimon/utils/IOUtils.java b/paimon-common/src/main/java/org/apache/paimon/utils/IOUtils.java index 3df81d2d11d3..29352b4e8161 100644 --- a/paimon-common/src/main/java/org/apache/paimon/utils/IOUtils.java +++ b/paimon-common/src/main/java/org/apache/paimon/utils/IOUtils.java @@ -38,7 +38,7 @@ public final class IOUtils { private static final Logger LOG = LoggerFactory.getLogger(IOUtils.class); /** The block size for byte operations in byte. */ - private static final int BLOCKSIZE = 4096; + public static final int BLOCKSIZE = 4096; // ------------------------------------------------------------------------ // Byte copy operations diff --git a/paimon-common/src/main/java/org/apache/paimon/utils/InternalRowPartitionComputer.java b/paimon-common/src/main/java/org/apache/paimon/utils/InternalRowPartitionComputer.java index 881f0f4d1053..6bb26d76138e 100644 --- a/paimon-common/src/main/java/org/apache/paimon/utils/InternalRowPartitionComputer.java +++ b/paimon-common/src/main/java/org/apache/paimon/utils/InternalRowPartitionComputer.java @@ -18,15 +18,22 @@ package org.apache.paimon.utils; +import org.apache.paimon.casting.CastExecutor; +import org.apache.paimon.casting.CastExecutors; import org.apache.paimon.data.BinaryRow; +import org.apache.paimon.data.GenericRow; import org.apache.paimon.data.InternalRow; +import org.apache.paimon.data.InternalRow.FieldGetter; +import org.apache.paimon.types.DataType; import org.apache.paimon.types.RowType; +import org.apache.paimon.types.VarCharType; -import java.util.Arrays; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import static org.apache.paimon.utils.InternalRowUtils.createNullCheckingFieldGetter; +import static org.apache.paimon.utils.Preconditions.checkArgument; import static org.apache.paimon.utils.TypeUtils.castFromString; /** PartitionComputer for {@link InternalRow}. */ @@ -34,21 +41,29 @@ public class InternalRowPartitionComputer { protected final String defaultPartValue; protected final String[] partitionColumns; - protected final InternalRow.FieldGetter[] partitionFieldGetters; + protected final FieldGetter[] partitionFieldGetters; + protected final CastExecutor[] partitionCastExecutors; + protected final List types; + protected final boolean legacyPartitionName; public InternalRowPartitionComputer( - String defaultPartValue, RowType rowType, String[] partitionColumns) { + String defaultPartValue, + RowType rowType, + String[] partitionColumns, + boolean legacyPartitionName) { this.defaultPartValue = defaultPartValue; this.partitionColumns = partitionColumns; + this.types = rowType.getFieldTypes(); + this.legacyPartitionName = legacyPartitionName; List columnList = rowType.getFieldNames(); - this.partitionFieldGetters = - Arrays.stream(partitionColumns) - .mapToInt(columnList::indexOf) - .mapToObj( - i -> - InternalRowUtils.createNullCheckingFieldGetter( - rowType.getTypeAt(i), i)) - .toArray(InternalRow.FieldGetter[]::new); + this.partitionFieldGetters = new FieldGetter[partitionColumns.length]; + this.partitionCastExecutors = new CastExecutor[partitionColumns.length]; + for (String partitionColumn : partitionColumns) { + int i = columnList.indexOf(partitionColumn); + DataType type = rowType.getTypeAt(i); + partitionFieldGetters[i] = createNullCheckingFieldGetter(type, i); + partitionCastExecutors[i] = CastExecutors.resolve(type, VarCharType.STRING_TYPE); + } } public LinkedHashMap generatePartValues(InternalRow in) { @@ -56,7 +71,17 @@ public LinkedHashMap generatePartValues(InternalRow in) { for (int i = 0; i < partitionFieldGetters.length; i++) { Object field = partitionFieldGetters[i].getFieldOrNull(in); - String partitionValue = field != null ? field.toString() : null; + String partitionValue = null; + if (field != null) { + if (legacyPartitionName) { + partitionValue = field.toString(); + } else { + Object casted = partitionCastExecutors[i].cast(field); + if (casted != null) { + partitionValue = casted.toString(); + } + } + } if (StringUtils.isNullOrWhitespaceOnly(partitionValue)) { partitionValue = defaultPartValue; } @@ -79,9 +104,25 @@ public static Map convertSpecToInternal( return partValues; } + public static GenericRow convertSpecToInternalRow( + Map spec, RowType partType, String defaultPartValue) { + checkArgument(spec.size() == partType.getFieldCount()); + GenericRow partRow = new GenericRow(spec.size()); + List fieldNames = partType.getFieldNames(); + for (Map.Entry entry : spec.entrySet()) { + Object value = + defaultPartValue.equals(entry.getValue()) + ? null + : castFromString( + entry.getValue(), partType.getField(entry.getKey()).type()); + partRow.setField(fieldNames.indexOf(entry.getKey()), value); + } + return partRow; + } + public static String partToSimpleString( RowType partitionType, BinaryRow partition, String delimiter, int maxLength) { - InternalRow.FieldGetter[] getters = partitionType.fieldGetters(); + FieldGetter[] getters = partitionType.fieldGetters(); StringBuilder builder = new StringBuilder(); for (int i = 0; i < getters.length; i++) { Object part = getters[i].getFieldOrNull(partition); diff --git a/paimon-common/src/main/java/org/apache/paimon/utils/InternalRowUtils.java b/paimon-common/src/main/java/org/apache/paimon/utils/InternalRowUtils.java index 0d0383747ffe..eaa077fa81ff 100644 --- a/paimon-common/src/main/java/org/apache/paimon/utils/InternalRowUtils.java +++ b/paimon-common/src/main/java/org/apache/paimon/utils/InternalRowUtils.java @@ -43,6 +43,8 @@ import org.apache.paimon.types.RowType; import org.apache.paimon.types.TimestampType; +import javax.annotation.Nullable; + import java.math.BigDecimal; import java.math.RoundingMode; import java.util.ArrayList; @@ -198,7 +200,11 @@ public static Object get(DataGetters dataGetters, int pos, DataType fieldType) { } } - public static InternalArray toStringArrayData(List list) { + public static InternalArray toStringArrayData(@Nullable List list) { + if (list == null) { + return null; + } + return new GenericArray(list.stream().map(BinaryString::fromString).toArray()); } diff --git a/paimon-common/src/main/java/org/apache/paimon/utils/ProjectedArray.java b/paimon-common/src/main/java/org/apache/paimon/utils/ProjectedArray.java new file mode 100644 index 000000000000..2182ea1a32bc --- /dev/null +++ b/paimon-common/src/main/java/org/apache/paimon/utils/ProjectedArray.java @@ -0,0 +1,206 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.utils; + +import org.apache.paimon.data.BinaryString; +import org.apache.paimon.data.Decimal; +import org.apache.paimon.data.InternalArray; +import org.apache.paimon.data.InternalMap; +import org.apache.paimon.data.InternalRow; +import org.apache.paimon.data.Timestamp; +import org.apache.paimon.types.DataType; + +/** + * An implementation of {@link InternalArray} which provides a projected view of the underlying + * {@link InternalArray}. + * + *

    Projection includes both reducing the accessible fields and reordering them. + * + *

    Note: This class supports only top-level projections, not nested projections. + */ +public class ProjectedArray implements InternalArray { + + private final int[] indexMapping; + + private InternalArray array; + + private ProjectedArray(int[] indexMapping) { + this.indexMapping = indexMapping; + } + + /** + * Replaces the underlying {@link InternalArray} backing this {@link ProjectedArray}. + * + *

    This method replaces the row data in place and does not return a new object. This is done + * for performance reasons. + */ + public ProjectedArray replaceArray(InternalArray array) { + this.array = array; + return this; + } + + // --------------------------------------------------------------------------------------------- + + @Override + public int size() { + return indexMapping.length; + } + + @Override + public boolean isNullAt(int pos) { + if (indexMapping[pos] < 0) { + return true; + } + return array.isNullAt(indexMapping[pos]); + } + + @Override + public boolean getBoolean(int pos) { + return array.getBoolean(indexMapping[pos]); + } + + @Override + public byte getByte(int pos) { + return array.getByte(indexMapping[pos]); + } + + @Override + public short getShort(int pos) { + return array.getShort(indexMapping[pos]); + } + + @Override + public int getInt(int pos) { + return array.getInt(indexMapping[pos]); + } + + @Override + public long getLong(int pos) { + return array.getLong(indexMapping[pos]); + } + + @Override + public float getFloat(int pos) { + return array.getFloat(indexMapping[pos]); + } + + @Override + public double getDouble(int pos) { + return array.getDouble(indexMapping[pos]); + } + + @Override + public BinaryString getString(int pos) { + return array.getString(indexMapping[pos]); + } + + @Override + public Decimal getDecimal(int pos, int precision, int scale) { + return array.getDecimal(indexMapping[pos], precision, scale); + } + + @Override + public Timestamp getTimestamp(int pos, int precision) { + return array.getTimestamp(indexMapping[pos], precision); + } + + @Override + public byte[] getBinary(int pos) { + return array.getBinary(indexMapping[pos]); + } + + @Override + public InternalArray getArray(int pos) { + return array.getArray(indexMapping[pos]); + } + + @Override + public InternalMap getMap(int pos) { + return array.getMap(indexMapping[pos]); + } + + @Override + public InternalRow getRow(int pos, int numFields) { + return array.getRow(indexMapping[pos], numFields); + } + + @Override + public boolean equals(Object o) { + throw new UnsupportedOperationException("Projected row data cannot be compared"); + } + + @Override + public int hashCode() { + throw new UnsupportedOperationException("Projected row data cannot be hashed"); + } + + @Override + public String toString() { + throw new UnsupportedOperationException("Projected row data cannot be toString"); + } + + /** + * Create an empty {@link ProjectedArray} starting from a {@code projection} array. + * + *

    The array represents the mapping of the fields of the original {@link DataType}. For + * example, {@code [0, 2, 1]} specifies to include in the following order the 1st field, the 3rd + * field and the 2nd field of the row. + * + * @see Projection + * @see ProjectedArray + */ + public static ProjectedArray from(int[] projection) { + return new ProjectedArray(projection); + } + + @Override + public boolean[] toBooleanArray() { + throw new UnsupportedOperationException(); + } + + @Override + public byte[] toByteArray() { + throw new UnsupportedOperationException(); + } + + @Override + public short[] toShortArray() { + throw new UnsupportedOperationException(); + } + + @Override + public int[] toIntArray() { + throw new UnsupportedOperationException(); + } + + @Override + public long[] toLongArray() { + throw new UnsupportedOperationException(); + } + + @Override + public float[] toFloatArray() { + throw new UnsupportedOperationException(); + } + + @Override + public double[] toDoubleArray() { + throw new UnsupportedOperationException(); + } +} diff --git a/paimon-common/src/main/java/org/apache/paimon/utils/RoaringBitmap32.java b/paimon-common/src/main/java/org/apache/paimon/utils/RoaringBitmap32.java index 7b86bae8a280..5f352f61cd3c 100644 --- a/paimon-common/src/main/java/org/apache/paimon/utils/RoaringBitmap32.java +++ b/paimon-common/src/main/java/org/apache/paimon/utils/RoaringBitmap32.java @@ -68,6 +68,18 @@ public long getCardinality() { return roaringBitmap.getLongCardinality(); } + public long rangeCardinality(long start, long end) { + return roaringBitmap.rangeCardinality(start, end); + } + + public int last() { + return roaringBitmap.last(); + } + + public RoaringBitmap32 clone() { + return new RoaringBitmap32(roaringBitmap.clone()); + } + public void serialize(DataOutput out) throws IOException { roaringBitmap.runOptimize(); roaringBitmap.serialize(out); @@ -149,4 +161,8 @@ public RoaringBitmap next() { } })); } + + public static RoaringBitmap32 andNot(final RoaringBitmap32 x1, final RoaringBitmap32 x2) { + return new RoaringBitmap32(RoaringBitmap.andNot(x1.roaringBitmap, x2.roaringBitmap)); + } } diff --git a/paimon-common/src/main/java/org/apache/paimon/utils/ThreadPoolUtils.java b/paimon-common/src/main/java/org/apache/paimon/utils/ThreadPoolUtils.java index 02b5d73fcf2c..e4b3da8ca8c3 100644 --- a/paimon-common/src/main/java/org/apache/paimon/utils/ThreadPoolUtils.java +++ b/paimon-common/src/main/java/org/apache/paimon/utils/ThreadPoolUtils.java @@ -30,11 +30,14 @@ import java.util.List; import java.util.NoSuchElementException; import java.util.Queue; +import java.util.concurrent.BlockingQueue; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.SynchronousQueue; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; @@ -54,18 +57,32 @@ public class ThreadPoolUtils { * is max thread number. */ public static ThreadPoolExecutor createCachedThreadPool(int threadNum, String namePrefix) { + return createCachedThreadPool(threadNum, namePrefix, new LinkedBlockingQueue<>()); + } + + /** + * Create a thread pool with max thread number and define queue. Inactive threads will + * automatically exit. + */ + public static ThreadPoolExecutor createCachedThreadPool( + int threadNum, String namePrefix, BlockingQueue workQueue) { ThreadPoolExecutor executor = new ThreadPoolExecutor( threadNum, threadNum, 1, TimeUnit.MINUTES, - new LinkedBlockingQueue<>(), + workQueue, newDaemonThreadFactory(namePrefix)); executor.allowCoreThreadTimeOut(true); return executor; } + public static ScheduledExecutorService createScheduledThreadPool( + int threadNum, String namePrefix) { + return new ScheduledThreadPoolExecutor(threadNum, newDaemonThreadFactory(namePrefix)); + } + /** This method aims to parallel process tasks with memory control and sequentially. */ public static Iterable sequentialBatchedExecute( ThreadPoolExecutor executor, @@ -110,7 +127,9 @@ private void advanceIfNeeded() { if (stack.isEmpty()) { return; } - activeList = randomlyExecute(executor, processor, stack.poll()); + activeList = + randomlyExecuteSequentialReturn( + executor, processor, stack.poll()); } } } @@ -132,7 +151,7 @@ public static void randomlyOnlyExecute( awaitAllFutures(futures); } - public static Iterator randomlyExecute( + public static Iterator randomlyExecuteSequentialReturn( ExecutorService executor, Function> processor, Collection input) { List>> futures = new ArrayList<>(input.size()); ClassLoader cl = Thread.currentThread().getContextClassLoader(); diff --git a/paimon-common/src/main/java/org/apache/paimon/utils/TimeUtils.java b/paimon-common/src/main/java/org/apache/paimon/utils/TimeUtils.java index 3b8e2ec81275..6b0f59000cca 100644 --- a/paimon-common/src/main/java/org/apache/paimon/utils/TimeUtils.java +++ b/paimon-common/src/main/java/org/apache/paimon/utils/TimeUtils.java @@ -104,6 +104,19 @@ public static Duration parseDuration(String text) { } } + /** + * Parse the given number and unitLabel to a java {@link Duration}. The usage is in format + * "(digital number, time unit label)", e.g. "(1, DAYS)". + * + * @param number a digital number + * @param unitLabel time unit label + */ + public static Duration parseDuration(long number, String unitLabel) { + ChronoUnit unit = LABEL_TO_UNIT_MAP.get(unitLabel.toLowerCase(Locale.US)); + checkNotNull(unit); + return Duration.of(number, unit); + } + private static Map initMap() { Map labelToUnit = new HashMap<>(); for (TimeUnit timeUnit : TimeUnit.values()) { diff --git a/paimon-common/src/main/java/org/apache/paimon/utils/VectorMappingUtils.java b/paimon-common/src/main/java/org/apache/paimon/utils/VectorMappingUtils.java index 8b01e644de57..02b011a2f1cf 100644 --- a/paimon-common/src/main/java/org/apache/paimon/utils/VectorMappingUtils.java +++ b/paimon-common/src/main/java/org/apache/paimon/utils/VectorMappingUtils.java @@ -97,8 +97,7 @@ public static ColumnVector createFixedVector( return dataType.accept(visitor); } - public static ColumnVector[] createIndexMappedVectors( - int[] indexMapping, ColumnVector[] vectors) { + public static ColumnVector[] createMappedVectors(int[] indexMapping, ColumnVector[] vectors) { ColumnVector[] newVectors = new ColumnVector[indexMapping.length]; for (int i = 0; i < indexMapping.length; i++) { int realIndex = indexMapping[i]; diff --git a/paimon-common/src/main/resources/META-INF/services/org.apache.paimon.fileindex.FileIndexerFactory b/paimon-common/src/main/resources/META-INF/services/org.apache.paimon.fileindex.FileIndexerFactory index 8a899eb232b8..30c908c72381 100644 --- a/paimon-common/src/main/resources/META-INF/services/org.apache.paimon.fileindex.FileIndexerFactory +++ b/paimon-common/src/main/resources/META-INF/services/org.apache.paimon.fileindex.FileIndexerFactory @@ -14,4 +14,5 @@ # limitations under the License. org.apache.paimon.fileindex.bloomfilter.BloomFilterFileIndexFactory -org.apache.paimon.fileindex.bitmap.BitmapFileIndexFactory \ No newline at end of file +org.apache.paimon.fileindex.bitmap.BitmapFileIndexFactory +org.apache.paimon.fileindex.bsi.BitSliceIndexBitmapFileIndexFactory \ No newline at end of file diff --git a/paimon-core/src/test/java/org/apache/paimon/casting/CastExecutorTest.java b/paimon-common/src/test/java/org/apache/paimon/casting/CastExecutorTest.java similarity index 100% rename from paimon-core/src/test/java/org/apache/paimon/casting/CastExecutorTest.java rename to paimon-common/src/test/java/org/apache/paimon/casting/CastExecutorTest.java diff --git a/paimon-common/src/test/java/org/apache/paimon/fileindex/bitmapindex/TestBitmapFileIndex.java b/paimon-common/src/test/java/org/apache/paimon/fileindex/bitmapindex/TestBitmapFileIndex.java index 84e2af304f7d..ba94f3c077dd 100644 --- a/paimon-common/src/test/java/org/apache/paimon/fileindex/bitmapindex/TestBitmapFileIndex.java +++ b/paimon-common/src/test/java/org/apache/paimon/fileindex/bitmapindex/TestBitmapFileIndex.java @@ -22,7 +22,7 @@ import org.apache.paimon.fileindex.FileIndexReader; import org.apache.paimon.fileindex.FileIndexWriter; import org.apache.paimon.fileindex.bitmap.BitmapFileIndex; -import org.apache.paimon.fileindex.bitmap.BitmapIndexResultLazy; +import org.apache.paimon.fileindex.bitmap.BitmapIndexResult; import org.apache.paimon.fs.ByteArraySeekableStream; import org.apache.paimon.predicate.FieldRef; import org.apache.paimon.types.IntType; @@ -63,21 +63,21 @@ public void testBitmapIndex1() { ByteArraySeekableStream seekableStream = new ByteArraySeekableStream(bytes); FileIndexReader reader = bitmapFileIndex.createReader(seekableStream, 0, bytes.length); - BitmapIndexResultLazy result1 = - (BitmapIndexResultLazy) reader.visitEqual(fieldRef, BinaryString.fromString("a")); + BitmapIndexResult result1 = + (BitmapIndexResult) reader.visitEqual(fieldRef, BinaryString.fromString("a")); assert result1.get().equals(RoaringBitmap32.bitmapOf(0, 4)); - BitmapIndexResultLazy result2 = - (BitmapIndexResultLazy) reader.visitEqual(fieldRef, BinaryString.fromString("b")); + BitmapIndexResult result2 = + (BitmapIndexResult) reader.visitEqual(fieldRef, BinaryString.fromString("b")); assert result2.get().equals(RoaringBitmap32.bitmapOf(2)); - BitmapIndexResultLazy result3 = (BitmapIndexResultLazy) reader.visitIsNull(fieldRef); + BitmapIndexResult result3 = (BitmapIndexResult) reader.visitIsNull(fieldRef); assert result3.get().equals(RoaringBitmap32.bitmapOf(1, 3)); - BitmapIndexResultLazy result4 = (BitmapIndexResultLazy) result1.and(result2); + BitmapIndexResult result4 = (BitmapIndexResult) result1.and(result2); assert result4.get().equals(RoaringBitmap32.bitmapOf()); - BitmapIndexResultLazy result5 = (BitmapIndexResultLazy) result1.or(result2); + BitmapIndexResult result5 = (BitmapIndexResult) result1.or(result2); assert result5.get().equals(RoaringBitmap32.bitmapOf(0, 2, 4)); } @@ -95,21 +95,21 @@ public void testBitmapIndex2() { ByteArraySeekableStream seekableStream = new ByteArraySeekableStream(bytes); FileIndexReader reader = bitmapFileIndex.createReader(seekableStream, 0, bytes.length); - BitmapIndexResultLazy result1 = (BitmapIndexResultLazy) reader.visitEqual(fieldRef, 1); + BitmapIndexResult result1 = (BitmapIndexResult) reader.visitEqual(fieldRef, 1); assert result1.get().equals(RoaringBitmap32.bitmapOf(1)); - BitmapIndexResultLazy result2 = (BitmapIndexResultLazy) reader.visitIsNull(fieldRef); + BitmapIndexResult result2 = (BitmapIndexResult) reader.visitIsNull(fieldRef); assert result2.get().equals(RoaringBitmap32.bitmapOf(2)); - BitmapIndexResultLazy result3 = (BitmapIndexResultLazy) reader.visitIsNotNull(fieldRef); + BitmapIndexResult result3 = (BitmapIndexResult) reader.visitIsNotNull(fieldRef); assert result3.get().equals(RoaringBitmap32.bitmapOf(0, 1)); - BitmapIndexResultLazy result4 = - (BitmapIndexResultLazy) reader.visitNotIn(fieldRef, Arrays.asList(1, 2)); + BitmapIndexResult result4 = + (BitmapIndexResult) reader.visitNotIn(fieldRef, Arrays.asList(1, 2)); assert result4.get().equals(RoaringBitmap32.bitmapOf(0, 2)); - BitmapIndexResultLazy result5 = - (BitmapIndexResultLazy) reader.visitNotIn(fieldRef, Arrays.asList(1, 0)); + BitmapIndexResult result5 = + (BitmapIndexResult) reader.visitNotIn(fieldRef, Arrays.asList(1, 0)); assert result5.get().equals(RoaringBitmap32.bitmapOf(2)); } @@ -131,11 +131,11 @@ public void testBitmapIndex3() { ByteArraySeekableStream seekableStream = new ByteArraySeekableStream(bytes); FileIndexReader reader = bitmapFileIndex.createReader(seekableStream, 0, bytes.length); - BitmapIndexResultLazy result1 = (BitmapIndexResultLazy) reader.visitEqual(fieldRef, 1); + BitmapIndexResult result1 = (BitmapIndexResult) reader.visitEqual(fieldRef, 1); assert result1.get().equals(RoaringBitmap32.bitmapOf(0, 2, 4)); // test read singleton bitmap - BitmapIndexResultLazy result2 = (BitmapIndexResultLazy) reader.visitIsNull(fieldRef); + BitmapIndexResult result2 = (BitmapIndexResult) reader.visitIsNull(fieldRef); assert result2.get().equals(RoaringBitmap32.bitmapOf(6)); } } diff --git a/paimon-common/src/test/java/org/apache/paimon/fileindex/bsi/BitSliceIndexBitmapFileIndexTest.java b/paimon-common/src/test/java/org/apache/paimon/fileindex/bsi/BitSliceIndexBitmapFileIndexTest.java new file mode 100644 index 000000000000..cd73f2b2a27d --- /dev/null +++ b/paimon-common/src/test/java/org/apache/paimon/fileindex/bsi/BitSliceIndexBitmapFileIndexTest.java @@ -0,0 +1,253 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.fileindex.bsi; + +import org.apache.paimon.fileindex.FileIndexReader; +import org.apache.paimon.fileindex.FileIndexWriter; +import org.apache.paimon.fileindex.bitmap.BitmapIndexResult; +import org.apache.paimon.fs.ByteArraySeekableStream; +import org.apache.paimon.predicate.FieldRef; +import org.apache.paimon.types.IntType; +import org.apache.paimon.utils.RoaringBitmap32; + +import org.junit.jupiter.api.Test; + +import java.util.Arrays; + +import static org.assertj.core.api.Assertions.assertThat; + +/** test for {@link BitSliceIndexBitmapFileIndex}. */ +public class BitSliceIndexBitmapFileIndexTest { + + @Test + public void testBitSliceIndexMix() { + IntType intType = new IntType(); + FieldRef fieldRef = new FieldRef(0, "", intType); + BitSliceIndexBitmapFileIndex bsiFileIndex = new BitSliceIndexBitmapFileIndex(intType, null); + FileIndexWriter writer = bsiFileIndex.createWriter(); + + Object[] arr = {1, 2, null, -2, -2, -1, null, 2, 0, 5, null}; + + for (Object o : arr) { + writer.write(o); + } + byte[] bytes = writer.serializedBytes(); + ByteArraySeekableStream stream = new ByteArraySeekableStream(bytes); + FileIndexReader reader = bsiFileIndex.createReader(stream, 0, bytes.length); + + // test eq + assertThat(((BitmapIndexResult) reader.visitEqual(fieldRef, 2)).get()) + .isEqualTo(RoaringBitmap32.bitmapOf(1, 7)); + assertThat(((BitmapIndexResult) reader.visitEqual(fieldRef, -2)).get()) + .isEqualTo(RoaringBitmap32.bitmapOf(3, 4)); + assertThat(((BitmapIndexResult) reader.visitEqual(fieldRef, 100)).get()) + .isEqualTo(RoaringBitmap32.bitmapOf()); + + // test neq + assertThat(((BitmapIndexResult) reader.visitNotEqual(fieldRef, 2)).get()) + .isEqualTo(RoaringBitmap32.bitmapOf(0, 3, 4, 5, 8, 9)); + assertThat(((BitmapIndexResult) reader.visitNotEqual(fieldRef, -2)).get()) + .isEqualTo(RoaringBitmap32.bitmapOf(0, 1, 5, 7, 8, 9)); + assertThat(((BitmapIndexResult) reader.visitNotEqual(fieldRef, 100)).get()) + .isEqualTo(RoaringBitmap32.bitmapOf(0, 1, 3, 4, 5, 7, 8, 9)); + + // test in + assertThat(((BitmapIndexResult) reader.visitIn(fieldRef, Arrays.asList(-1, 1, 2, 3))).get()) + .isEqualTo(RoaringBitmap32.bitmapOf(0, 1, 5, 7)); + + // test not in + assertThat( + ((BitmapIndexResult) + reader.visitNotIn(fieldRef, Arrays.asList(-1, 1, 2, 3))) + .get()) + .isEqualTo(RoaringBitmap32.bitmapOf(3, 4, 8, 9)); + + // test null + assertThat(((BitmapIndexResult) reader.visitIsNull(fieldRef)).get()) + .isEqualTo(RoaringBitmap32.bitmapOf(2, 6, 10)); + + // test is not null + assertThat(((BitmapIndexResult) reader.visitIsNotNull(fieldRef)).get()) + .isEqualTo(RoaringBitmap32.bitmapOf(0, 1, 3, 4, 5, 7, 8, 9)); + + // test lt + assertThat(((BitmapIndexResult) reader.visitLessThan(fieldRef, 2)).get()) + .isEqualTo(RoaringBitmap32.bitmapOf(0, 3, 4, 5, 8)); + assertThat(((BitmapIndexResult) reader.visitLessOrEqual(fieldRef, 2)).get()) + .isEqualTo(RoaringBitmap32.bitmapOf(0, 1, 3, 4, 5, 7, 8)); + assertThat(((BitmapIndexResult) reader.visitLessThan(fieldRef, -1)).get()) + .isEqualTo(RoaringBitmap32.bitmapOf(3, 4)); + assertThat(((BitmapIndexResult) reader.visitLessOrEqual(fieldRef, -1)).get()) + .isEqualTo(RoaringBitmap32.bitmapOf(3, 4, 5)); + + // test gt + assertThat(((BitmapIndexResult) reader.visitGreaterThan(fieldRef, -2)).get()) + .isEqualTo(RoaringBitmap32.bitmapOf(0, 1, 5, 7, 8, 9)); + assertThat(((BitmapIndexResult) reader.visitGreaterOrEqual(fieldRef, -2)).get()) + .isEqualTo(RoaringBitmap32.bitmapOf(0, 1, 3, 4, 5, 7, 8, 9)); + assertThat(((BitmapIndexResult) reader.visitGreaterThan(fieldRef, 2)).get()) + .isEqualTo(RoaringBitmap32.bitmapOf(9)); + assertThat(((BitmapIndexResult) reader.visitGreaterOrEqual(fieldRef, 2)).get()) + .isEqualTo(RoaringBitmap32.bitmapOf(1, 7, 9)); + } + + @Test + public void testBitSliceIndexPositiveOnly() { + IntType intType = new IntType(); + FieldRef fieldRef = new FieldRef(0, "", intType); + BitSliceIndexBitmapFileIndex bsiFileIndex = new BitSliceIndexBitmapFileIndex(intType, null); + FileIndexWriter writer = bsiFileIndex.createWriter(); + + Object[] arr = {0, 1, null, 3, 4, 5, 6, 0, null}; + + for (Object o : arr) { + writer.write(o); + } + byte[] bytes = writer.serializedBytes(); + ByteArraySeekableStream stream = new ByteArraySeekableStream(bytes); + FileIndexReader reader = bsiFileIndex.createReader(stream, 0, bytes.length); + + // test eq + assertThat(((BitmapIndexResult) reader.visitEqual(fieldRef, 0)).get()) + .isEqualTo(RoaringBitmap32.bitmapOf(0, 7)); + assertThat(((BitmapIndexResult) reader.visitEqual(fieldRef, 1)).get()) + .isEqualTo(RoaringBitmap32.bitmapOf(1)); + assertThat(((BitmapIndexResult) reader.visitEqual(fieldRef, -1)).get()) + .isEqualTo(RoaringBitmap32.bitmapOf()); + + // test neq + assertThat(((BitmapIndexResult) reader.visitNotEqual(fieldRef, 2)).get()) + .isEqualTo(RoaringBitmap32.bitmapOf(0, 1, 3, 4, 5, 6, 7)); + assertThat(((BitmapIndexResult) reader.visitNotEqual(fieldRef, -2)).get()) + .isEqualTo(RoaringBitmap32.bitmapOf(0, 1, 3, 4, 5, 6, 7)); + assertThat(((BitmapIndexResult) reader.visitNotEqual(fieldRef, 3)).get()) + .isEqualTo(RoaringBitmap32.bitmapOf(0, 1, 4, 5, 6, 7)); + + // test in + assertThat(((BitmapIndexResult) reader.visitIn(fieldRef, Arrays.asList(-1, 1, 2, 3))).get()) + .isEqualTo(RoaringBitmap32.bitmapOf(1, 3)); + + // test not in + assertThat( + ((BitmapIndexResult) + reader.visitNotIn(fieldRef, Arrays.asList(-1, 1, 2, 3))) + .get()) + .isEqualTo(RoaringBitmap32.bitmapOf(0, 4, 5, 6, 7)); + + // test null + assertThat(((BitmapIndexResult) reader.visitIsNull(fieldRef)).get()) + .isEqualTo(RoaringBitmap32.bitmapOf(2, 8)); + + // test is not null + assertThat(((BitmapIndexResult) reader.visitIsNotNull(fieldRef)).get()) + .isEqualTo(RoaringBitmap32.bitmapOf(0, 1, 3, 4, 5, 6, 7)); + + // test lt + assertThat(((BitmapIndexResult) reader.visitLessThan(fieldRef, 3)).get()) + .isEqualTo(RoaringBitmap32.bitmapOf(0, 1, 7)); + assertThat(((BitmapIndexResult) reader.visitLessOrEqual(fieldRef, 3)).get()) + .isEqualTo(RoaringBitmap32.bitmapOf(0, 1, 3, 7)); + assertThat(((BitmapIndexResult) reader.visitLessThan(fieldRef, -1)).get()) + .isEqualTo(RoaringBitmap32.bitmapOf()); + assertThat(((BitmapIndexResult) reader.visitLessOrEqual(fieldRef, -1)).get()) + .isEqualTo(RoaringBitmap32.bitmapOf()); + + // test gt + assertThat(((BitmapIndexResult) reader.visitGreaterThan(fieldRef, -2)).get()) + .isEqualTo(RoaringBitmap32.bitmapOf(0, 1, 3, 4, 5, 6, 7)); + assertThat(((BitmapIndexResult) reader.visitGreaterOrEqual(fieldRef, -2)).get()) + .isEqualTo(RoaringBitmap32.bitmapOf(0, 1, 3, 4, 5, 6, 7)); + assertThat(((BitmapIndexResult) reader.visitGreaterThan(fieldRef, 1)).get()) + .isEqualTo(RoaringBitmap32.bitmapOf(3, 4, 5, 6)); + assertThat(((BitmapIndexResult) reader.visitGreaterOrEqual(fieldRef, 1)).get()) + .isEqualTo(RoaringBitmap32.bitmapOf(1, 3, 4, 5, 6)); + } + + @Test + public void testBitSliceIndexNegativeOnly() { + IntType intType = new IntType(); + FieldRef fieldRef = new FieldRef(0, "", intType); + BitSliceIndexBitmapFileIndex bsiFileIndex = new BitSliceIndexBitmapFileIndex(intType, null); + FileIndexWriter writer = bsiFileIndex.createWriter(); + + Object[] arr = {null, -1, null, -3, -4, -5, -6, -1, null}; + + for (Object o : arr) { + writer.write(o); + } + byte[] bytes = writer.serializedBytes(); + ByteArraySeekableStream stream = new ByteArraySeekableStream(bytes); + FileIndexReader reader = bsiFileIndex.createReader(stream, 0, bytes.length); + + // test eq + assertThat(((BitmapIndexResult) reader.visitEqual(fieldRef, 1)).get()) + .isEqualTo(RoaringBitmap32.bitmapOf()); + assertThat(((BitmapIndexResult) reader.visitEqual(fieldRef, -2)).get()) + .isEqualTo(RoaringBitmap32.bitmapOf()); + assertThat(((BitmapIndexResult) reader.visitEqual(fieldRef, -1)).get()) + .isEqualTo(RoaringBitmap32.bitmapOf(1, 7)); + + // test neq + assertThat(((BitmapIndexResult) reader.visitNotEqual(fieldRef, -2)).get()) + .isEqualTo(RoaringBitmap32.bitmapOf(1, 3, 4, 5, 6, 7)); + assertThat(((BitmapIndexResult) reader.visitNotEqual(fieldRef, -3)).get()) + .isEqualTo(RoaringBitmap32.bitmapOf(1, 4, 5, 6, 7)); + + // test in + assertThat( + ((BitmapIndexResult) reader.visitIn(fieldRef, Arrays.asList(-1, -4, -2, 3))) + .get()) + .isEqualTo(RoaringBitmap32.bitmapOf(1, 4, 7)); + + // test not in + assertThat( + ((BitmapIndexResult) + reader.visitNotIn(fieldRef, Arrays.asList(-1, -4, -2, 3))) + .get()) + .isEqualTo(RoaringBitmap32.bitmapOf(3, 5, 6)); + + // test null + assertThat(((BitmapIndexResult) reader.visitIsNull(fieldRef)).get()) + .isEqualTo(RoaringBitmap32.bitmapOf(0, 2, 8)); + + // test is not null + assertThat(((BitmapIndexResult) reader.visitIsNotNull(fieldRef)).get()) + .isEqualTo(RoaringBitmap32.bitmapOf(1, 3, 4, 5, 6, 7)); + + // test lt + assertThat(((BitmapIndexResult) reader.visitLessThan(fieldRef, -3)).get()) + .isEqualTo(RoaringBitmap32.bitmapOf(4, 5, 6)); + assertThat(((BitmapIndexResult) reader.visitLessOrEqual(fieldRef, -3)).get()) + .isEqualTo(RoaringBitmap32.bitmapOf(3, 4, 5, 6)); + assertThat(((BitmapIndexResult) reader.visitLessThan(fieldRef, 1)).get()) + .isEqualTo(RoaringBitmap32.bitmapOf(1, 3, 4, 5, 6, 7)); + assertThat(((BitmapIndexResult) reader.visitLessOrEqual(fieldRef, 1)).get()) + .isEqualTo(RoaringBitmap32.bitmapOf(1, 3, 4, 5, 6, 7)); + + // test gt + assertThat(((BitmapIndexResult) reader.visitGreaterThan(fieldRef, -3)).get()) + .isEqualTo(RoaringBitmap32.bitmapOf(1, 7)); + assertThat(((BitmapIndexResult) reader.visitGreaterOrEqual(fieldRef, -3)).get()) + .isEqualTo(RoaringBitmap32.bitmapOf(1, 3, 7)); + assertThat(((BitmapIndexResult) reader.visitGreaterThan(fieldRef, 1)).get()) + .isEqualTo(RoaringBitmap32.bitmapOf()); + assertThat(((BitmapIndexResult) reader.visitGreaterOrEqual(fieldRef, 1)).get()) + .isEqualTo(RoaringBitmap32.bitmapOf()); + } +} diff --git a/paimon-common/src/test/java/org/apache/paimon/format/FormatReadWriteTest.java b/paimon-common/src/test/java/org/apache/paimon/format/FormatReadWriteTest.java index d393a9192523..d3114cee6d76 100644 --- a/paimon-common/src/test/java/org/apache/paimon/format/FormatReadWriteTest.java +++ b/paimon-common/src/test/java/org/apache/paimon/format/FormatReadWriteTest.java @@ -60,8 +60,8 @@ public abstract class FormatReadWriteTest { private final String formatType; - private FileIO fileIO; - private Path file; + protected FileIO fileIO; + protected Path file; protected FormatReadWriteTest(String formatType) { this.formatType = formatType; diff --git a/paimon-common/src/test/java/org/apache/paimon/io/cache/CacheManagerTest.java b/paimon-common/src/test/java/org/apache/paimon/io/cache/CacheManagerTest.java new file mode 100644 index 000000000000..cf8076ac8b80 --- /dev/null +++ b/paimon-common/src/test/java/org/apache/paimon/io/cache/CacheManagerTest.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.io.cache; + +import org.apache.paimon.memory.MemorySegment; +import org.apache.paimon.options.MemorySize; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; +import org.junit.jupiter.api.io.TempDir; + +import java.io.File; +import java.io.RandomAccessFile; +import java.nio.file.Path; +import java.util.Arrays; + +import static org.assertj.core.api.Assertions.assertThat; + +/** Tests for cache manager. */ +public class CacheManagerTest { + @TempDir Path tempDir; + + @Test + @Timeout(60) + void testCaffeineCache() throws Exception { + File file1 = new File(tempDir.toFile(), "test.caffeine1"); + assertThat(file1.createNewFile()).isTrue(); + CacheKey key1 = CacheKey.forPageIndex(new RandomAccessFile(file1, "r"), 0, 0); + + File file2 = new File(tempDir.toFile(), "test.caffeine2"); + assertThat(file2.createNewFile()).isTrue(); + CacheKey key2 = CacheKey.forPageIndex(new RandomAccessFile(file2, "r"), 0, 0); + + for (Cache.CacheType cacheType : Cache.CacheType.values()) { + CacheManager cacheManager = new CacheManager(cacheType, MemorySize.ofBytes(10), 0.1); + byte[] value = new byte[6]; + Arrays.fill(value, (byte) 1); + for (int i = 0; i < 10; i++) { + for (int j = 0; j < 10; j++) { + MemorySegment segment = + cacheManager.getPage( + j < 5 ? key1 : key2, + key -> { + byte[] result = new byte[6]; + Arrays.fill(result, (byte) 1); + return result; + }, + key -> {}); + assertThat(segment.getHeapMemory()).isEqualTo(value); + } + } + } + } +} diff --git a/paimon-common/src/test/java/org/apache/paimon/io/cache/FileBasedRandomInputViewTest.java b/paimon-common/src/test/java/org/apache/paimon/io/cache/FileBasedRandomInputViewTest.java index f9f11953cfd0..6486aead8c25 100644 --- a/paimon-common/src/test/java/org/apache/paimon/io/cache/FileBasedRandomInputViewTest.java +++ b/paimon-common/src/test/java/org/apache/paimon/io/cache/FileBasedRandomInputViewTest.java @@ -21,8 +21,11 @@ import org.apache.paimon.io.PageFileInput; import org.apache.paimon.memory.MemorySegment; import org.apache.paimon.options.MemorySize; +import org.apache.paimon.testutils.junit.parameterized.ParameterizedTestExtension; +import org.apache.paimon.testutils.junit.parameterized.Parameters; -import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestTemplate; +import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.api.io.TempDir; import java.io.File; @@ -30,6 +33,8 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardOpenOption; +import java.util.Arrays; +import java.util.List; import java.util.UUID; import java.util.concurrent.ThreadLocalRandom; @@ -37,23 +42,34 @@ import static org.assertj.core.api.Assertions.assertThatCode; /** Test for {@link FileBasedRandomInputView}. */ +@ExtendWith(ParameterizedTestExtension.class) public class FileBasedRandomInputViewTest { @TempDir Path tempDir; private final ThreadLocalRandom rnd = ThreadLocalRandom.current(); + private final Cache.CacheType cacheType; - @Test + public FileBasedRandomInputViewTest(Cache.CacheType cacheType) { + this.cacheType = cacheType; + } + + @Parameters(name = "{0}") + public static List getVarSeg() { + return Arrays.asList(Cache.CacheType.CAFFEINE, Cache.CacheType.GUAVA); + } + + @TestTemplate public void testMatched() throws IOException { innerTest(1024 * 512, 5000); } - @Test + @TestTemplate public void testNotMatched() throws IOException { innerTest(131092, 1000); } - @Test + @TestTemplate public void testRandom() throws IOException { innerTest(rnd.nextInt(5000, 100000), 100); } @@ -66,7 +82,7 @@ private void innerTest(int len, int maxFileReadCount) throws IOException { } File file = writeFile(bytes); - CacheManager cacheManager = new CacheManager(MemorySize.ofKibiBytes(128)); + CacheManager cacheManager = new CacheManager(cacheType, MemorySize.ofKibiBytes(128), 0); FileBasedRandomInputView view = new FileBasedRandomInputView( PageFileInput.create(file, 1024, null, 0, null), cacheManager); @@ -101,7 +117,8 @@ private void innerTest(int len, int maxFileReadCount) throws IOException { // hot key in LRU, should have good cache hit rate assertThat(cacheManager.fileReadCount()).isLessThan(maxFileReadCount); - assertThat(cacheManager.cache().asMap().size()).isEqualTo(0); + assertThat(cacheManager.dataCache().asMap().size()).isEqualTo(0); + assertThat(cacheManager.indexCache().asMap().size()).isEqualTo(0); } private File writeFile(byte[] bytes) throws IOException { diff --git a/paimon-common/src/test/java/org/apache/paimon/lookup/sort/SortLookupStoreFactoryTest.java b/paimon-common/src/test/java/org/apache/paimon/lookup/sort/SortLookupStoreFactoryTest.java index a2299c68fe99..7ba3f8283aea 100644 --- a/paimon-common/src/test/java/org/apache/paimon/lookup/sort/SortLookupStoreFactoryTest.java +++ b/paimon-common/src/test/java/org/apache/paimon/lookup/sort/SortLookupStoreFactoryTest.java @@ -88,12 +88,9 @@ public void before() throws Exception { @TestTemplate public void testNormal() throws IOException { + CacheManager cacheManager = new CacheManager(MemorySize.ofMebiBytes(1)); SortLookupStoreFactory factory = - new SortLookupStoreFactory( - Comparator.naturalOrder(), - new CacheManager(MemorySize.ofMebiBytes(1)), - 1024, - compress); + new SortLookupStoreFactory(Comparator.naturalOrder(), cacheManager, 1024, compress); SortLookupStoreWriter writer = factory.createWriter(file, createBloomFiler(bloomFilterEnabled)); @@ -113,6 +110,8 @@ public void testNormal() throws IOException { assertThat(reader.lookup(toBytes(VALUE_COUNT + 1000))).isNull(); reader.close(); + assertThat(cacheManager.dataCache().asMap()).isEmpty(); + assertThat(cacheManager.indexCache().asMap()).isEmpty(); } @TestTemplate diff --git a/paimon-common/src/test/java/org/apache/paimon/utils/BitSliceIndexRoaringBitmapTest.java b/paimon-common/src/test/java/org/apache/paimon/utils/BitSliceIndexRoaringBitmapTest.java new file mode 100644 index 000000000000..83252a8bf188 --- /dev/null +++ b/paimon-common/src/test/java/org/apache/paimon/utils/BitSliceIndexRoaringBitmapTest.java @@ -0,0 +1,270 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.utils; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.Random; + +import static org.apache.paimon.utils.BitSliceIndexRoaringBitmap.Operation.EQ; +import static org.apache.paimon.utils.BitSliceIndexRoaringBitmap.Operation.GT; +import static org.apache.paimon.utils.BitSliceIndexRoaringBitmap.Operation.GTE; +import static org.apache.paimon.utils.BitSliceIndexRoaringBitmap.Operation.LT; +import static org.apache.paimon.utils.BitSliceIndexRoaringBitmap.Operation.LTE; +import static org.apache.paimon.utils.BitSliceIndexRoaringBitmap.Operation.NEQ; +import static org.assertj.core.api.Assertions.assertThat; + +/** Test for {@link BitSliceIndexRoaringBitmap}. */ +public class BitSliceIndexRoaringBitmapTest { + + public static final int NUM_OF_ROWS = 100000; + public static final int VALUE_BOUND = 1000; + public static final int VALUE_LT_MIN = 0; + public static final int VALUE_GT_MAX = VALUE_BOUND + 100; + + private Random random; + private List pairs; + private BitSliceIndexRoaringBitmap bsi; + + @BeforeEach + public void setup() throws IOException { + this.random = new Random(); + List pairs = new ArrayList<>(); + long min = 0; + long max = 0; + for (int i = 0; i < NUM_OF_ROWS; i++) { + if (i % 5 == 0) { + pairs.add(new Pair(i, null)); + continue; + } + long next = generateNextValue(); + min = Math.min(min == 0 ? next : min, next); + max = Math.max(max == 0 ? next : max, next); + pairs.add(new Pair(i, next)); + } + BitSliceIndexRoaringBitmap.Appender appender = + new BitSliceIndexRoaringBitmap.Appender(min, max); + for (Pair pair : pairs) { + if (pair.value == null) { + continue; + } + appender.append(pair.index, pair.value); + } + this.bsi = appender.build(); + this.pairs = Collections.unmodifiableList(pairs); + } + + @Test + public void testSerde() throws IOException { + BitSliceIndexRoaringBitmap.Appender appender = + new BitSliceIndexRoaringBitmap.Appender(0, 10); + appender.append(0, 0); + appender.append(1, 1); + appender.append(2, 2); + appender.append(10, 6); + + ByteArrayOutputStream out = new ByteArrayOutputStream(); + appender.serialize(new DataOutputStream(out)); + + ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray()); + assertThat(BitSliceIndexRoaringBitmap.map(new DataInputStream(in))) + .isEqualTo(appender.build()); + } + + @Test + public void testEQ() { + // test predicate in the value bound + for (int i = 0; i < 10; i++) { + long predicate = generateNextValue(); + assertThat(bsi.eq(predicate)) + .isEqualTo( + pairs.stream() + .filter(x -> Objects.equals(x.value, predicate)) + .map(x -> x.index) + .collect( + RoaringBitmap32::new, + RoaringBitmap32::add, + (x, y) -> x.or(y))); + } + + // test predicate out of the value bound + assertThat(bsi.eq(VALUE_LT_MIN)).isEqualTo(new RoaringBitmap32()); + assertThat(bsi.eq(VALUE_GT_MAX)).isEqualTo(new RoaringBitmap32()); + } + + @Test + public void testLT() { + // test predicate in the value bound + for (int i = 0; i < 10; i++) { + long predicate = generateNextValue(); + assertThat(bsi.lt(predicate)) + .isEqualTo( + pairs.stream() + .filter(x -> x.value != null) + .filter(x -> x.value < predicate) + .map(x -> x.index) + .collect( + RoaringBitmap32::new, + RoaringBitmap32::add, + (x, y) -> x.or(y))); + } + + // test predicate out of the value bound + assertThat(bsi.lt(VALUE_LT_MIN)).isEqualTo(new RoaringBitmap32()); + assertThat(bsi.lt(VALUE_GT_MAX)).isEqualTo(bsi.isNotNull()); + } + + @Test + public void testLTE() { + // test predicate in the value bound + for (int i = 0; i < 10; i++) { + long predicate = generateNextValue(); + assertThat(bsi.lte(predicate)) + .isEqualTo( + pairs.stream() + .filter(x -> x.value != null) + .filter(x -> x.value <= predicate) + .map(x -> x.index) + .collect( + RoaringBitmap32::new, + RoaringBitmap32::add, + (x, y) -> x.or(y))); + } + + // test predicate out of the value bound + assertThat(bsi.lte(VALUE_LT_MIN)).isEqualTo(new RoaringBitmap32()); + assertThat(bsi.lte(VALUE_GT_MAX)).isEqualTo(bsi.isNotNull()); + } + + @Test + public void testGT() { + // test predicate in the value bound + for (int i = 0; i < 10; i++) { + long predicate = generateNextValue(); + assertThat(bsi.gt(predicate)) + .isEqualTo( + pairs.stream() + .filter(x -> x.value != null) + .filter(x -> x.value > predicate) + .map(x -> x.index) + .collect( + RoaringBitmap32::new, + RoaringBitmap32::add, + (x, y) -> x.or(y))); + } + + // test predicate out of the value bound + assertThat(bsi.gt(VALUE_LT_MIN)).isEqualTo(bsi.isNotNull()); + assertThat(bsi.gt(VALUE_GT_MAX)).isEqualTo(new RoaringBitmap32()); + } + + @Test + public void testGTE() { + // test predicate in the value bound + for (int i = 0; i < 10; i++) { + long predicate = generateNextValue(); + assertThat(bsi.gte(predicate)) + .isEqualTo( + pairs.stream() + .filter(x -> x.value != null) + .filter(x -> x.value >= predicate) + .map(x -> x.index) + .collect( + RoaringBitmap32::new, + RoaringBitmap32::add, + (x, y) -> x.or(y))); + } + + // test predicate out of the value bound + assertThat(bsi.gte(VALUE_LT_MIN)).isEqualTo(bsi.isNotNull()); + assertThat(bsi.gte(VALUE_GT_MAX)).isEqualTo(new RoaringBitmap32()); + } + + @Test + public void testIsNotNull() { + assertThat(bsi.isNotNull()) + .isEqualTo( + pairs.stream() + .filter(x -> x.value != null) + .map(x -> x.index) + .collect( + RoaringBitmap32::new, + RoaringBitmap32::add, + (x, y) -> x.or(y))); + } + + @Test + public void testCompareUsingMinMax() { + // a predicate in the value bound + final int predicate = generateNextValue(); + final Optional empty = Optional.of(new RoaringBitmap32()); + final Optional notNul = Optional.of(bsi.isNotNull()); + final Optional inBound = Optional.empty(); + + // test eq & neq + assertThat(bsi.compareUsingMinMax(EQ, predicate, null)).isEqualTo(inBound); + assertThat(bsi.compareUsingMinMax(EQ, VALUE_LT_MIN, null)).isEqualTo(empty); + assertThat(bsi.compareUsingMinMax(EQ, VALUE_GT_MAX, null)).isEqualTo(empty); + assertThat(bsi.compareUsingMinMax(NEQ, predicate, null)).isEqualTo(inBound); + assertThat(bsi.compareUsingMinMax(NEQ, VALUE_LT_MIN, null)).isEqualTo(notNul); + assertThat(bsi.compareUsingMinMax(NEQ, VALUE_GT_MAX, null)).isEqualTo(notNul); + + // test lt & lte + assertThat(bsi.compareUsingMinMax(LT, predicate, null)).isEqualTo(inBound); + assertThat(bsi.compareUsingMinMax(LTE, predicate, null)).isEqualTo(inBound); + assertThat(bsi.compareUsingMinMax(LT, VALUE_LT_MIN, null)).isEqualTo(empty); + assertThat(bsi.compareUsingMinMax(LTE, VALUE_LT_MIN, null)).isEqualTo(empty); + assertThat(bsi.compareUsingMinMax(LT, VALUE_GT_MAX, null)).isEqualTo(notNul); + assertThat(bsi.compareUsingMinMax(LTE, VALUE_GT_MAX, null)).isEqualTo(notNul); + + // test gt & gte + assertThat(bsi.compareUsingMinMax(GT, predicate, null)).isEqualTo(inBound); + assertThat(bsi.compareUsingMinMax(GTE, predicate, null)).isEqualTo(inBound); + assertThat(bsi.compareUsingMinMax(GT, VALUE_LT_MIN, null)).isEqualTo(notNul); + assertThat(bsi.compareUsingMinMax(GTE, VALUE_LT_MIN, null)).isEqualTo(notNul); + assertThat(bsi.compareUsingMinMax(GT, VALUE_GT_MAX, null)).isEqualTo(empty); + assertThat(bsi.compareUsingMinMax(GT, VALUE_GT_MAX, null)).isEqualTo(empty); + } + + private int generateNextValue() { + // return a value in the range [1, VALUE_BOUND) + return random.nextInt(VALUE_BOUND) + 1; + } + + private static class Pair { + int index; + Long value; + + public Pair(int index, Long value) { + this.index = index; + this.value = value; + } + } +} diff --git a/paimon-common/src/test/java/org/apache/paimon/utils/FileBasedBloomFilterTest.java b/paimon-common/src/test/java/org/apache/paimon/utils/FileBasedBloomFilterTest.java index 970219e7b4f7..d1471fd74afb 100644 --- a/paimon-common/src/test/java/org/apache/paimon/utils/FileBasedBloomFilterTest.java +++ b/paimon-common/src/test/java/org/apache/paimon/utils/FileBasedBloomFilterTest.java @@ -19,12 +19,16 @@ package org.apache.paimon.utils; import org.apache.paimon.io.PageFileInput; +import org.apache.paimon.io.cache.Cache; import org.apache.paimon.io.cache.CacheManager; import org.apache.paimon.memory.MemorySegment; import org.apache.paimon.options.MemorySize; +import org.apache.paimon.testutils.junit.parameterized.ParameterizedTestExtension; +import org.apache.paimon.testutils.junit.parameterized.Parameters; import org.assertj.core.api.Assertions; -import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestTemplate; +import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.api.io.TempDir; import java.io.File; @@ -33,14 +37,26 @@ import java.nio.file.Path; import java.nio.file.StandardOpenOption; import java.util.Arrays; +import java.util.List; import java.util.UUID; /** Test for {@link FileBasedBloomFilter}. */ +@ExtendWith(ParameterizedTestExtension.class) public class FileBasedBloomFilterTest { @TempDir Path tempDir; + private final Cache.CacheType cacheType; - @Test + public FileBasedBloomFilterTest(Cache.CacheType cacheType) { + this.cacheType = cacheType; + } + + @Parameters(name = "{0}") + public static List getVarSeg() { + return Arrays.asList(Cache.CacheType.CAFFEINE, Cache.CacheType.GUAVA); + } + + @TestTemplate public void testProbe() throws IOException { MemorySegment segment = MemorySegment.wrap(new byte[1000]); BloomFilter.Builder builder = new BloomFilter.Builder(segment, 100); @@ -48,7 +64,7 @@ public void testProbe() throws IOException { Arrays.stream(inputs).forEach(i -> builder.addHash(Integer.hashCode(i))); File file = writeFile(segment.getArray()); - CacheManager cacheManager = new CacheManager(MemorySize.ofMebiBytes(1)); + CacheManager cacheManager = new CacheManager(cacheType, MemorySize.ofMebiBytes(1), 0.1); FileBasedBloomFilter filter = new FileBasedBloomFilter( PageFileInput.create(file, 1024, null, 0, null), @@ -59,7 +75,9 @@ public void testProbe() throws IOException { Arrays.stream(inputs) .forEach(i -> Assertions.assertThat(filter.testHash(Integer.hashCode(i))).isTrue()); - cacheManager.cache().invalidateAll(); + filter.close(); + Assertions.assertThat(cacheManager.dataCache().asMap()).isEmpty(); + Assertions.assertThat(cacheManager.indexCache().asMap()).isEmpty(); Assertions.assertThat(filter.bloomFilter().getMemorySegment()).isNull(); } diff --git a/paimon-common/src/test/java/org/apache/paimon/utils/VectorMappingUtilsTest.java b/paimon-common/src/test/java/org/apache/paimon/utils/VectorMappingUtilsTest.java index c5fac9c880db..571a0d7189d6 100644 --- a/paimon-common/src/test/java/org/apache/paimon/utils/VectorMappingUtilsTest.java +++ b/paimon-common/src/test/java/org/apache/paimon/utils/VectorMappingUtilsTest.java @@ -82,7 +82,7 @@ public void testCreateIndexMappedVectors() { int[] mapping = new int[] {0, 2, 1, 3, 2, 3, 1, 0, 4}; ColumnVector[] newColumnVectors = - VectorMappingUtils.createIndexMappedVectors(mapping, columnVectors); + VectorMappingUtils.createMappedVectors(mapping, columnVectors); for (int i = 0; i < mapping.length; i++) { Assertions.assertThat(newColumnVectors[i]).isEqualTo(columnVectors[mapping[i]]); diff --git a/paimon-core/pom.xml b/paimon-core/pom.xml index 513e356526f9..e137d57a6db1 100644 --- a/paimon-core/pom.xml +++ b/paimon-core/pom.xml @@ -33,6 +33,7 @@ under the License. 6.20.3-ververica-2.0 + 4.12.0 @@ -63,6 +64,14 @@ under the License. provided + + + + com.squareup.okhttp3 + okhttp + ${okhttp.version} + + @@ -193,14 +202,28 @@ under the License. org.apache.iceberg iceberg-core - 1.5.2 + ${iceberg.version} test org.apache.iceberg iceberg-data - 1.5.2 + ${iceberg.version} + test + + + + com.squareup.okhttp3 + mockwebserver + ${okhttp.version} + test + + + org.mockito + mockito-core + ${mockito.version} + jar test @@ -219,6 +242,40 @@ under the License. + + org.apache.maven.plugins + maven-shade-plugin + + + shade-paimon + package + + shade + + + + + * + + okhttp3/internal/publicsuffix/NOTICE + + + + + + com.squareup.okhttp3:okhttp + + + + + okhttp3 + org.apache.paimon.shade.okhttp3 + + + + + + diff --git a/paimon-core/src/main/java/org/apache/paimon/AbstractFileStore.java b/paimon-core/src/main/java/org/apache/paimon/AbstractFileStore.java index 53bd1291f1f0..1caff252a654 100644 --- a/paimon-core/src/main/java/org/apache/paimon/AbstractFileStore.java +++ b/paimon-core/src/main/java/org/apache/paimon/AbstractFileStore.java @@ -54,6 +54,8 @@ import org.apache.paimon.utils.SnapshotManager; import org.apache.paimon.utils.TagManager; +import org.apache.paimon.shade.caffeine2.com.github.benmanes.caffeine.cache.Cache; + import javax.annotation.Nullable; import java.time.Duration; @@ -72,23 +74,27 @@ abstract class AbstractFileStore implements FileStore { protected final FileIO fileIO; protected final SchemaManager schemaManager; protected final TableSchema schema; + protected final String tableName; protected final CoreOptions options; protected final RowType partitionType; private final CatalogEnvironment catalogEnvironment; @Nullable private final SegmentsCache writeManifestCache; @Nullable private SegmentsCache readManifestCache; + @Nullable private Cache snapshotCache; protected AbstractFileStore( FileIO fileIO, SchemaManager schemaManager, TableSchema schema, + String tableName, CoreOptions options, RowType partitionType, CatalogEnvironment catalogEnvironment) { this.fileIO = fileIO; this.schemaManager = schemaManager; this.schema = schema; + this.tableName = tableName; this.options = options; this.partitionType = partitionType; this.catalogEnvironment = catalogEnvironment; @@ -99,18 +105,26 @@ protected AbstractFileStore( @Override public FileStorePathFactory pathFactory() { + return pathFactory(options.fileFormat().getFormatIdentifier()); + } + + protected FileStorePathFactory pathFactory(String format) { return new FileStorePathFactory( options.path(), partitionType, options.partitionDefaultName(), - options.fileFormat().getFormatIdentifier(), + format, options.dataFilePrefix(), - options.changelogFilePrefix()); + options.changelogFilePrefix(), + options.legacyPartitionName(), + options.fileSuffixIncludeCompression(), + options.fileCompression(), + options.dataFilePathDirectory()); } @Override public SnapshotManager snapshotManager() { - return new SnapshotManager(fileIO, options.path(), options.branch()); + return new SnapshotManager(fileIO, options.path(), options.branch(), snapshotCache); } @Override @@ -206,8 +220,10 @@ public FileStoreCommitImpl newCommit(String commitUser, List cal return new FileStoreCommitImpl( fileIO, schemaManager, + tableName, commitUser, partitionType, + options, options.partitionDefaultName(), pathFactory(), snapshotManager(), @@ -226,7 +242,8 @@ public FileStoreCommitImpl newCommit(String commitUser, List cal bucketMode(), options.scanManifestParallelism(), callbacks, - options.commitMaxRetries()); + options.commitMaxRetries(), + options.commitTimeout()); } @Override @@ -298,7 +315,8 @@ public PartitionExpire newPartitionExpire(String commitUser) { newScan(), newCommit(commitUser), metastoreClient, - options.endInputCheckPartitionExpire()); + options.endInputCheckPartitionExpire(), + options.partitionExpireMaxNum()); } @Override @@ -333,4 +351,9 @@ public ServiceManager newServiceManager() { public void setManifestCache(SegmentsCache manifestCache) { this.readManifestCache = manifestCache; } + + @Override + public void setSnapshotCache(Cache cache) { + this.snapshotCache = cache; + } } diff --git a/paimon-core/src/main/java/org/apache/paimon/AppendOnlyFileStore.java b/paimon-core/src/main/java/org/apache/paimon/AppendOnlyFileStore.java index 30208cebb8cc..a06b98d7b30c 100644 --- a/paimon-core/src/main/java/org/apache/paimon/AppendOnlyFileStore.java +++ b/paimon-core/src/main/java/org/apache/paimon/AppendOnlyFileStore.java @@ -49,7 +49,6 @@ public class AppendOnlyFileStore extends AbstractFileStore { private final RowType bucketKeyType; private final RowType rowType; - private final String tableName; public AppendOnlyFileStore( FileIO fileIO, @@ -61,10 +60,9 @@ public AppendOnlyFileStore( RowType rowType, String tableName, CatalogEnvironment catalogEnvironment) { - super(fileIO, schemaManager, schema, options, partitionType, catalogEnvironment); + super(fileIO, schemaManager, schema, tableName, options, partitionType, catalogEnvironment); this.bucketKeyType = bucketKeyType; this.rowType = rowType; - this.tableName = tableName; } @Override diff --git a/paimon-core/src/main/java/org/apache/paimon/FileStore.java b/paimon-core/src/main/java/org/apache/paimon/FileStore.java index f9bf4c8440bd..e50d4ada1397 100644 --- a/paimon-core/src/main/java/org/apache/paimon/FileStore.java +++ b/paimon-core/src/main/java/org/apache/paimon/FileStore.java @@ -44,6 +44,8 @@ import org.apache.paimon.utils.SnapshotManager; import org.apache.paimon.utils.TagManager; +import org.apache.paimon.shade.caffeine2.com.github.benmanes.caffeine.cache.Cache; + import javax.annotation.Nullable; import java.util.List; @@ -107,4 +109,6 @@ public interface FileStore { List createTagCallbacks(); void setManifestCache(SegmentsCache manifestCache); + + void setSnapshotCache(Cache cache); } diff --git a/paimon-core/src/main/java/org/apache/paimon/KeyValue.java b/paimon-core/src/main/java/org/apache/paimon/KeyValue.java index f2a6c0bdeb7d..36ac88996ce3 100644 --- a/paimon-core/src/main/java/org/apache/paimon/KeyValue.java +++ b/paimon-core/src/main/java/org/apache/paimon/KeyValue.java @@ -31,9 +31,9 @@ import java.util.stream.Collectors; import java.util.stream.IntStream; -import static org.apache.paimon.table.SystemFields.LEVEL; -import static org.apache.paimon.table.SystemFields.SEQUENCE_NUMBER; -import static org.apache.paimon.table.SystemFields.VALUE_KIND; +import static org.apache.paimon.table.SpecialFields.LEVEL; +import static org.apache.paimon.table.SpecialFields.SEQUENCE_NUMBER; +import static org.apache.paimon.table.SpecialFields.VALUE_KIND; /** * A key value, including user key, sequence number, value kind and value. This object can be diff --git a/paimon-core/src/main/java/org/apache/paimon/KeyValueFileStore.java b/paimon-core/src/main/java/org/apache/paimon/KeyValueFileStore.java index cc4579898401..8cf45105c01b 100644 --- a/paimon-core/src/main/java/org/apache/paimon/KeyValueFileStore.java +++ b/paimon-core/src/main/java/org/apache/paimon/KeyValueFileStore.java @@ -70,7 +70,6 @@ public class KeyValueFileStore extends AbstractFileStore { private final Supplier> keyComparatorSupplier; private final Supplier logDedupEqualSupplier; private final MergeFunctionFactory mfFactory; - private final String tableName; public KeyValueFileStore( FileIO fileIO, @@ -86,7 +85,7 @@ public KeyValueFileStore( MergeFunctionFactory mfFactory, String tableName, CatalogEnvironment catalogEnvironment) { - super(fileIO, schemaManager, schema, options, partitionType, catalogEnvironment); + super(fileIO, schemaManager, schema, tableName, options, partitionType, catalogEnvironment); this.crossPartitionUpdate = crossPartitionUpdate; this.bucketKeyType = bucketKeyType; this.keyType = keyType; @@ -99,7 +98,6 @@ public KeyValueFileStore( options.changelogRowDeduplicate() ? ValueEqualiserSupplier.fromIgnoreFields(valueType, logDedupIgnoreFields) : () -> null; - this.tableName = tableName; } @Override @@ -196,17 +194,7 @@ private Map format2PathFactory() { Map pathFactoryMap = new HashMap<>(); Set formats = new HashSet<>(options.fileFormatPerLevel().values()); formats.add(options.fileFormat().getFormatIdentifier()); - formats.forEach( - format -> - pathFactoryMap.put( - format, - new FileStorePathFactory( - options.path(), - partitionType, - options.partitionDefaultName(), - format, - options.dataFilePrefix(), - options.changelogFilePrefix()))); + formats.forEach(format -> pathFactoryMap.put(format, pathFactory(format))); return pathFactoryMap; } @@ -238,7 +226,8 @@ private KeyValueFileStoreScan newScan(boolean forWrite) { options.scanManifestParallelism(), options.deletionVectorsEnabled(), options.mergeEngine(), - options.changelogProducer()); + options.changelogProducer(), + options.fileIndexReadEnabled() && options.deletionVectorsEnabled()); } @Override diff --git a/paimon-core/src/main/java/org/apache/paimon/KeyValueThinSerializer.java b/paimon-core/src/main/java/org/apache/paimon/KeyValueThinSerializer.java new file mode 100644 index 000000000000..6dd41a42506a --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/KeyValueThinSerializer.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon; + +import org.apache.paimon.data.GenericRow; +import org.apache.paimon.data.InternalRow; +import org.apache.paimon.data.JoinedRow; +import org.apache.paimon.types.RowKind; +import org.apache.paimon.types.RowType; +import org.apache.paimon.utils.ObjectSerializer; + +/** Serialize KeyValue to InternalRow with ignorance of key. Only used to write KeyValue to disk. */ +public class KeyValueThinSerializer extends ObjectSerializer { + + private static final long serialVersionUID = 1L; + + private final GenericRow reusedMeta; + private final JoinedRow reusedKeyWithMeta; + + public KeyValueThinSerializer(RowType keyType, RowType valueType) { + super(KeyValue.schema(keyType, valueType)); + + this.reusedMeta = new GenericRow(2); + this.reusedKeyWithMeta = new JoinedRow(); + } + + public InternalRow toRow(KeyValue record) { + return toRow(record.sequenceNumber(), record.valueKind(), record.value()); + } + + public InternalRow toRow(long sequenceNumber, RowKind valueKind, InternalRow value) { + reusedMeta.setField(0, sequenceNumber); + reusedMeta.setField(1, valueKind.toByteValue()); + return reusedKeyWithMeta.replace(reusedMeta, value); + } + + @Override + public KeyValue fromRow(InternalRow row) { + throw new UnsupportedOperationException( + "KeyValue cannot be deserialized from InternalRow by this serializer."); + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/Snapshot.java b/paimon-core/src/main/java/org/apache/paimon/Snapshot.java index 3b8d2fa15b4b..baee7bad950e 100644 --- a/paimon-core/src/main/java/org/apache/paimon/Snapshot.java +++ b/paimon-core/src/main/java/org/apache/paimon/Snapshot.java @@ -29,9 +29,6 @@ import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonInclude; import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import javax.annotation.Nullable; import java.io.FileNotFoundException; @@ -65,7 +62,6 @@ @Public @JsonIgnoreProperties(ignoreUnknown = true) public class Snapshot { - private static final Logger LOG = LoggerFactory.getLogger(Snapshot.class); public static final long FIRST_SNAPSHOT_ID = 1; @@ -355,28 +351,6 @@ public String toJson() { return JsonSerdeUtil.toJson(this); } - public static Snapshot fromJson(String json) { - return JsonSerdeUtil.fromJson(json, Snapshot.class); - } - - public static Snapshot fromPath(FileIO fileIO, Path path) { - try { - return Snapshot.fromJson(fileIO.readFileUtf8(path)); - } catch (FileNotFoundException e) { - String errorMessage = - String.format( - "Snapshot file %s does not exist. " - + "It might have been expired by other jobs operating on this table. " - + "In this case, you can avoid concurrent modification issues by configuring " - + "write-only = true and use a dedicated compaction job, or configuring " - + "different expiration thresholds for different jobs.", - path); - throw new RuntimeException(errorMessage, e); - } catch (IOException e) { - throw new RuntimeException("Fails to read snapshot from path " + path, e); - } - } - @Override public int hashCode() { return Objects.hash( @@ -437,4 +411,36 @@ public enum CommitKind { /** Collect statistics. */ ANALYZE } + + // =================== Utils for reading ========================= + + public static Snapshot fromJson(String json) { + return JsonSerdeUtil.fromJson(json, Snapshot.class); + } + + public static Snapshot fromPath(FileIO fileIO, Path path) { + try { + return tryFromPath(fileIO, path); + } catch (FileNotFoundException e) { + String errorMessage = + String.format( + "Snapshot file %s does not exist. " + + "It might have been expired by other jobs operating on this table. " + + "In this case, you can avoid concurrent modification issues by configuring " + + "write-only = true and use a dedicated compaction job, or configuring " + + "different expiration thresholds for different jobs.", + path); + throw new RuntimeException(errorMessage, e); + } + } + + public static Snapshot tryFromPath(FileIO fileIO, Path path) throws FileNotFoundException { + try { + return Snapshot.fromJson(fileIO.readFileUtf8(path)); + } catch (FileNotFoundException e) { + throw e; + } catch (IOException e) { + throw new RuntimeException("Fails to read snapshot from path " + path, e); + } + } } diff --git a/paimon-core/src/main/java/org/apache/paimon/append/AppendOnlyWriter.java b/paimon-core/src/main/java/org/apache/paimon/append/AppendOnlyWriter.java index 402c6c1f45f4..a3087e362864 100644 --- a/paimon-core/src/main/java/org/apache/paimon/append/AppendOnlyWriter.java +++ b/paimon-core/src/main/java/org/apache/paimon/append/AppendOnlyWriter.java @@ -75,6 +75,7 @@ public class AppendOnlyWriter implements BatchRecordWriter, MemoryOwner { private final IOFunction, RecordReaderIterator> bucketFileRead; private final boolean forceCompact; private final boolean asyncFileWrite; + private final boolean statsDenseStore; private final List newFiles; private final List deletedFiles; private final List compactBefore; @@ -111,7 +112,8 @@ public AppendOnlyWriter( SimpleColStatsCollector.Factory[] statsCollectors, MemorySize maxDiskSize, FileIndexOptions fileIndexOptions, - boolean asyncFileWrite) { + boolean asyncFileWrite, + boolean statsDenseStore) { this.fileIO = fileIO; this.schemaId = schemaId; this.fileFormat = fileFormat; @@ -122,6 +124,7 @@ public AppendOnlyWriter( this.bucketFileRead = bucketFileRead; this.forceCompact = forceCompact; this.asyncFileWrite = asyncFileWrite; + this.statsDenseStore = statsDenseStore; this.newFiles = new ArrayList<>(); this.deletedFiles = new ArrayList<>(); this.compactBefore = new ArrayList<>(); @@ -208,6 +211,7 @@ public CommitIncrement prepareCommit(boolean waitCompaction) throws Exception { @Override public boolean isCompacting() { + compactManager.triggerCompaction(false); return compactManager.isCompacting(); } @@ -286,7 +290,8 @@ private RowDataRollingFileWriter createRollingRowWriter() { statsCollectors, fileIndexOptions, FileSource.APPEND, - asyncFileWrite); + asyncFileWrite, + statsDenseStore); } private void trySyncLatestCompaction(boolean blocking) diff --git a/paimon-core/src/main/java/org/apache/paimon/append/BucketedAppendCompactManager.java b/paimon-core/src/main/java/org/apache/paimon/append/BucketedAppendCompactManager.java index 3f9114069ace..02f469624c2a 100644 --- a/paimon-core/src/main/java/org/apache/paimon/append/BucketedAppendCompactManager.java +++ b/paimon-core/src/main/java/org/apache/paimon/append/BucketedAppendCompactManager.java @@ -115,10 +115,17 @@ private void triggerFullCompaction() { targetFileSize, rewriter, metricsReporter)); + recordCompactionsQueuedRequest(); compacting = new ArrayList<>(toCompact); toCompact.clear(); } + private void recordCompactionsQueuedRequest() { + if (metricsReporter != null) { + metricsReporter.increaseCompactionsQueuedCount(); + } + } + private void triggerCompactionWithBestEffort() { if (taskFuture != null) { return; @@ -130,6 +137,7 @@ private void triggerCompactionWithBestEffort() { executor.submit( new AutoCompactTask( dvMaintainer, compacting, rewriter, metricsReporter)); + recordCompactionsQueuedRequest(); } } diff --git a/paimon-core/src/main/java/org/apache/paimon/append/UnawareAppendTableCompactionCoordinator.java b/paimon-core/src/main/java/org/apache/paimon/append/UnawareAppendTableCompactionCoordinator.java index 9a54ea72e7cc..490bda9d4cf1 100644 --- a/paimon-core/src/main/java/org/apache/paimon/append/UnawareAppendTableCompactionCoordinator.java +++ b/paimon-core/src/main/java/org/apache/paimon/append/UnawareAppendTableCompactionCoordinator.java @@ -27,6 +27,7 @@ import org.apache.paimon.index.IndexFileHandler; import org.apache.paimon.index.IndexFileMeta; import org.apache.paimon.io.DataFileMeta; +import org.apache.paimon.manifest.FileKind; import org.apache.paimon.manifest.ManifestEntry; import org.apache.paimon.predicate.Predicate; import org.apache.paimon.table.FileStoreTable; @@ -379,6 +380,10 @@ public FilesIterator( if (filter != null) { snapshotReader.withFilter(filter); } + // drop stats to reduce memory + if (table.coreOptions().manifestDeleteFileDropStats()) { + snapshotReader.dropStats(); + } this.streamingMode = isStreaming; } @@ -387,6 +392,9 @@ private void assignNewIterator() { if (nextSnapshot == null) { nextSnapshot = snapshotManager.latestSnapshotId(); if (nextSnapshot == null) { + if (!streamingMode) { + throw new EndOfScanException(); + } return; } snapshotReader.withMode(ScanMode.ALL); @@ -438,7 +446,12 @@ public ManifestEntry next() { } if (currentIterator.hasNext()) { - return currentIterator.next(); + ManifestEntry entry = currentIterator.next(); + if (entry.kind() == FileKind.DELETE) { + continue; + } else { + return entry; + } } currentIterator = null; } diff --git a/paimon-core/src/main/java/org/apache/paimon/catalog/AbstractCatalog.java b/paimon-core/src/main/java/org/apache/paimon/catalog/AbstractCatalog.java index 5ea714530f35..b56fec279ab1 100644 --- a/paimon-core/src/main/java/org/apache/paimon/catalog/AbstractCatalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/catalog/AbstractCatalog.java @@ -19,11 +19,13 @@ package org.apache.paimon.catalog; import org.apache.paimon.CoreOptions; +import org.apache.paimon.TableType; import org.apache.paimon.factories.FactoryUtil; import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.FileStatus; import org.apache.paimon.fs.Path; -import org.apache.paimon.lineage.LineageMetaFactory; +import org.apache.paimon.manifest.PartitionEntry; +import org.apache.paimon.metastore.MetastoreClient; import org.apache.paimon.operation.FileStoreCommit; import org.apache.paimon.operation.Lock; import org.apache.paimon.options.Options; @@ -36,16 +38,21 @@ import org.apache.paimon.table.FileStoreTableFactory; import org.apache.paimon.table.FormatTable; import org.apache.paimon.table.Table; +import org.apache.paimon.table.object.ObjectTable; import org.apache.paimon.table.sink.BatchWriteBuilder; import org.apache.paimon.table.system.SystemTableLoader; +import org.apache.paimon.types.RowType; import org.apache.paimon.utils.Preconditions; import javax.annotation.Nullable; import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Optional; @@ -53,13 +60,12 @@ import static org.apache.paimon.CoreOptions.TYPE; import static org.apache.paimon.CoreOptions.createCommitUser; -import static org.apache.paimon.TableType.FORMAT_TABLE; import static org.apache.paimon.options.CatalogOptions.ALLOW_UPPER_CASE; -import static org.apache.paimon.options.CatalogOptions.LINEAGE_META; import static org.apache.paimon.options.CatalogOptions.LOCK_ENABLED; import static org.apache.paimon.options.CatalogOptions.LOCK_TYPE; import static org.apache.paimon.utils.BranchManager.DEFAULT_MAIN_BRANCH; import static org.apache.paimon.utils.Preconditions.checkArgument; +import static org.apache.paimon.utils.Preconditions.checkNotNull; /** Common implementation of {@link Catalog}. */ public abstract class AbstractCatalog implements Catalog { @@ -68,19 +74,14 @@ public abstract class AbstractCatalog implements Catalog { protected final Map tableDefaultOptions; protected final Options catalogOptions; - @Nullable protected final LineageMetaFactory lineageMetaFactory; - protected AbstractCatalog(FileIO fileIO) { this.fileIO = fileIO; - this.lineageMetaFactory = null; this.tableDefaultOptions = new HashMap<>(); this.catalogOptions = new Options(); } protected AbstractCatalog(FileIO fileIO, Options options) { this.fileIO = fileIO; - this.lineageMetaFactory = - findAndCreateLineageMeta(options, AbstractCatalog.class.getClassLoader()); this.tableDefaultOptions = Catalog.tableDefaultOptions(options.toMap()); this.catalogOptions = options; } @@ -95,7 +96,6 @@ public FileIO fileIO() { return fileIO; } - @Override public Optional lockFactory() { if (!lockEnabled()) { return Optional.empty(); @@ -115,7 +115,6 @@ public Optional defaultLockFactory() { return Optional.empty(); } - @Override public Optional lockContext() { return Optional.of(CatalogLockContext.fromOptions(catalogOptions)); } @@ -129,30 +128,60 @@ public boolean allowUpperCase() { return catalogOptions.getOptional(ALLOW_UPPER_CASE).orElse(true); } + protected boolean allowCustomTablePath() { + return false; + } + @Override public void createDatabase(String name, boolean ignoreIfExists, Map properties) throws DatabaseAlreadyExistException { checkNotSystemDatabase(name); - if (databaseExists(name)) { + try { + getDatabase(name); if (ignoreIfExists) { return; } throw new DatabaseAlreadyExistException(name); + } catch (DatabaseNotExistException ignored) { } createDatabaseImpl(name, properties); } @Override - public Map loadDatabaseProperties(String name) - throws DatabaseNotExistException { + public Database getDatabase(String name) throws DatabaseNotExistException { if (isSystemDatabase(name)) { - return Collections.emptyMap(); + return Database.of(name); } - return loadDatabasePropertiesImpl(name); + return getDatabaseImpl(name); } - protected abstract Map loadDatabasePropertiesImpl(String name) - throws DatabaseNotExistException; + protected abstract Database getDatabaseImpl(String name) throws DatabaseNotExistException; + + @Override + public void createPartition(Identifier identifier, Map partitionSpec) + throws TableNotExistException { + Identifier tableIdentifier = + Identifier.create(identifier.getDatabaseName(), identifier.getTableName()); + FileStoreTable table = (FileStoreTable) getTable(tableIdentifier); + + if (table.partitionKeys().isEmpty() || !table.coreOptions().partitionedTableInMetastore()) { + throw new UnsupportedOperationException( + "The table is not partitioned table in metastore."); + } + + MetastoreClient.Factory metastoreFactory = + table.catalogEnvironment().metastoreClientFactory(); + if (metastoreFactory == null) { + throw new UnsupportedOperationException( + "The catalog must have metastore to create partition."); + } + + try (MetastoreClient client = metastoreFactory.create()) { + client.addPartition(new LinkedHashMap<>(partitionSpec)); + } catch (Exception e) { + throw new RuntimeException(e); + } + } @Override public void dropPartition(Identifier identifier, Map partitionSpec) @@ -170,13 +199,21 @@ public void dropPartition(Identifier identifier, Map partitionSp } } + @Override + public List listPartitions(Identifier identifier) + throws TableNotExistException { + return getTable(identifier).newReadBuilder().newScan().listPartitionEntries(); + } + protected abstract void createDatabaseImpl(String name, Map properties); @Override public void dropDatabase(String name, boolean ignoreIfNotExists, boolean cascade) throws DatabaseNotExistException, DatabaseNotEmptyException { checkNotSystemDatabase(name); - if (!databaseExists(name)) { + try { + getDatabase(name); + } catch (DatabaseNotExistException e) { if (ignoreIfNotExists) { return; } @@ -197,9 +234,9 @@ public List listTables(String databaseName) throws DatabaseNotExistExcep if (isSystemDatabase(databaseName)) { return SystemTableLoader.loadGlobalTableNames(); } - if (!databaseExists(databaseName)) { - throw new DatabaseNotExistException(databaseName); - } + + // check db exists + getDatabase(databaseName); return listTablesImpl(databaseName).stream().sorted().collect(Collectors.toList()); } @@ -212,7 +249,9 @@ public void dropTable(Identifier identifier, boolean ignoreIfNotExists) checkNotBranch(identifier, "dropTable"); checkNotSystemTable(identifier, "dropTable"); - if (!tableExists(identifier)) { + try { + getTable(identifier); + } catch (TableNotExistException e) { if (ignoreIfNotExists) { return; } @@ -232,27 +271,51 @@ public void createTable(Identifier identifier, Schema schema, boolean ignoreIfEx validateIdentifierNameCaseInsensitive(identifier); validateFieldNameCaseInsensitive(schema.rowType().getFieldNames()); validateAutoCreateClose(schema.options()); + validateCustomTablePath(schema.options()); - if (!databaseExists(identifier.getDatabaseName())) { - throw new DatabaseNotExistException(identifier.getDatabaseName()); - } + // check db exists + getDatabase(identifier.getDatabaseName()); - if (tableExists(identifier)) { + try { + getTable(identifier); if (ignoreIfExists) { return; } throw new TableAlreadyExistException(identifier); + } catch (TableNotExistException ignored) { } copyTableDefaultOptions(schema.options()); - if (Options.fromMap(schema.options()).get(TYPE) == FORMAT_TABLE) { - createFormatTable(identifier, schema); - } else { - createTableImpl(identifier, schema); + switch (Options.fromMap(schema.options()).get(TYPE)) { + case TABLE: + case MATERIALIZED_TABLE: + createTableImpl(identifier, schema); + break; + case OBJECT_TABLE: + createObjectTable(identifier, schema); + break; + case FORMAT_TABLE: + createFormatTable(identifier, schema); + break; } } + private void createObjectTable(Identifier identifier, Schema schema) { + RowType rowType = schema.rowType(); + checkArgument( + rowType.getFields().isEmpty() + || new HashSet<>(ObjectTable.SCHEMA.getFields()) + .containsAll(rowType.getFields()), + "Schema of Object Table can be empty or %s, but is %s.", + ObjectTable.SCHEMA, + rowType); + checkArgument( + schema.options().containsKey(CoreOptions.OBJECT_LOCATION.key()), + "Object table should have object-location option."); + createTableImpl(identifier, schema.copy(ObjectTable.SCHEMA)); + } + protected abstract void createTableImpl(Identifier identifier, Schema schema); @Override @@ -264,15 +327,19 @@ public void renameTable(Identifier fromTable, Identifier toTable, boolean ignore checkNotSystemTable(toTable, "renameTable"); validateIdentifierNameCaseInsensitive(toTable); - if (!tableExists(fromTable)) { + try { + getTable(fromTable); + } catch (TableNotExistException e) { if (ignoreIfNotExists) { return; } throw new TableNotExistException(fromTable); } - if (tableExists(toTable)) { + try { + getTable(toTable); throw new TableAlreadyExistException(toTable); + } catch (TableNotExistException ignored) { } renameTableImpl(fromTable, toTable); @@ -288,7 +355,9 @@ public void alterTable( validateIdentifierNameCaseInsensitive(identifier); validateFieldNameCaseInsensitiveInSchemaChange(changes); - if (!tableExists(identifier)) { + try { + getTable(identifier); + } catch (TableNotExistException e) { if (ignoreIfNotExists) { return; } @@ -301,80 +370,79 @@ public void alterTable( protected abstract void alterTableImpl(Identifier identifier, List changes) throws TableNotExistException, ColumnAlreadyExistException, ColumnNotExistException; - @Nullable - private LineageMetaFactory findAndCreateLineageMeta(Options options, ClassLoader classLoader) { - return options.getOptional(LINEAGE_META) - .map( - meta -> - FactoryUtil.discoverFactory( - classLoader, LineageMetaFactory.class, meta)) - .orElse(null); - } - @Override public Table getTable(Identifier identifier) throws TableNotExistException { if (isSystemDatabase(identifier.getDatabaseName())) { String tableName = identifier.getTableName(); Table table = SystemTableLoader.loadGlobal( - tableName, - fileIO, - this::allTablePaths, - catalogOptions, - lineageMetaFactory); + tableName, fileIO, this::allTablePaths, catalogOptions); if (table == null) { throw new TableNotExistException(identifier); } return table; - } else if (isSpecifiedSystemTable(identifier)) { - FileStoreTable originTable = - getDataTable( + } else if (identifier.isSystemTable()) { + Table originTable = + getDataOrFormatTable( new Identifier( identifier.getDatabaseName(), identifier.getTableName(), identifier.getBranchName(), null)); + if (!(originTable instanceof FileStoreTable)) { + throw new UnsupportedOperationException( + String.format( + "Only data table support system tables, but this table %s is %s.", + identifier, originTable.getClass())); + } Table table = SystemTableLoader.load( Preconditions.checkNotNull(identifier.getSystemTableName()), - originTable); + (FileStoreTable) originTable); if (table == null) { throw new TableNotExistException(identifier); } return table; } else { - try { - return getDataTable(identifier); - } catch (TableNotExistException e) { - return getFormatTable(identifier); - } + return getDataOrFormatTable(identifier); } } - private FileStoreTable getDataTable(Identifier identifier) throws TableNotExistException { + protected Table getDataOrFormatTable(Identifier identifier) throws TableNotExistException { Preconditions.checkArgument(identifier.getSystemTableName() == null); - TableSchema tableSchema = getDataTableSchema(identifier); - return FileStoreTableFactory.create( - fileIO, - getTableLocation(identifier), - tableSchema, - new CatalogEnvironment( - identifier, - Lock.factory( - lockFactory().orElse(null), lockContext().orElse(null), identifier), - metastoreClientFactory(identifier).orElse(null), - lineageMetaFactory)); + TableMeta tableMeta = getDataTableMeta(identifier); + FileStoreTable table = + FileStoreTableFactory.create( + fileIO, + getTableLocation(identifier), + tableMeta.schema, + new CatalogEnvironment( + identifier, + tableMeta.uuid, + Lock.factory( + lockFactory().orElse(null), + lockContext().orElse(null), + identifier), + metastoreClientFactory(identifier, tableMeta.schema).orElse(null))); + CoreOptions options = table.coreOptions(); + if (options.type() == TableType.OBJECT_TABLE) { + String objectLocation = options.objectLocation(); + checkNotNull(objectLocation, "Object location should not be null for object table."); + table = + ObjectTable.builder() + .underlyingTable(table) + .objectLocation(objectLocation) + .objectFileIO(objectFileIO(objectLocation)) + .build(); + } + return table; } /** - * Return a {@link FormatTable} identified by the given {@link Identifier}. - * - * @param identifier Path of the table - * @return The requested table - * @throws Catalog.TableNotExistException if the target does not exist + * Catalog implementation may override this method to provide {@link FileIO} to object table. */ - public FormatTable getFormatTable(Identifier identifier) throws Catalog.TableNotExistException { - throw new Catalog.TableNotExistException(identifier); + protected FileIO objectFileIO(String objectLocation) { + return fileIO; } /** @@ -415,9 +483,19 @@ public Map> allTablePaths() { } } + protected TableMeta getDataTableMeta(Identifier identifier) throws TableNotExistException { + return new TableMeta(getDataTableSchema(identifier), null); + } + protected abstract TableSchema getDataTableSchema(Identifier identifier) throws TableNotExistException; + /** Get metastore client factory for the table specified by {@code identifier}. */ + public Optional metastoreClientFactory( + Identifier identifier, TableSchema schema) { + return Optional.empty(); + } + @Override public Path getTableLocation(Identifier identifier) { return new Path(newDatabasePath(identifier.getDatabaseName()), identifier.getTableName()); @@ -441,16 +519,12 @@ protected void assertMainBranch(Identifier identifier) { } } - public static boolean isSpecifiedSystemTable(Identifier identifier) { - return identifier.getSystemTableName() != null; - } - - protected static boolean isSystemTable(Identifier identifier) { - return isSystemDatabase(identifier.getDatabaseName()) || isSpecifiedSystemTable(identifier); + protected static boolean isTableInSystemDatabase(Identifier identifier) { + return isSystemDatabase(identifier.getDatabaseName()) || identifier.isSystemTable(); } protected static void checkNotSystemTable(Identifier identifier, String method) { - if (isSystemTable(identifier)) { + if (isTableInSystemDatabase(identifier)) { throw new IllegalArgumentException( String.format( "Cannot '%s' for system table '%s', please use data table.", @@ -495,7 +569,7 @@ private void validateFieldNameCaseInsensitiveInSchemaChange(List c for (SchemaChange change : changes) { if (change instanceof SchemaChange.AddColumn) { SchemaChange.AddColumn addColumn = (SchemaChange.AddColumn) change; - fieldNames.add(addColumn.fieldName()); + fieldNames.addAll(Arrays.asList(addColumn.fieldNames())); } else if (change instanceof SchemaChange.RenameColumn) { SchemaChange.RenameColumn rename = (SchemaChange.RenameColumn) change; fieldNames.add(rename.newName()); @@ -519,6 +593,15 @@ private void validateAutoCreateClose(Map options) { CoreOptions.AUTO_CREATE.key(), Boolean.FALSE)); } + private void validateCustomTablePath(Map options) { + if (!allowCustomTablePath() && options.containsKey(CoreOptions.PATH.key())) { + throw new UnsupportedOperationException( + String.format( + "The current catalog %s does not support specifying the table path when creating a table.", + this.getClass().getSimpleName())); + } + } + // =============================== Meta in File System ===================================== protected List listDatabasesInFileSystem(Path warehouse) throws IOException { @@ -544,7 +627,16 @@ protected List listTablesInFileSystem(Path databasePath) throws IOExcept } protected boolean tableExistsInFileSystem(Path tablePath, String branchName) { - return !new SchemaManager(fileIO, tablePath, branchName).listAllIds().isEmpty(); + SchemaManager schemaManager = new SchemaManager(fileIO, tablePath, branchName); + + // in order to improve the performance, check the schema-0 firstly. + boolean schemaZeroExists = schemaManager.schemaExists(0); + if (schemaZeroExists) { + return true; + } else { + // if schema-0 not exists, fallback to check other schemas + return !schemaManager.listAllIds().isEmpty(); + } } public Optional tableSchemaInFileSystem(Path tablePath, String branchName) { @@ -561,4 +653,25 @@ public Optional tableSchemaInFileSystem(Path tablePath, String bran } }); } + + /** Table metadata. */ + protected static class TableMeta { + + private final TableSchema schema; + @Nullable private final String uuid; + + public TableMeta(TableSchema schema, @Nullable String uuid) { + this.schema = schema; + this.uuid = uuid; + } + + public TableSchema schema() { + return schema; + } + + @Nullable + public String uuid() { + return uuid; + } + } } diff --git a/paimon-core/src/main/java/org/apache/paimon/catalog/CachingCatalog.java b/paimon-core/src/main/java/org/apache/paimon/catalog/CachingCatalog.java index 0777759456f8..82d503b7a272 100644 --- a/paimon-core/src/main/java/org/apache/paimon/catalog/CachingCatalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/catalog/CachingCatalog.java @@ -19,24 +19,19 @@ package org.apache.paimon.catalog; import org.apache.paimon.fs.Path; +import org.apache.paimon.manifest.PartitionEntry; import org.apache.paimon.options.MemorySize; import org.apache.paimon.options.Options; import org.apache.paimon.schema.SchemaChange; import org.apache.paimon.table.FileStoreTable; import org.apache.paimon.table.Table; import org.apache.paimon.table.system.SystemTableLoader; -import org.apache.paimon.utils.Preconditions; import org.apache.paimon.utils.SegmentsCache; import org.apache.paimon.shade.caffeine2.com.github.benmanes.caffeine.cache.Cache; import org.apache.paimon.shade.caffeine2.com.github.benmanes.caffeine.cache.Caffeine; -import org.apache.paimon.shade.caffeine2.com.github.benmanes.caffeine.cache.RemovalCause; -import org.apache.paimon.shade.caffeine2.com.github.benmanes.caffeine.cache.RemovalListener; import org.apache.paimon.shade.caffeine2.com.github.benmanes.caffeine.cache.Ticker; - -import org.checkerframework.checker.nullness.qual.NonNull; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import org.apache.paimon.shade.caffeine2.com.github.benmanes.caffeine.cache.Weigher; import javax.annotation.Nullable; @@ -46,41 +41,52 @@ import java.util.Map; import java.util.Optional; -import static org.apache.paimon.catalog.AbstractCatalog.isSpecifiedSystemTable; import static org.apache.paimon.options.CatalogOptions.CACHE_ENABLED; import static org.apache.paimon.options.CatalogOptions.CACHE_EXPIRATION_INTERVAL_MS; import static org.apache.paimon.options.CatalogOptions.CACHE_MANIFEST_MAX_MEMORY; import static org.apache.paimon.options.CatalogOptions.CACHE_MANIFEST_SMALL_FILE_MEMORY; import static org.apache.paimon.options.CatalogOptions.CACHE_MANIFEST_SMALL_FILE_THRESHOLD; -import static org.apache.paimon.table.system.SystemTableLoader.SYSTEM_TABLES; +import static org.apache.paimon.options.CatalogOptions.CACHE_PARTITION_MAX_NUM; +import static org.apache.paimon.options.CatalogOptions.CACHE_SNAPSHOT_MAX_NUM_PER_TABLE; +import static org.apache.paimon.utils.Preconditions.checkNotNull; /** A {@link Catalog} to cache databases and tables and manifests. */ public class CachingCatalog extends DelegateCatalog { - private static final Logger LOG = LoggerFactory.getLogger(CachingCatalog.class); + private final Duration expirationInterval; + private final int snapshotMaxNumPerTable; - protected final Cache> databaseCache; + protected final Cache databaseCache; protected final Cache tableCache; @Nullable protected final SegmentsCache manifestCache; + // partition cache will affect data latency + @Nullable protected final Cache> partitionCache; + public CachingCatalog(Catalog wrapped) { this( wrapped, CACHE_EXPIRATION_INTERVAL_MS.defaultValue(), CACHE_MANIFEST_SMALL_FILE_MEMORY.defaultValue(), - CACHE_MANIFEST_SMALL_FILE_THRESHOLD.defaultValue().getBytes()); + CACHE_MANIFEST_SMALL_FILE_THRESHOLD.defaultValue().getBytes(), + CACHE_PARTITION_MAX_NUM.defaultValue(), + CACHE_SNAPSHOT_MAX_NUM_PER_TABLE.defaultValue()); } public CachingCatalog( Catalog wrapped, Duration expirationInterval, MemorySize manifestMaxMemory, - long manifestCacheThreshold) { + long manifestCacheThreshold, + long cachedPartitionMaxNum, + int snapshotMaxNumPerTable) { this( wrapped, expirationInterval, manifestMaxMemory, manifestCacheThreshold, + cachedPartitionMaxNum, + snapshotMaxNumPerTable, Ticker.systemTicker()); } @@ -89,6 +95,8 @@ public CachingCatalog( Duration expirationInterval, MemorySize manifestMaxMemory, long manifestCacheThreshold, + long cachedPartitionMaxNum, + int snapshotMaxNumPerTable, Ticker ticker) { super(wrapped); if (expirationInterval.isZero() || expirationInterval.isNegative()) { @@ -96,6 +104,9 @@ public CachingCatalog( "When cache.expiration-interval is set to negative or 0, the catalog cache should be disabled."); } + this.expirationInterval = expirationInterval; + this.snapshotMaxNumPerTable = snapshotMaxNumPerTable; + this.databaseCache = Caffeine.newBuilder() .softValues() @@ -106,12 +117,24 @@ public CachingCatalog( this.tableCache = Caffeine.newBuilder() .softValues() - .removalListener(new TableInvalidatingRemovalListener()) .executor(Runnable::run) .expireAfterAccess(expirationInterval) .ticker(ticker) .build(); this.manifestCache = SegmentsCache.create(manifestMaxMemory, manifestCacheThreshold); + this.partitionCache = + cachedPartitionMaxNum == 0 + ? null + : Caffeine.newBuilder() + .softValues() + .executor(Runnable::run) + .expireAfterAccess(expirationInterval) + .weigher( + (Weigher>) + (identifier, v) -> v.size()) + .maximumWeight(cachedPartitionMaxNum) + .ticker(ticker) + .build(); } public static Catalog tryToCreate(Catalog catalog, Options options) { @@ -131,20 +154,21 @@ public static Catalog tryToCreate(Catalog catalog, Options options) { catalog, options.get(CACHE_EXPIRATION_INTERVAL_MS), manifestMaxMemory, - manifestThreshold); + manifestThreshold, + options.get(CACHE_PARTITION_MAX_NUM), + options.get(CACHE_SNAPSHOT_MAX_NUM_PER_TABLE)); } @Override - public Map loadDatabaseProperties(String databaseName) - throws DatabaseNotExistException { - Map properties = databaseCache.getIfPresent(databaseName); - if (properties != null) { - return properties; + public Database getDatabase(String databaseName) throws DatabaseNotExistException { + Database database = databaseCache.getIfPresent(databaseName); + if (database != null) { + return database; } - properties = super.loadDatabaseProperties(databaseName); - databaseCache.put(databaseName, properties); - return properties; + database = super.getDatabase(databaseName); + databaseCache.put(databaseName, database); + return database; } @Override @@ -168,6 +192,13 @@ public void dropTable(Identifier identifier, boolean ignoreIfNotExists) throws TableNotExistException { super.dropTable(identifier, ignoreIfNotExists); invalidateTable(identifier); + + // clear all branch tables of this table + for (Identifier i : tableCache.asMap().keySet()) { + if (identifier.getTableName().equals(i.getTableName())) { + tableCache.invalidate(i); + } + } } @Override @@ -192,26 +223,23 @@ public Table getTable(Identifier identifier) throws TableNotExistException { return table; } - if (isSpecifiedSystemTable(identifier)) { + // For system table, do not cache it directly. Instead, cache the origin table and then wrap + // it to generate the system table. + if (identifier.isSystemTable()) { Identifier originIdentifier = new Identifier( identifier.getDatabaseName(), identifier.getTableName(), identifier.getBranchName(), null); - Table originTable = tableCache.getIfPresent(originIdentifier); - if (originTable == null) { - originTable = wrapped.getTable(originIdentifier); - putTableCache(originIdentifier, originTable); - } + Table originTable = getTable(originIdentifier); table = SystemTableLoader.load( - Preconditions.checkNotNull(identifier.getSystemTableName()), + checkNotNull(identifier.getSystemTableName()), (FileStoreTable) originTable); if (table == null) { throw new TableNotExistException(identifier); } - putTableCache(identifier, table); return table; } @@ -221,39 +249,69 @@ public Table getTable(Identifier identifier) throws TableNotExistException { } private void putTableCache(Identifier identifier, Table table) { - if (manifestCache != null && table instanceof FileStoreTable) { - ((FileStoreTable) table).setManifestCache(manifestCache); + if (table instanceof FileStoreTable) { + FileStoreTable storeTable = (FileStoreTable) table; + storeTable.setSnapshotCache( + Caffeine.newBuilder() + .softValues() + .expireAfterAccess(expirationInterval) + .maximumSize(snapshotMaxNumPerTable) + .executor(Runnable::run) + .build()); + storeTable.setStatsCache( + Caffeine.newBuilder() + .softValues() + .expireAfterAccess(expirationInterval) + .maximumSize(5) + .executor(Runnable::run) + .build()); + if (manifestCache != null) { + storeTable.setManifestCache(manifestCache); + } } + tableCache.put(identifier, table); } - private class TableInvalidatingRemovalListener implements RemovalListener { - @Override - public void onRemoval(Identifier identifier, Table table, @NonNull RemovalCause cause) { - LOG.debug("Evicted {} from the table cache ({})", identifier, cause); - if (RemovalCause.EXPIRED.equals(cause)) { - tryInvalidateSysTables(identifier); - } + @Override + public List listPartitions(Identifier identifier) + throws TableNotExistException { + if (partitionCache == null) { + return wrapped.listPartitions(identifier); + } + + List result = partitionCache.getIfPresent(identifier); + if (result == null) { + result = wrapped.listPartitions(identifier); + partitionCache.put(identifier, result); } + return result; } @Override - public void invalidateTable(Identifier identifier) { - tableCache.invalidate(identifier); - tryInvalidateSysTables(identifier); + public void dropPartition(Identifier identifier, Map partitions) + throws TableNotExistException, PartitionNotExistException { + wrapped.dropPartition(identifier, partitions); + if (partitionCache != null) { + partitionCache.invalidate(identifier); + } } - private void tryInvalidateSysTables(Identifier identifier) { - if (!isSpecifiedSystemTable(identifier)) { - tableCache.invalidateAll(allSystemTables(identifier)); + @Override + public void invalidateTable(Identifier identifier) { + tableCache.invalidate(identifier); + if (partitionCache != null) { + partitionCache.invalidate(identifier); } } - private static Iterable allSystemTables(Identifier ident) { - List tables = new ArrayList<>(); - for (String type : SYSTEM_TABLES) { - tables.add(Identifier.fromString(ident.getFullName() + SYSTEM_TABLE_SPLITTER + type)); + // ================================== refresh ================================================ + // following caches will affect the latency of table, so refresh method is provided for engine + + public void refreshPartitions(Identifier identifier) throws TableNotExistException { + if (partitionCache != null) { + List result = wrapped.listPartitions(identifier); + partitionCache.put(identifier, result); } - return tables; } } diff --git a/paimon-core/src/main/java/org/apache/paimon/catalog/Catalog.java b/paimon-core/src/main/java/org/apache/paimon/catalog/Catalog.java index 106f7b15afd7..d919c5978297 100644 --- a/paimon-core/src/main/java/org/apache/paimon/catalog/Catalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/catalog/Catalog.java @@ -21,17 +21,17 @@ import org.apache.paimon.annotation.Public; import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.Path; -import org.apache.paimon.metastore.MetastoreClient; +import org.apache.paimon.manifest.PartitionEntry; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaChange; import org.apache.paimon.table.Table; +import org.apache.paimon.view.View; import java.io.Serializable; import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.stream.Collectors; import static org.apache.paimon.options.OptionsUtils.convertToPropertiesPrefixKey; @@ -52,11 +52,18 @@ public interface Catalog extends AutoCloseable { String SYSTEM_TABLE_SPLITTER = "$"; String SYSTEM_DATABASE_NAME = "sys"; String SYSTEM_BRANCH_PREFIX = "branch_"; - String COMMENT_PROP = "comment"; String TABLE_DEFAULT_OPTION_PREFIX = "table-default."; - String DB_LOCATION_PROP = "location"; String DB_SUFFIX = ".db"; + String COMMENT_PROP = "comment"; + String OWNER_PROP = "owner"; + String DB_LOCATION_PROP = "location"; + String NUM_ROWS_PROP = "numRows"; + String NUM_FILES_PROP = "numFiles"; + String TOTAL_SIZE_PROP = "totalSize"; + String LAST_UPDATE_TIME_PROP = "lastUpdateTime"; + String HIVE_LAST_UPDATE_TIME_PROP = "transient_lastDdlTime"; + /** Warehouse root path containing all database directories in this catalog. */ String warehouse(); @@ -65,22 +72,6 @@ public interface Catalog extends AutoCloseable { FileIO fileIO(); - /** - * Get lock factory from catalog. Lock is used to support multiple concurrent writes on the - * object store. - */ - Optional lockFactory(); - - /** Get lock context for lock factory to create a lock. */ - default Optional lockContext() { - return Optional.empty(); - } - - /** Get metastore client factory for the table specified by {@code identifier}. */ - default Optional metastoreClientFactory(Identifier identifier) { - return Optional.empty(); - } - /** * Get the names of all databases in this catalog. * @@ -88,21 +79,6 @@ default Optional metastoreClientFactory(Identifier iden */ List listDatabases(); - /** - * Check if a database exists in this catalog. - * - * @param databaseName Name of the database - * @return true if the given database exists in the catalog false otherwise - */ - default boolean databaseExists(String databaseName) { - try { - loadDatabaseProperties(databaseName); - return true; - } catch (DatabaseNotExistException e) { - return false; - } - } - /** * Create a database, see {@link Catalog#createDatabase(String name, boolean ignoreIfExists, Map * properties)}. @@ -127,13 +103,13 @@ void createDatabase(String name, boolean ignoreIfExists, Map pro throws DatabaseAlreadyExistException; /** - * Load database properties. + * Return a {@link Database} identified by the given name. * * @param name Database name - * @return The requested database's properties + * @return The requested {@link Database} * @throws DatabaseNotExistException if the requested database does not exist */ - Map loadDatabaseProperties(String name) throws DatabaseNotExistException; + Database getDatabase(String name) throws DatabaseNotExistException; /** * Drop a database. @@ -178,20 +154,6 @@ void dropDatabase(String name, boolean ignoreIfNotExists, boolean cascade) */ List listTables(String databaseName) throws DatabaseNotExistException; - /** - * Check if a table exists in this catalog. - * - * @param identifier Path of the table - * @return true if the given table exists in the catalog false otherwise - */ - default boolean tableExists(Identifier identifier) { - try { - return getTable(identifier) != null; - } catch (TableNotExistException e) { - return false; - } - } - /** * Drop a table. * @@ -262,6 +224,19 @@ void alterTable(Identifier identifier, List changes, boolean ignor */ default void invalidateTable(Identifier identifier) {} + /** + * Create the partition of the specify table. + * + *

    Only catalog with metastore can support this method, and only table with + * 'metastore.partitioned-table' can support this method. + * + * @param identifier path of the table to drop partition + * @param partitionSpec the partition to be created + * @throws TableNotExistException if the table does not exist + */ + void createPartition(Identifier identifier, Map partitionSpec) + throws TableNotExistException; + /** * Drop the partition of the specify table. * @@ -273,6 +248,14 @@ default void invalidateTable(Identifier identifier) {} void dropPartition(Identifier identifier, Map partitions) throws TableNotExistException, PartitionNotExistException; + /** + * Get PartitionEntry of all partitions of the table. + * + * @param identifier path of the table to list partitions + * @throws TableNotExistException if the table does not exist + */ + List listPartitions(Identifier identifier) throws TableNotExistException; + /** * Modify an existing table from a {@link SchemaChange}. * @@ -289,6 +272,68 @@ default void alterTable(Identifier identifier, SchemaChange change, boolean igno alterTable(identifier, Collections.singletonList(change), ignoreIfNotExists); } + /** + * Return a {@link View} identified by the given {@link Identifier}. + * + * @param identifier Path of the view + * @return The requested view + * @throws ViewNotExistException if the target does not exist + */ + default View getView(Identifier identifier) throws ViewNotExistException { + throw new ViewNotExistException(identifier); + } + + /** + * Drop a view. + * + * @param identifier Path of the view to be dropped + * @param ignoreIfNotExists Flag to specify behavior when the view does not exist: if set to + * false, throw an exception, if set to true, do nothing. + * @throws ViewNotExistException if the view does not exist + */ + default void dropView(Identifier identifier, boolean ignoreIfNotExists) + throws ViewNotExistException { + throw new UnsupportedOperationException(); + } + + /** + * Create a new view. + * + * @param identifier path of the view to be created + * @param view the view definition + * @param ignoreIfExists flag to specify behavior when a view already exists at the given path: + * if set to false, it throws a ViewAlreadyExistException, if set to true, do nothing. + * @throws ViewAlreadyExistException if view already exists and ignoreIfExists is false + * @throws DatabaseNotExistException if the database in identifier doesn't exist + */ + default void createView(Identifier identifier, View view, boolean ignoreIfExists) + throws ViewAlreadyExistException, DatabaseNotExistException { + throw new UnsupportedOperationException(); + } + + /** + * Get names of all views under this database. An empty list is returned if none exists. + * + * @return a list of the names of all views in this database + * @throws DatabaseNotExistException if the database does not exist + */ + default List listViews(String databaseName) throws DatabaseNotExistException { + return Collections.emptyList(); + } + + /** + * Rename a view. + * + * @param fromView identifier of the view to rename + * @param toView new view identifier + * @throws ViewNotExistException if the fromView does not exist + * @throws ViewAlreadyExistException if the toView already exists + */ + default void renameView(Identifier fromView, Identifier toView, boolean ignoreIfNotExists) + throws ViewNotExistException, ViewAlreadyExistException { + throw new UnsupportedOperationException(); + } + /** Return a boolean that indicates whether this catalog allow upper case. */ boolean allowUpperCase(); @@ -522,6 +567,48 @@ public String column() { } } + /** Exception for trying to create a view that already exists. */ + class ViewAlreadyExistException extends Exception { + + private static final String MSG = "View %s already exists."; + + private final Identifier identifier; + + public ViewAlreadyExistException(Identifier identifier) { + this(identifier, null); + } + + public ViewAlreadyExistException(Identifier identifier, Throwable cause) { + super(String.format(MSG, identifier.getFullName()), cause); + this.identifier = identifier; + } + + public Identifier identifier() { + return identifier; + } + } + + /** Exception for trying to operate on a view that doesn't exist. */ + class ViewNotExistException extends Exception { + + private static final String MSG = "View %s does not exist."; + + private final Identifier identifier; + + public ViewNotExistException(Identifier identifier) { + this(identifier, null); + } + + public ViewNotExistException(Identifier identifier, Throwable cause) { + super(String.format(MSG, identifier.getFullName()), cause); + this.identifier = identifier; + } + + public Identifier identifier() { + return identifier; + } + } + /** Loader of {@link Catalog}. */ @FunctionalInterface interface Loader extends Serializable { diff --git a/paimon-core/src/main/java/org/apache/paimon/catalog/Database.java b/paimon-core/src/main/java/org/apache/paimon/catalog/Database.java new file mode 100644 index 000000000000..f855e57e9143 --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/catalog/Database.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.catalog; + +import org.apache.paimon.annotation.Public; + +import javax.annotation.Nullable; + +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; + +/** + * Interface of a database in a catalog. + * + * @since 1.0 + */ +@Public +public interface Database { + + /** A name to identify this database. */ + String name(); + + /** Options of this database. */ + Map options(); + + /** Optional comment of this database. */ + Optional comment(); + + static Database of(String name, Map options, @Nullable String comment) { + return new DatabaseImpl(name, options, comment); + } + + static Database of(String name) { + return new DatabaseImpl(name, new HashMap<>(), null); + } + + /** Implementation of {@link Database}. */ + class DatabaseImpl implements Database { + + private final String name; + private final Map options; + @Nullable private final String comment; + + public DatabaseImpl(String name, Map options, @Nullable String comment) { + this.name = name; + this.options = options; + this.comment = comment; + } + + @Override + public String name() { + return name; + } + + @Override + public Map options() { + return options; + } + + @Override + public Optional comment() { + return Optional.ofNullable(comment); + } + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/catalog/DelegateCatalog.java b/paimon-core/src/main/java/org/apache/paimon/catalog/DelegateCatalog.java index c2e36dea3065..ec14d53a2b03 100644 --- a/paimon-core/src/main/java/org/apache/paimon/catalog/DelegateCatalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/catalog/DelegateCatalog.java @@ -20,14 +20,14 @@ import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.Path; -import org.apache.paimon.metastore.MetastoreClient; +import org.apache.paimon.manifest.PartitionEntry; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaChange; import org.apache.paimon.table.Table; +import org.apache.paimon.view.View; import java.util.List; import java.util.Map; -import java.util.Optional; /** A {@link Catalog} to delegate all operations to another {@link Catalog}. */ public class DelegateCatalog implements Catalog { @@ -62,21 +62,6 @@ public FileIO fileIO() { return wrapped.fileIO(); } - @Override - public Optional lockFactory() { - return wrapped.lockFactory(); - } - - @Override - public Optional lockContext() { - return wrapped.lockContext(); - } - - @Override - public Optional metastoreClientFactory(Identifier identifier) { - return wrapped.metastoreClientFactory(identifier); - } - @Override public List listDatabases() { return wrapped.listDatabases(); @@ -89,9 +74,8 @@ public void createDatabase(String name, boolean ignoreIfExists, Map loadDatabaseProperties(String name) - throws DatabaseNotExistException { - return wrapped.loadDatabaseProperties(name); + public Database getDatabase(String name) throws DatabaseNotExistException { + return wrapped.getDatabase(name); } @Override @@ -135,17 +119,57 @@ public Table getTable(Identifier identifier) throws TableNotExistException { return wrapped.getTable(identifier); } + @Override + public View getView(Identifier identifier) throws ViewNotExistException { + return wrapped.getView(identifier); + } + + @Override + public void dropView(Identifier identifier, boolean ignoreIfNotExists) + throws ViewNotExistException { + wrapped.dropView(identifier, ignoreIfNotExists); + } + + @Override + public void createView(Identifier identifier, View view, boolean ignoreIfExists) + throws ViewAlreadyExistException, DatabaseNotExistException { + wrapped.createView(identifier, view, ignoreIfExists); + } + + @Override + public List listViews(String databaseName) throws DatabaseNotExistException { + return wrapped.listViews(databaseName); + } + + @Override + public void renameView(Identifier fromView, Identifier toView, boolean ignoreIfNotExists) + throws ViewNotExistException, ViewAlreadyExistException { + wrapped.renameView(fromView, toView, ignoreIfNotExists); + } + @Override public Path getTableLocation(Identifier identifier) { return wrapped.getTableLocation(identifier); } + @Override + public void createPartition(Identifier identifier, Map partitions) + throws TableNotExistException { + wrapped.createPartition(identifier, partitions); + } + @Override public void dropPartition(Identifier identifier, Map partitions) throws TableNotExistException, PartitionNotExistException { wrapped.dropPartition(identifier, partitions); } + @Override + public List listPartitions(Identifier identifier) + throws TableNotExistException { + return wrapped.listPartitions(identifier); + } + @Override public void repairCatalog() { wrapped.repairCatalog(); diff --git a/paimon-core/src/main/java/org/apache/paimon/catalog/FileSystemCatalog.java b/paimon-core/src/main/java/org/apache/paimon/catalog/FileSystemCatalog.java index 14b4d171835c..9264a54647b1 100644 --- a/paimon-core/src/main/java/org/apache/paimon/catalog/FileSystemCatalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/catalog/FileSystemCatalog.java @@ -30,7 +30,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.Collections; import java.util.List; import java.util.Map; import java.util.concurrent.Callable; @@ -70,16 +69,22 @@ protected void createDatabaseImpl(String name, Map properties) { "Currently filesystem catalog can't store database properties, discard properties: {}", properties); } - uncheck(() -> fileIO.mkdirs(newDatabasePath(name))); + + Path databasePath = newDatabasePath(name); + if (!uncheck(() -> fileIO.mkdirs(databasePath))) { + throw new RuntimeException( + String.format( + "Create database location failed, " + "database: %s, location: %s", + name, databasePath)); + } } @Override - public Map loadDatabasePropertiesImpl(String name) - throws DatabaseNotExistException { + public Database getDatabaseImpl(String name) throws DatabaseNotExistException { if (!uncheck(() -> fileIO.exists(newDatabasePath(name)))) { throw new DatabaseNotExistException(name); } - return Collections.emptyMap(); + return Database.of(name); } @Override @@ -92,16 +97,6 @@ protected List listTablesImpl(String databaseName) { return uncheck(() -> listTablesInFileSystem(newDatabasePath(databaseName))); } - @Override - public boolean tableExists(Identifier identifier) { - if (isSystemTable(identifier)) { - return super.tableExists(identifier); - } - - return tableExistsInFileSystem( - getTableLocation(identifier), identifier.getBranchNameOrDefault()); - } - @Override public TableSchema getDataTableSchema(Identifier identifier) throws TableNotExistException { return tableSchemaInFileSystem( diff --git a/paimon-core/src/main/java/org/apache/paimon/catalog/FileSystemCatalogOptions.java b/paimon-core/src/main/java/org/apache/paimon/catalog/FileSystemCatalogOptions.java index 962b249bac67..e656742b42e9 100644 --- a/paimon-core/src/main/java/org/apache/paimon/catalog/FileSystemCatalogOptions.java +++ b/paimon-core/src/main/java/org/apache/paimon/catalog/FileSystemCatalogOptions.java @@ -28,6 +28,7 @@ public final class FileSystemCatalogOptions { ConfigOptions.key("case-sensitive") .booleanType() .defaultValue(true) + .withFallbackKeys("allow-upper-case") .withDescription( "Is case sensitive. If case insensitive, you need to set this option to false, and the table name and fields be converted to lowercase."); diff --git a/paimon-core/src/main/java/org/apache/paimon/catalog/Identifier.java b/paimon-core/src/main/java/org/apache/paimon/catalog/Identifier.java index 72da69b67b83..01456f0b3ae1 100644 --- a/paimon-core/src/main/java/org/apache/paimon/catalog/Identifier.java +++ b/paimon-core/src/main/java/org/apache/paimon/catalog/Identifier.java @@ -65,6 +65,10 @@ public Identifier(String database, String object) { this.object = object; } + public Identifier(String database, String table, @Nullable String branch) { + this(database, table, branch, null); + } + public Identifier( String database, String table, @Nullable String branch, @Nullable String systemTable) { this.database = database; @@ -119,6 +123,10 @@ public String getBranchNameOrDefault() { return systemTable; } + public boolean isSystemTable() { + return getSystemTableName() != null; + } + private void splitObjectName() { if (table != null) { return; diff --git a/paimon-core/src/main/java/org/apache/paimon/codegen/CodeGenUtils.java b/paimon-core/src/main/java/org/apache/paimon/codegen/CodeGenUtils.java index 76aeae54732b..18f8e628f20d 100644 --- a/paimon-core/src/main/java/org/apache/paimon/codegen/CodeGenUtils.java +++ b/paimon-core/src/main/java/org/apache/paimon/codegen/CodeGenUtils.java @@ -70,16 +70,20 @@ public static NormalizedKeyComputer newNormalizedKeyComputer( } public static RecordComparator newRecordComparator(List inputTypes) { - return newRecordComparator(inputTypes, IntStream.range(0, inputTypes.size()).toArray()); + return newRecordComparator( + inputTypes, IntStream.range(0, inputTypes.size()).toArray(), true); } public static RecordComparator newRecordComparator( - List inputTypes, int[] sortFields) { + List inputTypes, int[] sortFields, boolean isAscendingOrder) { return generate( RecordComparator.class, inputTypes, sortFields, - () -> getCodeGenerator().generateRecordComparator(inputTypes, sortFields)); + () -> + getCodeGenerator() + .generateRecordComparator( + inputTypes, sortFields, isAscendingOrder)); } public static RecordEqualiser newRecordEqualiser(List fieldTypes) { @@ -103,14 +107,7 @@ private static T generate( try { Pair, Object[]> result = - COMPILED_CLASS_CACHE.get( - classKey, - () -> { - GeneratedClass generatedClass = supplier.get(); - return Pair.of( - generatedClass.compile(CodeGenUtils.class.getClassLoader()), - generatedClass.getReferences()); - }); + COMPILED_CLASS_CACHE.get(classKey, () -> generateClass(supplier)); //noinspection unchecked return (T) GeneratedClass.newInstance(result.getLeft(), result.getRight()); @@ -120,6 +117,34 @@ private static T generate( } } + private static Pair, Object[]> generateClass( + Supplier> supplier) { + long time = System.currentTimeMillis(); + OutOfMemoryError toThrow; + + do { + try { + GeneratedClass generatedClass = supplier.get(); + return Pair.of( + generatedClass.compile(CodeGenUtils.class.getClassLoader()), + generatedClass.getReferences()); + } catch (OutOfMemoryError error) { + // try to gc meta space + System.gc(); + try { + Thread.sleep(5_000); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw error; + } + toThrow = error; + } + } while ((System.currentTimeMillis() - time) < 120_000); + + // retry fail + throw toThrow; + } + private static class ClassKey { private final Class classType; diff --git a/paimon-core/src/main/java/org/apache/paimon/compact/CompactTask.java b/paimon-core/src/main/java/org/apache/paimon/compact/CompactTask.java index 3e5079c03d73..c8da0f3ef2c5 100644 --- a/paimon-core/src/main/java/org/apache/paimon/compact/CompactTask.java +++ b/paimon-core/src/main/java/org/apache/paimon/compact/CompactTask.java @@ -53,6 +53,17 @@ public CompactResult call() throws Exception { if (metricsReporter != null) { metricsReporter.reportCompactionTime( System.currentTimeMillis() - startMillis); + metricsReporter.increaseCompactionsCompletedCount(); + metricsReporter.reportCompactionInputSize( + result.before().stream() + .map(DataFileMeta::fileSize) + .reduce(Long::sum) + .orElse(0L)); + metricsReporter.reportCompactionOutputSize( + result.after().stream() + .map(DataFileMeta::fileSize) + .reduce(Long::sum) + .orElse(0L)); } }, LOG); @@ -63,6 +74,13 @@ public CompactResult call() throws Exception { return result; } finally { MetricUtils.safeCall(this::stopTimer, LOG); + MetricUtils.safeCall(this::decreaseCompactionsQueuedCount, LOG); + } + } + + private void decreaseCompactionsQueuedCount() { + if (metricsReporter != null) { + metricsReporter.decreaseCompactionsQueuedCount(); } } diff --git a/paimon-core/src/main/java/org/apache/paimon/crosspartition/GlobalIndexAssigner.java b/paimon-core/src/main/java/org/apache/paimon/crosspartition/GlobalIndexAssigner.java index 311b997a4bc8..4700e7399802 100644 --- a/paimon-core/src/main/java/org/apache/paimon/crosspartition/GlobalIndexAssigner.java +++ b/paimon-core/src/main/java/org/apache/paimon/crosspartition/GlobalIndexAssigner.java @@ -306,7 +306,8 @@ private void bulkLoadBootstrapRecords() { coreOptions.pageSize(), coreOptions.localSortMaxNumFileHandles(), coreOptions.spillCompressOptions(), - coreOptions.writeBufferSpillDiskSize()); + coreOptions.writeBufferSpillDiskSize(), + coreOptions.sequenceFieldSortOrderIsAscending()); Function iteratorFunction = sortOrder -> { diff --git a/paimon-core/src/main/java/org/apache/paimon/deletionvectors/ApplyDeletionVectorReader.java b/paimon-core/src/main/java/org/apache/paimon/deletionvectors/ApplyDeletionVectorReader.java index 18ab033fb276..2fc292e54d34 100644 --- a/paimon-core/src/main/java/org/apache/paimon/deletionvectors/ApplyDeletionVectorReader.java +++ b/paimon-core/src/main/java/org/apache/paimon/deletionvectors/ApplyDeletionVectorReader.java @@ -20,23 +20,22 @@ import org.apache.paimon.data.InternalRow; import org.apache.paimon.reader.FileRecordIterator; +import org.apache.paimon.reader.FileRecordReader; import org.apache.paimon.reader.RecordReader; import javax.annotation.Nullable; import java.io.IOException; -import static org.apache.paimon.utils.Preconditions.checkArgument; - /** A {@link RecordReader} which apply {@link DeletionVector} to filter record. */ -public class ApplyDeletionVectorReader implements RecordReader { +public class ApplyDeletionVectorReader implements FileRecordReader { - private final RecordReader reader; + private final FileRecordReader reader; private final DeletionVector deletionVector; public ApplyDeletionVectorReader( - RecordReader reader, DeletionVector deletionVector) { + FileRecordReader reader, DeletionVector deletionVector) { this.reader = reader; this.deletionVector = deletionVector; } @@ -51,19 +50,14 @@ public DeletionVector deletionVector() { @Nullable @Override - public RecordIterator readBatch() throws IOException { - RecordIterator batch = reader.readBatch(); + public FileRecordIterator readBatch() throws IOException { + FileRecordIterator batch = reader.readBatch(); if (batch == null) { return null; } - checkArgument( - batch instanceof FileRecordIterator, - "There is a bug, RecordIterator in ApplyDeletionVectorReader must be RecordWithPositionIterator"); - - return new ApplyDeletionFileRecordIterator( - (FileRecordIterator) batch, deletionVector); + return new ApplyDeletionFileRecordIterator(batch, deletionVector); } @Override diff --git a/paimon-core/src/main/java/org/apache/paimon/deletionvectors/BitmapDeletionVector.java b/paimon-core/src/main/java/org/apache/paimon/deletionvectors/BitmapDeletionVector.java index a2c592596646..51ae729c2193 100644 --- a/paimon-core/src/main/java/org/apache/paimon/deletionvectors/BitmapDeletionVector.java +++ b/paimon-core/src/main/java/org/apache/paimon/deletionvectors/BitmapDeletionVector.java @@ -117,4 +117,9 @@ public boolean equals(Object o) { BitmapDeletionVector that = (BitmapDeletionVector) o; return Objects.equals(this.roaringBitmap, that.roaringBitmap); } + + @Override + public int hashCode() { + return Objects.hashCode(roaringBitmap); + } } diff --git a/paimon-core/src/main/java/org/apache/paimon/deletionvectors/DeletionVectorIndexFileWriter.java b/paimon-core/src/main/java/org/apache/paimon/deletionvectors/DeletionVectorIndexFileWriter.java index f8c8330f190c..5246d35d4b31 100644 --- a/paimon-core/src/main/java/org/apache/paimon/deletionvectors/DeletionVectorIndexFileWriter.java +++ b/paimon-core/src/main/java/org/apache/paimon/deletionvectors/DeletionVectorIndexFileWriter.java @@ -20,9 +20,9 @@ import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.Path; +import org.apache.paimon.index.DeletionVectorMeta; import org.apache.paimon.index.IndexFileMeta; import org.apache.paimon.options.MemorySize; -import org.apache.paimon.utils.Pair; import org.apache.paimon.utils.PathFactory; import org.apache.paimon.utils.Preconditions; @@ -104,13 +104,13 @@ private class SingleIndexFileWriter implements Closeable { private final Path path; private final DataOutputStream dataOutputStream; - private final LinkedHashMap> dvRanges; + private final LinkedHashMap dvMetas; private SingleIndexFileWriter() throws IOException { this.path = indexPathFactory.newPath(); this.dataOutputStream = new DataOutputStream(fileIO.newOutputStream(path, true)); dataOutputStream.writeByte(VERSION_ID_V1); - this.dvRanges = new LinkedHashMap<>(); + this.dvMetas = new LinkedHashMap<>(); } private long writtenSizeInBytes() { @@ -121,7 +121,10 @@ private void write(String key, DeletionVector deletionVector) throws IOException Preconditions.checkNotNull(dataOutputStream); byte[] data = deletionVector.serializeToBytes(); int size = data.length; - dvRanges.put(key, Pair.of(dataOutputStream.size(), size)); + dvMetas.put( + key, + new DeletionVectorMeta( + key, dataOutputStream.size(), size, deletionVector.getCardinality())); dataOutputStream.writeInt(size); dataOutputStream.write(data); dataOutputStream.writeInt(calculateChecksum(data)); @@ -132,8 +135,8 @@ public IndexFileMeta writtenIndexFile() { DELETION_VECTORS_INDEX, path.getName(), writtenSizeInBytes(), - dvRanges.size(), - dvRanges); + dvMetas.size(), + dvMetas); } @Override diff --git a/paimon-core/src/main/java/org/apache/paimon/deletionvectors/DeletionVectorsIndexFile.java b/paimon-core/src/main/java/org/apache/paimon/deletionvectors/DeletionVectorsIndexFile.java index 798404e001e5..77abb2d72985 100644 --- a/paimon-core/src/main/java/org/apache/paimon/deletionvectors/DeletionVectorsIndexFile.java +++ b/paimon-core/src/main/java/org/apache/paimon/deletionvectors/DeletionVectorsIndexFile.java @@ -21,11 +21,11 @@ import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.Path; import org.apache.paimon.fs.SeekableInputStream; +import org.apache.paimon.index.DeletionVectorMeta; import org.apache.paimon.index.IndexFile; import org.apache.paimon.index.IndexFileMeta; import org.apache.paimon.options.MemorySize; import org.apache.paimon.table.source.DeletionFile; -import org.apache.paimon.utils.Pair; import org.apache.paimon.utils.PathFactory; import java.io.DataInputStream; @@ -63,9 +63,9 @@ public DeletionVectorsIndexFile( * @throws UncheckedIOException If an I/O error occurs while reading from the file. */ public Map readAllDeletionVectors(IndexFileMeta fileMeta) { - LinkedHashMap> deletionVectorRanges = - fileMeta.deletionVectorsRanges(); - checkNotNull(deletionVectorRanges); + LinkedHashMap deletionVectorMetas = + fileMeta.deletionVectorMetas(); + checkNotNull(deletionVectorMetas); String indexFileName = fileMeta.fileName(); Map deletionVectors = new HashMap<>(); @@ -73,18 +73,17 @@ public Map readAllDeletionVectors(IndexFileMeta fileMeta try (SeekableInputStream inputStream = fileIO.newInputStream(filePath)) { checkVersion(inputStream); DataInputStream dataInputStream = new DataInputStream(inputStream); - for (Map.Entry> entry : - deletionVectorRanges.entrySet()) { + for (DeletionVectorMeta deletionVectorMeta : deletionVectorMetas.values()) { deletionVectors.put( - entry.getKey(), - readDeletionVector(dataInputStream, entry.getValue().getRight())); + deletionVectorMeta.dataFileName(), + readDeletionVector(dataInputStream, deletionVectorMeta.length())); } } catch (Exception e) { throw new RuntimeException( "Unable to read deletion vectors from file: " + filePath - + ", deletionVectorRanges: " - + deletionVectorRanges, + + ", deletionVectorMetas: " + + deletionVectorMetas, e); } return deletionVectors; diff --git a/paimon-core/src/main/java/org/apache/paimon/disk/FileChannelManagerImpl.java b/paimon-core/src/main/java/org/apache/paimon/disk/FileChannelManagerImpl.java index ce175e90bbd1..99690d426fb5 100644 --- a/paimon-core/src/main/java/org/apache/paimon/disk/FileChannelManagerImpl.java +++ b/paimon-core/src/main/java/org/apache/paimon/disk/FileChannelManagerImpl.java @@ -29,7 +29,9 @@ import java.io.File; import java.io.IOException; import java.io.UncheckedIOException; +import java.util.ArrayList; import java.util.Arrays; +import java.util.List; import java.util.Random; import java.util.UUID; import java.util.concurrent.atomic.AtomicLong; @@ -63,24 +65,32 @@ public FileChannelManagerImpl(String[] tempDirs, String prefix) { } private static File[] createFiles(String[] tempDirs, String prefix) { - File[] files = new File[tempDirs.length]; + List filesList = new ArrayList<>(); for (int i = 0; i < tempDirs.length; i++) { File baseDir = new File(tempDirs[i]); String subfolder = String.format("paimon-%s-%s", prefix, UUID.randomUUID()); File storageDir = new File(baseDir, subfolder); if (!storageDir.exists() && !storageDir.mkdirs()) { - throw new RuntimeException( - "Could not create storage directory for FileChannelManager: " - + storageDir.getAbsolutePath()); + LOG.warn( + "Failed to create directory {}, temp directory {} will not be used", + storageDir.getAbsolutePath(), + tempDirs[i]); + continue; } - files[i] = storageDir; + + filesList.add(storageDir); LOG.debug( "FileChannelManager uses directory {} for spill files.", storageDir.getAbsolutePath()); } - return files; + + if (filesList.isEmpty()) { + throw new RuntimeException("No available temporary directories"); + } + + return filesList.toArray(new File[0]); } @Override diff --git a/paimon-core/src/main/java/org/apache/paimon/hash/BytesHashMap.java b/paimon-core/src/main/java/org/apache/paimon/hash/BytesHashMap.java new file mode 100644 index 000000000000..d739289e6bee --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/hash/BytesHashMap.java @@ -0,0 +1,362 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.hash; + +import org.apache.paimon.annotation.VisibleForTesting; +import org.apache.paimon.data.AbstractPagedInputView; +import org.apache.paimon.data.BinaryRow; +import org.apache.paimon.data.RandomAccessInputView; +import org.apache.paimon.data.SimpleCollectingOutputView; +import org.apache.paimon.data.serializer.BinaryRowSerializer; +import org.apache.paimon.data.serializer.PagedTypeSerializer; +import org.apache.paimon.memory.MemorySegment; +import org.apache.paimon.memory.MemorySegmentPool; +import org.apache.paimon.types.DataType; +import org.apache.paimon.utils.KeyValueIterator; +import org.apache.paimon.utils.MathUtils; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.EOFException; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/** + * Bytes based hash map. It can be used for performing aggregations where the aggregated values are + * fixed-width, because the data is stored in continuous memory, AggBuffer of variable length cannot + * be applied to this HashMap. The KeyValue form in hash map is designed to reduce the cost of key + * fetching in lookup. The memory is divided into two areas: + * + *

    Bucket area: pointer + hashcode. + * + *

      + *
    • Bytes 0 to 4: a pointer to the record in the record area + *
    • Bytes 4 to 8: key's full 32-bit hashcode + *
    + * + *

    Record area: the actual data in linked list records, a record has four parts: + * + *

      + *
    • Bytes 0 to 4: len(k) + *
    • Bytes 4 to 4 + len(k): key data + *
    • Bytes 4 + len(k) to 8 + len(k): len(v) + *
    • Bytes 8 + len(k) to 8 + len(k) + len(v): value data + *
    + * + *

    {@code BytesHashMap} are influenced by Apache Spark BytesToBytesMap. + */ +public class BytesHashMap extends BytesMap { + + private static final Logger LOG = LoggerFactory.getLogger(BytesHashMap.class); + + /** + * Set true when valueTypeInfos.length == 0. Usually in this case the BytesHashMap will be used + * as a HashSet. The value from {@link BytesHashMap#append(LookupInfo info, BinaryRow value)} + * will be ignored when hashSetMode set. The reusedValue will always point to a 16 bytes long + * MemorySegment acted as each BytesHashMap entry's value part when appended to make the + * BytesHashMap's spilling work compatible. + */ + private final boolean hashSetMode; + + /** Used to serialize map key into RecordArea's MemorySegments. */ + protected final PagedTypeSerializer keySerializer; + + /** Used to serialize hash map value into RecordArea's MemorySegments. */ + private final BinaryRowSerializer valueSerializer; + + private volatile RecordArea.EntryIterator destructiveIterator = null; + + public BytesHashMap( + MemorySegmentPool memoryPool, PagedTypeSerializer keySerializer, int valueArity) { + super(memoryPool, keySerializer); + + this.recordArea = new RecordArea(); + + this.keySerializer = keySerializer; + this.valueSerializer = new BinaryRowSerializer(valueArity); + if (valueArity == 0) { + this.hashSetMode = true; + this.reusedValue = new BinaryRow(0); + this.reusedValue.pointTo(MemorySegment.wrap(new byte[8]), 0, 8); + LOG.info("BytesHashMap with hashSetMode = true."); + } else { + this.hashSetMode = false; + this.reusedValue = this.valueSerializer.createInstance(); + } + + final int initBucketSegmentNum = + MathUtils.roundDownToPowerOf2((int) (INIT_BUCKET_MEMORY_IN_BYTES / segmentSize)); + + // allocate and initialize MemorySegments for bucket area + initBucketSegments(initBucketSegmentNum); + + LOG.info( + "BytesHashMap with initial memory segments {}, {} in bytes, init allocating {} for bucket area.", + reservedNumBuffers, + reservedNumBuffers * segmentSize, + initBucketSegmentNum); + } + + // ----------------------- Abstract Interface ----------------------- + + @Override + public long getNumKeys() { + return numElements; + } + + // ----------------------- Public interface ----------------------- + + /** + * Append an value into the hash map's record area. + * + * @return An BinaryRow mapping to the memory segments in the map's record area belonging to the + * newly appended value. + * @throws EOFException if the map can't allocate much more memory. + */ + public BinaryRow append(LookupInfo lookupInfo, BinaryRow value) + throws IOException { + try { + if (numElements >= growthThreshold) { + growAndRehash(); + // update info's bucketSegmentIndex and bucketOffset + lookup(lookupInfo.key); + } + BinaryRow toAppend = hashSetMode ? reusedValue : value; + int pointerToAppended = recordArea.appendRecord(lookupInfo, toAppend); + bucketSegments + .get(lookupInfo.bucketSegmentIndex) + .putInt(lookupInfo.bucketOffset, pointerToAppended); + bucketSegments + .get(lookupInfo.bucketSegmentIndex) + .putInt(lookupInfo.bucketOffset + ELEMENT_POINT_LENGTH, lookupInfo.keyHashCode); + numElements++; + recordArea.setReadPosition(pointerToAppended); + ((RecordArea) recordArea).skipKey(); + return recordArea.readValue(reusedValue); + } catch (EOFException e) { + numSpillFiles++; + spillInBytes += recordArea.getSegmentsSize(); + throw e; + } + } + + public long getNumSpillFiles() { + return numSpillFiles; + } + + public long getUsedMemoryInBytes() { + return bucketSegments.size() * ((long) segmentSize) + recordArea.getSegmentsSize(); + } + + public long getSpillInBytes() { + return spillInBytes; + } + + public int getNumElements() { + return numElements; + } + + /** Returns an iterator for iterating over the entries of this map. */ + @SuppressWarnings("WeakerAccess") + public KeyValueIterator getEntryIterator(boolean requiresCopy) { + if (destructiveIterator != null) { + throw new IllegalArgumentException( + "DestructiveIterator is not null, so this method can't be invoke!"); + } + return ((RecordArea) recordArea).entryIterator(requiresCopy); + } + + /** @return the underlying memory segments of the hash map's record area */ + @SuppressWarnings("WeakerAccess") + public ArrayList getRecordAreaMemorySegments() { + return ((RecordArea) recordArea).segments; + } + + @SuppressWarnings("WeakerAccess") + public List getBucketAreaMemorySegments() { + return bucketSegments; + } + + public void free() { + recordArea.release(); + destructiveIterator = null; + super.free(); + } + + /** reset the map's record and bucket area's memory segments for reusing. */ + public void reset() { + // reset the record segments. + recordArea.reset(); + destructiveIterator = null; + super.reset(); + } + + /** + * @return true when BytesHashMap's valueTypeInfos.length == 0. Any appended value will be + * ignored and replaced with a reusedValue as a present tag. + */ + @VisibleForTesting + boolean isHashSetMode() { + return hashSetMode; + } + + // ----------------------- Private methods ----------------------- + + static int getVariableLength(DataType[] types) { + int length = 0; + for (DataType type : types) { + if (!BinaryRow.isInFixedLengthPart(type)) { + // find a better way of computing generic type field variable-length + // right now we use a small value assumption + length += 16; + } + } + return length; + } + + // ----------------------- Record Area ----------------------- + + private final class RecordArea implements BytesMap.RecordArea { + private final ArrayList segments = new ArrayList<>(); + + private final RandomAccessInputView inView; + private final SimpleCollectingOutputView outView; + + RecordArea() { + this.outView = new SimpleCollectingOutputView(segments, memoryPool, segmentSize); + this.inView = new RandomAccessInputView(segments, segmentSize); + } + + public void release() { + returnSegments(segments); + segments.clear(); + } + + public void reset() { + release(); + // request a new memory segment from freeMemorySegments + // reset segmentNum and positionInSegment + outView.reset(); + inView.setReadPosition(0); + } + + // ----------------------- Append ----------------------- + public int appendRecord(LookupInfo lookupInfo, BinaryRow value) + throws IOException { + final long oldLastPosition = outView.getCurrentOffset(); + // serialize the key into the BytesHashMap record area + int skip = keySerializer.serializeToPages(lookupInfo.getKey(), outView); + long offset = oldLastPosition + skip; + + // serialize the value into the BytesHashMap record area + valueSerializer.serializeToPages(value, outView); + if (offset > Integer.MAX_VALUE) { + LOG.warn( + "We can't handle key area with more than Integer.MAX_VALUE bytes," + + " because the pointer is a integer."); + throw new EOFException(); + } + return (int) offset; + } + + @Override + public long getSegmentsSize() { + return segments.size() * ((long) segmentSize); + } + + // ----------------------- Read ----------------------- + public void setReadPosition(int position) { + inView.setReadPosition(position); + } + + public boolean readKeyAndEquals(K lookupKey) throws IOException { + reusedKey = keySerializer.mapFromPages(reusedKey, inView); + return lookupKey.equals(reusedKey); + } + + /** @throws IOException when invalid memory address visited. */ + void skipKey() throws IOException { + keySerializer.skipRecordFromPages(inView); + } + + public BinaryRow readValue(BinaryRow reuse) throws IOException { + // depends on BinaryRowSerializer to check writing skip + // and to find the real start offset of the data + return valueSerializer.mapFromPages(reuse, inView); + } + + // ----------------------- Iterator ----------------------- + + private KeyValueIterator entryIterator(boolean requiresCopy) { + return new EntryIterator(requiresCopy); + } + + private final class EntryIterator extends AbstractPagedInputView + implements KeyValueIterator { + + private int count = 0; + private int currentSegmentIndex = 0; + private final boolean requiresCopy; + + private EntryIterator(boolean requiresCopy) { + super(segments.get(0), segmentSize); + destructiveIterator = this; + this.requiresCopy = requiresCopy; + } + + @Override + public boolean advanceNext() throws IOException { + if (count < numElements) { + count++; + // segment already is useless any more. + keySerializer.mapFromPages(reusedKey, this); + valueSerializer.mapFromPages(reusedValue, this); + return true; + } + return false; + } + + @Override + public K getKey() { + return requiresCopy ? keySerializer.copy(reusedKey) : reusedKey; + } + + @Override + public BinaryRow getValue() { + return requiresCopy ? reusedValue.copy() : reusedValue; + } + + public boolean hasNext() { + return count < numElements; + } + + @Override + protected int getLimitForSegment(MemorySegment segment) { + return segmentSize; + } + + @Override + protected MemorySegment nextSegment(MemorySegment current) { + return segments.get(++currentSegmentIndex); + } + } + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/hash/BytesMap.java b/paimon-core/src/main/java/org/apache/paimon/hash/BytesMap.java new file mode 100644 index 000000000000..215cfface98e --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/hash/BytesMap.java @@ -0,0 +1,379 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.hash; + +import org.apache.paimon.data.BinaryRow; +import org.apache.paimon.data.serializer.PagedTypeSerializer; +import org.apache.paimon.memory.MemorySegment; +import org.apache.paimon.memory.MemorySegmentPool; +import org.apache.paimon.utils.MathUtils; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.EOFException; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/** + * Base class for {@code BytesHashMap}. + * + * @param type of the map key. + * @param type of the map value. + */ +public abstract class BytesMap { + + private static final Logger LOG = LoggerFactory.getLogger(BytesMap.class); + + public static final int BUCKET_SIZE = 8; + protected static final int END_OF_LIST = Integer.MAX_VALUE; + protected static final int STEP_INCREMENT = 1; + protected static final int ELEMENT_POINT_LENGTH = 4; + public static final int RECORD_EXTRA_LENGTH = 8; + protected static final int BUCKET_SIZE_BITS = 3; + + protected final int numBucketsPerSegment; + protected final int numBucketsPerSegmentBits; + protected final int numBucketsPerSegmentMask; + protected final int lastBucketPosition; + + protected final int segmentSize; + protected final MemorySegmentPool memoryPool; + protected List bucketSegments; + + protected final int reservedNumBuffers; + + protected int numElements = 0; + protected int numBucketsMask; + // get the second hashcode based log2NumBuckets and numBucketsMask2 + protected int log2NumBuckets; + protected int numBucketsMask2; + + protected static final double LOAD_FACTOR = 0.75; + // a smaller bucket can make the best of l1/l2/l3 cache. + protected static final long INIT_BUCKET_MEMORY_IN_BYTES = 1024 * 1024L; + + /** The map will be expanded once the number of elements exceeds this threshold. */ + protected int growthThreshold; + + /** The segments where the actual data is stored. */ + protected RecordArea recordArea; + + /** Used as a reused object when lookup and iteration. */ + protected K reusedKey; + + /** Used as a reused object when retrieve the map's value by key and iteration. */ + protected V reusedValue; + + /** Used as a reused object which lookup returned. */ + private final LookupInfo reuseLookupInfo; + + // metric + protected long numSpillFiles; + protected long spillInBytes; + + public BytesMap(MemorySegmentPool memoryPool, PagedTypeSerializer keySerializer) { + this.memoryPool = memoryPool; + this.segmentSize = memoryPool.pageSize(); + this.reservedNumBuffers = memoryPool.freePages(); + this.numBucketsPerSegment = segmentSize / BUCKET_SIZE; + this.numBucketsPerSegmentBits = MathUtils.log2strict(this.numBucketsPerSegment); + this.numBucketsPerSegmentMask = (1 << this.numBucketsPerSegmentBits) - 1; + this.lastBucketPosition = (numBucketsPerSegment - 1) * BUCKET_SIZE; + + this.reusedKey = keySerializer.createReuseInstance(); + this.reuseLookupInfo = new LookupInfo<>(); + } + + /** Returns the number of keys in this map. */ + public abstract long getNumKeys(); + + protected void initBucketSegments(int numBucketSegments) { + if (numBucketSegments < 1) { + throw new RuntimeException("Too small memory allocated for BytesHashMap"); + } + this.bucketSegments = new ArrayList<>(numBucketSegments); + for (int i = 0; i < numBucketSegments; i++) { + MemorySegment segment = memoryPool.nextSegment(); + if (segment == null) { + throw new RuntimeException("Memory for hash map is too small."); + } + bucketSegments.add(i, segment); + } + + resetBucketSegments(this.bucketSegments); + int numBuckets = numBucketSegments * numBucketsPerSegment; + this.log2NumBuckets = MathUtils.log2strict(numBuckets); + this.numBucketsMask = (1 << MathUtils.log2strict(numBuckets)) - 1; + this.numBucketsMask2 = (1 << MathUtils.log2strict(numBuckets >> 1)) - 1; + this.growthThreshold = (int) (numBuckets * LOAD_FACTOR); + } + + protected void resetBucketSegments(List resetBucketSegs) { + for (MemorySegment segment : resetBucketSegs) { + for (int j = 0; j <= lastBucketPosition; j += BUCKET_SIZE) { + segment.putInt(j, END_OF_LIST); + } + } + } + + public long getNumSpillFiles() { + return numSpillFiles; + } + + public long getSpillInBytes() { + return spillInBytes; + } + + public int getNumElements() { + return numElements; + } + + public void free() { + returnSegments(this.bucketSegments); + this.bucketSegments.clear(); + numElements = 0; + } + + /** reset the map's record and bucket area's memory segments for reusing. */ + public void reset() { + setBucketVariables(bucketSegments); + resetBucketSegments(bucketSegments); + numElements = 0; + LOG.debug( + "reset BytesHashMap with record memory segments {}, {} in bytes, init allocating {} for bucket area.", + memoryPool.freePages(), + memoryPool.freePages() * segmentSize, + bucketSegments.size()); + } + + /** + * @param key by which looking up the value in the hash map. Only support the key in the + * BinaryRowData form who has only one MemorySegment. + * @return {@link LookupInfo} + */ + public LookupInfo lookup(K key) { + final int hashCode1 = key.hashCode(); + int newPos = hashCode1 & numBucketsMask; + // which segment contains the bucket + int bucketSegmentIndex = newPos >>> numBucketsPerSegmentBits; + // offset of the bucket in the segment + int bucketOffset = (newPos & numBucketsPerSegmentMask) << BUCKET_SIZE_BITS; + + boolean found = false; + int step = STEP_INCREMENT; + int hashCode2 = 0; + int findElementPtr; + try { + do { + findElementPtr = bucketSegments.get(bucketSegmentIndex).getInt(bucketOffset); + if (findElementPtr == END_OF_LIST) { + // This is a new key. + break; + } else { + final int storedHashCode = + bucketSegments + .get(bucketSegmentIndex) + .getInt(bucketOffset + ELEMENT_POINT_LENGTH); + if (hashCode1 == storedHashCode) { + recordArea.setReadPosition(findElementPtr); + if (recordArea.readKeyAndEquals(key)) { + // we found an element with a matching key, and not just a hash + // collision + found = true; + reusedValue = recordArea.readValue(reusedValue); + break; + } + } + } + if (step == 1) { + hashCode2 = calcSecondHashCode(hashCode1); + } + newPos = (hashCode1 + step * hashCode2) & numBucketsMask; + // which segment contains the bucket + bucketSegmentIndex = newPos >>> numBucketsPerSegmentBits; + // offset of the bucket in the segment + bucketOffset = (newPos & numBucketsPerSegmentMask) << BUCKET_SIZE_BITS; + step += STEP_INCREMENT; + } while (true); + } catch (IOException ex) { + throw new RuntimeException( + "Error reading record from the aggregate map: " + ex.getMessage(), ex); + } + reuseLookupInfo.set(found, hashCode1, key, reusedValue, bucketSegmentIndex, bucketOffset); + return reuseLookupInfo; + } + + /** @throws EOFException if the map can't allocate much more memory. */ + protected void growAndRehash() throws EOFException { + // allocate the new data structures + int required = 2 * bucketSegments.size(); + if (required * (long) numBucketsPerSegment > Integer.MAX_VALUE) { + LOG.warn( + "We can't handle more than Integer.MAX_VALUE buckets (eg. because hash functions return int)"); + throw new EOFException(); + } + + int numAllocatedSegments = required - memoryPool.freePages(); + if (numAllocatedSegments > 0) { + LOG.warn( + "BytesHashMap can't allocate {} pages, and now used {} pages", + required, + reservedNumBuffers); + throw new EOFException(); + } + + List newBucketSegments = new ArrayList<>(required); + for (int i = 0; i < required; i++) { + newBucketSegments.add(memoryPool.nextSegment()); + } + setBucketVariables(newBucketSegments); + + long reHashStartTime = System.currentTimeMillis(); + resetBucketSegments(newBucketSegments); + // Re-mask (we don't recompute the hashcode because we stored all 32 bits of it) + for (MemorySegment memorySegment : bucketSegments) { + for (int j = 0; j < numBucketsPerSegment; j++) { + final int recordPointer = memorySegment.getInt(j * BUCKET_SIZE); + if (recordPointer != END_OF_LIST) { + final int hashCode1 = + memorySegment.getInt(j * BUCKET_SIZE + ELEMENT_POINT_LENGTH); + int newPos = hashCode1 & numBucketsMask; + int bucketSegmentIndex = newPos >>> numBucketsPerSegmentBits; + int bucketOffset = (newPos & numBucketsPerSegmentMask) << BUCKET_SIZE_BITS; + int step = STEP_INCREMENT; + long hashCode2 = 0; + while (newBucketSegments.get(bucketSegmentIndex).getInt(bucketOffset) + != END_OF_LIST) { + if (step == 1) { + hashCode2 = calcSecondHashCode(hashCode1); + } + newPos = (int) ((hashCode1 + step * hashCode2) & numBucketsMask); + // which segment contains the bucket + bucketSegmentIndex = newPos >>> numBucketsPerSegmentBits; + // offset of the bucket in the segment + bucketOffset = (newPos & numBucketsPerSegmentMask) << BUCKET_SIZE_BITS; + step += STEP_INCREMENT; + } + newBucketSegments.get(bucketSegmentIndex).putInt(bucketOffset, recordPointer); + newBucketSegments + .get(bucketSegmentIndex) + .putInt(bucketOffset + ELEMENT_POINT_LENGTH, hashCode1); + } + } + } + LOG.info( + "The rehash take {} ms for {} segments", + (System.currentTimeMillis() - reHashStartTime), + required); + this.memoryPool.returnAll(this.bucketSegments); + this.bucketSegments = newBucketSegments; + } + + protected void returnSegments(List segments) { + memoryPool.returnAll(segments); + } + + private void setBucketVariables(List bucketSegments) { + int numBuckets = bucketSegments.size() * numBucketsPerSegment; + this.log2NumBuckets = MathUtils.log2strict(numBuckets); + this.numBucketsMask = (1 << MathUtils.log2strict(numBuckets)) - 1; + this.numBucketsMask2 = (1 << MathUtils.log2strict(numBuckets >> 1)) - 1; + this.growthThreshold = (int) (numBuckets * LOAD_FACTOR); + } + + // M(the num of buckets) is the nth power of 2, so the second hash code must be odd, and always + // is + // H2(K) = 1 + 2 * ((H1(K)/M) mod (M-1)) + protected int calcSecondHashCode(final int firstHashCode) { + return ((((firstHashCode >> log2NumBuckets)) & numBucketsMask2) << 1) + 1; + } + + /** Record area. */ + interface RecordArea { + + void setReadPosition(int position); + + boolean readKeyAndEquals(K lookupKey) throws IOException; + + V readValue(V reuse) throws IOException; + + int appendRecord(LookupInfo lookupInfo, BinaryRow value) throws IOException; + + long getSegmentsSize(); + + void release(); + + void reset(); + } + + /** Result fetched when looking up a key. */ + public static final class LookupInfo { + boolean found; + K key; + V value; + + /** + * The hashcode of the look up key passed to {@link BytesMap#lookup(K)}, Caching this + * hashcode here allows us to avoid re-hashing the key when inserting a value for that key. + * The same purpose with bucketSegmentIndex, bucketOffset. + */ + int keyHashCode; + + int bucketSegmentIndex; + int bucketOffset; + + LookupInfo() { + this.found = false; + this.keyHashCode = -1; + this.key = null; + this.value = null; + this.bucketSegmentIndex = -1; + this.bucketOffset = -1; + } + + void set( + boolean found, + int keyHashCode, + K key, + V value, + int bucketSegmentIndex, + int bucketOffset) { + this.found = found; + this.keyHashCode = keyHashCode; + this.key = key; + this.value = value; + this.bucketSegmentIndex = bucketSegmentIndex; + this.bucketOffset = bucketOffset; + } + + public boolean isFound() { + return found; + } + + public K getKey() { + return key; + } + + public V getValue() { + return value; + } + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/iceberg/AbstractIcebergCommitCallback.java b/paimon-core/src/main/java/org/apache/paimon/iceberg/AbstractIcebergCommitCallback.java index e78ca5818d28..f561546e8bb3 100644 --- a/paimon-core/src/main/java/org/apache/paimon/iceberg/AbstractIcebergCommitCallback.java +++ b/paimon-core/src/main/java/org/apache/paimon/iceberg/AbstractIcebergCommitCallback.java @@ -23,6 +23,8 @@ import org.apache.paimon.data.BinaryRow; import org.apache.paimon.data.GenericArray; import org.apache.paimon.data.GenericRow; +import org.apache.paimon.factories.FactoryException; +import org.apache.paimon.factories.FactoryUtil; import org.apache.paimon.fs.Path; import org.apache.paimon.iceberg.manifest.IcebergConversions; import org.apache.paimon.iceberg.manifest.IcebergDataFileMeta; @@ -31,6 +33,7 @@ import org.apache.paimon.iceberg.manifest.IcebergManifestFileMeta; import org.apache.paimon.iceberg.manifest.IcebergManifestList; import org.apache.paimon.iceberg.manifest.IcebergPartitionSummary; +import org.apache.paimon.iceberg.metadata.IcebergDataField; import org.apache.paimon.iceberg.metadata.IcebergMetadata; import org.apache.paimon.iceberg.metadata.IcebergPartitionField; import org.apache.paimon.iceberg.metadata.IcebergPartitionSpec; @@ -40,19 +43,15 @@ import org.apache.paimon.io.DataFileMeta; import org.apache.paimon.manifest.ManifestCommittable; import org.apache.paimon.manifest.ManifestEntry; -import org.apache.paimon.options.ConfigOption; -import org.apache.paimon.options.ConfigOptions; import org.apache.paimon.options.Options; import org.apache.paimon.partition.PartitionPredicate; import org.apache.paimon.schema.SchemaManager; -import org.apache.paimon.schema.TableSchema; import org.apache.paimon.table.FileStoreTable; import org.apache.paimon.table.sink.CommitCallback; import org.apache.paimon.table.source.DataSplit; import org.apache.paimon.table.source.RawFile; import org.apache.paimon.table.source.ScanMode; import org.apache.paimon.table.source.snapshot.SnapshotReader; -import org.apache.paimon.types.DataField; import org.apache.paimon.types.DataType; import org.apache.paimon.types.RowType; import org.apache.paimon.utils.FileStorePathFactory; @@ -60,6 +59,8 @@ import org.apache.paimon.utils.Pair; import org.apache.paimon.utils.SnapshotManager; +import javax.annotation.Nullable; + import java.io.IOException; import java.io.UncheckedIOException; import java.util.ArrayList; @@ -85,20 +86,13 @@ public abstract class AbstractIcebergCommitCallback implements CommitCallback { // see org.apache.iceberg.hadoop.Util private static final String VERSION_HINT_FILENAME = "version-hint.text"; - static final ConfigOption COMPACT_MIN_FILE_NUM = - ConfigOptions.key("metadata.iceberg.compaction.min.file-num") - .intType() - .defaultValue(10); - static final ConfigOption COMPACT_MAX_FILE_NUM = - ConfigOptions.key("metadata.iceberg.compaction.max.file-num") - .intType() - .defaultValue(50); - protected final FileStoreTable table; private final String commitUser; + private final IcebergPathFactory pathFactory; - private final FileStorePathFactory fileStorePathFactory; + private final @Nullable IcebergMetadataCommitter metadataCommitter; + private final FileStorePathFactory fileStorePathFactory; private final IcebergManifestFile manifestFile; private final IcebergManifestList manifestList; @@ -109,12 +103,58 @@ public abstract class AbstractIcebergCommitCallback implements CommitCallback { public AbstractIcebergCommitCallback(FileStoreTable table, String commitUser) { this.table = table; this.commitUser = commitUser; - this.pathFactory = new IcebergPathFactory(table.location()); + + IcebergOptions.StorageType storageType = + table.coreOptions().toConfiguration().get(IcebergOptions.METADATA_ICEBERG_STORAGE); + switch (storageType) { + case TABLE_LOCATION: + this.pathFactory = new IcebergPathFactory(new Path(table.location(), "metadata")); + break; + case HADOOP_CATALOG: + case HIVE_CATALOG: + this.pathFactory = new IcebergPathFactory(catalogTableMetadataPath(table)); + break; + default: + throw new UnsupportedOperationException( + "Unknown storage type " + storageType.name()); + } + + IcebergMetadataCommitterFactory metadataCommitterFactory; + try { + metadataCommitterFactory = + FactoryUtil.discoverFactory( + AbstractIcebergCommitCallback.class.getClassLoader(), + IcebergMetadataCommitterFactory.class, + storageType.toString()); + } catch (FactoryException ignore) { + metadataCommitterFactory = null; + } + this.metadataCommitter = + metadataCommitterFactory == null ? null : metadataCommitterFactory.create(table); + this.fileStorePathFactory = table.store().pathFactory(); this.manifestFile = IcebergManifestFile.create(table, pathFactory); this.manifestList = IcebergManifestList.create(table, pathFactory); } + public static Path catalogTableMetadataPath(FileStoreTable table) { + Path icebergDBPath = catalogDatabasePath(table); + return new Path(icebergDBPath, String.format("%s/metadata", table.location().getName())); + } + + public static Path catalogDatabasePath(FileStoreTable table) { + Path dbPath = table.location().getParent(); + final String dbSuffix = ".db"; + if (dbPath.getName().endsWith(dbSuffix)) { + String dbName = + dbPath.getName().substring(0, dbPath.getName().length() - dbSuffix.length()); + return new Path(dbPath.getParent(), String.format("iceberg/%s/", dbName)); + } else { + throw new UnsupportedOperationException( + "Storage type ICEBERG_WAREHOUSE can only be used on Paimon tables in a Paimon warehouse."); + } + } + @Override public void close() throws Exception {} @@ -152,6 +192,13 @@ public void retry(ManifestCommittable committable) { private void createMetadata(long snapshotId, FileChangesCollector fileChangesCollector) { try { + if (snapshotId == Snapshot.FIRST_SNAPSHOT_ID) { + // If Iceberg metadata is stored separately in another directory, dropping the table + // will not delete old Iceberg metadata. So we delete them here, when the table is + // created again and the first snapshot is committed. + table.fileIO().delete(pathFactory.metadataDirectory(), true); + } + if (table.fileIO().exists(pathFactory.toMetadataPath(snapshotId))) { return; } @@ -173,19 +220,23 @@ private void createMetadata(long snapshotId, FileChangesCollector fileChangesCol private void createMetadataWithoutBase(long snapshotId) throws IOException { SnapshotReader snapshotReader = table.newSnapshotReader().withSnapshot(snapshotId); - SchemaCache schemas = new SchemaCache(); + SchemaCache schemaCache = new SchemaCache(); Iterator entryIterator = snapshotReader.read().dataSplits().stream() .filter(DataSplit::rawConvertible) - .flatMap(s -> dataSplitToManifestEntries(s, snapshotId, schemas).stream()) + .flatMap( + s -> + dataSplitToManifestEntries(s, snapshotId, schemaCache) + .stream()) .iterator(); List manifestFileMetas = manifestFile.rollingWrite(entryIterator, snapshotId); String manifestListFileName = manifestList.writeWithoutRolling(manifestFileMetas); - List partitionFields = - getPartitionFields(table.schema().logicalPartitionType()); int schemaId = (int) table.schema().id(); + IcebergSchema icebergSchema = schemaCache.get(schemaId); + List partitionFields = + getPartitionFields(table.schema().partitionKeys(), icebergSchema); IcebergSnapshot snapshot = new IcebergSnapshot( snapshotId, @@ -201,8 +252,8 @@ private void createMetadataWithoutBase(long snapshotId) throws IOException { tableUuid, table.location().toString(), snapshotId, - table.schema().highestFieldId(), - Collections.singletonList(new IcebergSchema(table.schema())), + icebergSchema.highestFieldId(), + Collections.singletonList(icebergSchema), schemaId, Collections.singletonList(new IcebergPartitionSpec(partitionFields)), partitionFields.stream() @@ -213,17 +264,23 @@ private void createMetadataWithoutBase(long snapshotId) throws IOException { IcebergPartitionField.FIRST_FIELD_ID - 1), Collections.singletonList(snapshot), (int) snapshotId); - table.fileIO().tryToWriteAtomic(pathFactory.toMetadataPath(snapshotId), metadata.toJson()); + + Path metadataPath = pathFactory.toMetadataPath(snapshotId); + table.fileIO().tryToWriteAtomic(metadataPath, metadata.toJson()); table.fileIO() .overwriteFileUtf8( new Path(pathFactory.metadataDirectory(), VERSION_HINT_FILENAME), String.valueOf(snapshotId)); expireAllBefore(snapshotId); + + if (metadataCommitter != null) { + metadataCommitter.commitMetadata(metadataPath, null); + } } private List dataSplitToManifestEntries( - DataSplit dataSplit, long snapshotId, SchemaCache schemas) { + DataSplit dataSplit, long snapshotId, SchemaCache schemaCache) { List result = new ArrayList<>(); List rawFiles = dataSplit.convertToRawFiles().get(); for (int i = 0; i < dataSplit.dataFiles().size(); i++) { @@ -237,8 +294,9 @@ private List dataSplitToManifestEntries( dataSplit.partition(), rawFile.rowCount(), rawFile.fileSize(), - schemas.get(paimonFileMeta.schemaId()), - paimonFileMeta.valueStats()); + schemaCache.get(paimonFileMeta.schemaId()), + paimonFileMeta.valueStats(), + paimonFileMeta.valueStatsCols()); result.add( new IcebergManifestEntry( IcebergManifestEntry.Status.ADDED, @@ -250,11 +308,17 @@ private List dataSplitToManifestEntries( return result; } - private List getPartitionFields(RowType partitionType) { + private List getPartitionFields( + List partitionKeys, IcebergSchema icebergSchema) { + Map fields = new HashMap<>(); + for (IcebergDataField field : icebergSchema.fields()) { + fields.put(field.name(), field); + } + List result = new ArrayList<>(); int fieldId = IcebergPartitionField.FIRST_FIELD_ID; - for (DataField field : partitionType.getFields()) { - result.add(new IcebergPartitionField(field, fieldId)); + for (String partitionKey : partitionKeys) { + result.add(new IcebergPartitionField(fields.get(partitionKey), fieldId)); fieldId++; } return result; @@ -307,11 +371,13 @@ private void createMetadataWithBase( compactMetadataIfNeeded(newManifestFileMetas, snapshotId)); // add new schema if needed + SchemaCache schemaCache = new SchemaCache(); int schemaId = (int) table.schema().id(); + IcebergSchema icebergSchema = schemaCache.get(schemaId); List schemas = baseMetadata.schemas(); if (baseMetadata.currentSchemaId() != schemaId) { schemas = new ArrayList<>(schemas); - schemas.add(new IcebergSchema(table.schema())); + schemas.add(icebergSchema); } List snapshots = new ArrayList<>(baseMetadata.snapshots()); @@ -341,14 +407,16 @@ private void createMetadataWithBase( baseMetadata.tableUuid(), baseMetadata.location(), snapshotId, - table.schema().highestFieldId(), + icebergSchema.highestFieldId(), schemas, schemaId, baseMetadata.partitionSpecs(), baseMetadata.lastPartitionId(), snapshots, (int) snapshotId); - table.fileIO().tryToWriteAtomic(pathFactory.toMetadataPath(snapshotId), metadata.toJson()); + + Path metadataPath = pathFactory.toMetadataPath(snapshotId); + table.fileIO().tryToWriteAtomic(metadataPath, metadata.toJson()); table.fileIO() .overwriteFileUtf8( new Path(pathFactory.metadataDirectory(), VERSION_HINT_FILENAME), @@ -360,6 +428,10 @@ private void createMetadataWithBase( new Path(toExpireExceptLast.get(i).manifestList()).getName(), new Path(toExpireExceptLast.get(i + 1).manifestList()).getName()); } + + if (metadataCommitter != null) { + metadataCommitter.commitMetadata(metadataPath, baseMetadataPath); + } } private interface FileChangesCollector { @@ -423,7 +495,7 @@ private List createNewlyAddedManifestFileMetas( return Collections.emptyList(); } - SchemaCache schemas = new SchemaCache(); + SchemaCache schemaCache = new SchemaCache(); return manifestFile.rollingWrite( addedFiles.entrySet().stream() .map( @@ -437,8 +509,9 @@ private List createNewlyAddedManifestFileMetas( e.getValue().getLeft(), paimonFileMeta.rowCount(), paimonFileMeta.fileSize(), - schemas.get(paimonFileMeta.schemaId()), - paimonFileMeta.valueStats()); + schemaCache.get(paimonFileMeta.schemaId()), + paimonFileMeta.valueStats(), + paimonFileMeta.valueStatsCols()); return new IcebergManifestEntry( IcebergManifestEntry.Status.ADDED, currentSnapshotId, @@ -562,10 +635,10 @@ private List compactMetadataIfNeeded( } Options options = new Options(table.options()); - if (candidates.size() < options.get(COMPACT_MIN_FILE_NUM)) { + if (candidates.size() < options.get(IcebergOptions.COMPACT_MIN_FILE_NUM)) { return toCompact; } - if (candidates.size() < options.get(COMPACT_MAX_FILE_NUM) + if (candidates.size() < options.get(IcebergOptions.COMPACT_MAX_FILE_NUM) && totalSizeInBytes < targetSizeInBytes) { return toCompact; } @@ -681,10 +754,11 @@ private void expireAllBefore(long snapshotId) throws IOException { private class SchemaCache { SchemaManager schemaManager = new SchemaManager(table.fileIO(), table.location()); - Map tableSchemas = new HashMap<>(); + Map schemas = new HashMap<>(); - private TableSchema get(long schemaId) { - return tableSchemas.computeIfAbsent(schemaId, id -> schemaManager.schema(id)); + private IcebergSchema get(long schemaId) { + return schemas.computeIfAbsent( + schemaId, id -> IcebergSchema.create(schemaManager.schema(id))); } } } diff --git a/paimon-common/src/main/java/org/apache/paimon/lineage/DataLineageEntity.java b/paimon-core/src/main/java/org/apache/paimon/iceberg/IcebergMetadataCommitter.java similarity index 70% rename from paimon-common/src/main/java/org/apache/paimon/lineage/DataLineageEntity.java rename to paimon-core/src/main/java/org/apache/paimon/iceberg/IcebergMetadataCommitter.java index e7401a9be3b7..2fe131498358 100644 --- a/paimon-common/src/main/java/org/apache/paimon/lineage/DataLineageEntity.java +++ b/paimon-core/src/main/java/org/apache/paimon/iceberg/IcebergMetadataCommitter.java @@ -16,18 +16,17 @@ * limitations under the License. */ -package org.apache.paimon.lineage; +package org.apache.paimon.iceberg; -import org.apache.paimon.data.Timestamp; +import org.apache.paimon.fs.Path; + +import javax.annotation.Nullable; /** - * Data lineage entity with table lineage, barrier id and snapshot id for table source and sink - * lineage. + * Commit Iceberg metadata to metastore. Each kind of Iceberg catalog should have its own + * implementation. */ -public interface DataLineageEntity extends TableLineageEntity { - long getBarrierId(); - - long getSnapshotId(); +public interface IcebergMetadataCommitter { - Timestamp getCreateTime(); + void commitMetadata(Path newMetadataPath, @Nullable Path baseMetadataPath); } diff --git a/paimon-core/src/main/java/org/apache/paimon/iceberg/IcebergMetadataCommitterFactory.java b/paimon-core/src/main/java/org/apache/paimon/iceberg/IcebergMetadataCommitterFactory.java new file mode 100644 index 000000000000..dc168ff092b6 --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/iceberg/IcebergMetadataCommitterFactory.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.iceberg; + +import org.apache.paimon.factories.Factory; +import org.apache.paimon.table.FileStoreTable; + +/** Factory to create {@link IcebergMetadataCommitter}. */ +public interface IcebergMetadataCommitterFactory extends Factory { + + IcebergMetadataCommitter create(FileStoreTable table); +} diff --git a/paimon-core/src/main/java/org/apache/paimon/iceberg/IcebergOptions.java b/paimon-core/src/main/java/org/apache/paimon/iceberg/IcebergOptions.java new file mode 100644 index 000000000000..55fbab5158fa --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/iceberg/IcebergOptions.java @@ -0,0 +1,124 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.iceberg; + +import org.apache.paimon.options.ConfigOption; +import org.apache.paimon.options.ConfigOptions; +import org.apache.paimon.options.description.DescribedEnum; +import org.apache.paimon.options.description.InlineElement; +import org.apache.paimon.options.description.TextElement; + +import static org.apache.paimon.options.ConfigOptions.key; + +/** Config options for Paimon Iceberg compatibility. */ +public class IcebergOptions { + + public static final ConfigOption METADATA_ICEBERG_STORAGE = + key("metadata.iceberg.storage") + .enumType(StorageType.class) + .defaultValue(StorageType.DISABLED) + .withDescription( + "When set, produce Iceberg metadata after a snapshot is committed, " + + "so that Iceberg readers can read Paimon's raw data files."); + + public static final ConfigOption COMPACT_MIN_FILE_NUM = + ConfigOptions.key("metadata.iceberg.compaction.min.file-num") + .intType() + .defaultValue(10) + .withDescription( + "Minimum number of Iceberg metadata files to trigger metadata compaction."); + + public static final ConfigOption COMPACT_MAX_FILE_NUM = + ConfigOptions.key("metadata.iceberg.compaction.max.file-num") + .intType() + .defaultValue(50) + .withDescription( + "If number of small Iceberg metadata files exceeds this limit, " + + "always trigger metadata compaction regardless of their total size."); + + public static final ConfigOption URI = + key("metadata.iceberg.uri") + .stringType() + .noDefaultValue() + .withDescription("Hive metastore uri for Iceberg Hive catalog."); + + public static final ConfigOption HIVE_CONF_DIR = + key("metadata.iceberg.hive-conf-dir") + .stringType() + .noDefaultValue() + .withDescription("hive-conf-dir for Iceberg Hive catalog."); + + public static final ConfigOption HADOOP_CONF_DIR = + key("metadata.iceberg.hadoop-conf-dir") + .stringType() + .noDefaultValue() + .withDescription("hadoop-conf-dir for Iceberg Hive catalog."); + + public static final ConfigOption MANIFEST_COMPRESSION = + key("metadata.iceberg.manifest-compression") + .stringType() + .defaultValue( + "snappy") // some Iceberg reader cannot support zstd, for example DuckDB + .withDescription("Compression for Iceberg manifest files."); + + public static final ConfigOption MANIFEST_LEGACY_VERSION = + key("metadata.iceberg.manifest-legacy-version") + .booleanType() + .defaultValue(false) + .withDescription( + "Should use the legacy manifest version to generate Iceberg's 1.4 manifest files."); + + public static final ConfigOption HIVE_CLIENT_CLASS = + key("metadata.iceberg.hive-client-class") + .stringType() + .defaultValue("org.apache.hadoop.hive.metastore.HiveMetaStoreClient") + .withDescription("Hive client class name for Iceberg Hive Catalog."); + + /** Where to store Iceberg metadata. */ + public enum StorageType implements DescribedEnum { + DISABLED("disabled", "Disable Iceberg compatibility support."), + TABLE_LOCATION("table-location", "Store Iceberg metadata in each table's directory."), + HADOOP_CATALOG( + "hadoop-catalog", + "Store Iceberg metadata in a separate directory. " + + "This directory can be specified as the warehouse directory of an Iceberg Hadoop catalog."), + HIVE_CATALOG( + "hive-catalog", + "Not only store Iceberg metadata like hadoop-catalog, " + + "but also create Iceberg external table in Hive."); + + private final String value; + private final String description; + + StorageType(String value, String description) { + this.value = value; + this.description = description; + } + + @Override + public String toString() { + return value; + } + + @Override + public InlineElement getDescription() { + return TextElement.text(description); + } + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/iceberg/IcebergPathFactory.java b/paimon-core/src/main/java/org/apache/paimon/iceberg/IcebergPathFactory.java index fee590abfbaa..74d2e8e48f1b 100644 --- a/paimon-core/src/main/java/org/apache/paimon/iceberg/IcebergPathFactory.java +++ b/paimon-core/src/main/java/org/apache/paimon/iceberg/IcebergPathFactory.java @@ -37,8 +37,8 @@ public class IcebergPathFactory { private int manifestFileCount; private int manifestListCount; - public IcebergPathFactory(Path root) { - this.metadataDirectory = new Path(root, "metadata"); + public IcebergPathFactory(Path metadataDirectory) { + this.metadataDirectory = metadataDirectory; this.uuid = UUID.randomUUID().toString(); } diff --git a/paimon-core/src/main/java/org/apache/paimon/iceberg/manifest/IcebergConversions.java b/paimon-core/src/main/java/org/apache/paimon/iceberg/manifest/IcebergConversions.java index 1d9e1c3b16e9..9048d46e44f6 100644 --- a/paimon-core/src/main/java/org/apache/paimon/iceberg/manifest/IcebergConversions.java +++ b/paimon-core/src/main/java/org/apache/paimon/iceberg/manifest/IcebergConversions.java @@ -20,8 +20,12 @@ import org.apache.paimon.data.BinaryString; import org.apache.paimon.data.Decimal; +import org.apache.paimon.data.Timestamp; import org.apache.paimon.types.DataType; import org.apache.paimon.types.DecimalType; +import org.apache.paimon.types.LocalZonedTimestampType; +import org.apache.paimon.types.TimestampType; +import org.apache.paimon.utils.Preconditions; import java.nio.ByteBuffer; import java.nio.ByteOrder; @@ -79,11 +83,26 @@ public static ByteBuffer toByteBuffer(DataType type, Object value) { case DECIMAL: Decimal decimal = (Decimal) value; return ByteBuffer.wrap((decimal.toUnscaledBytes())); + case TIMESTAMP_WITHOUT_TIME_ZONE: + return timestampToByteBuffer( + (Timestamp) value, ((TimestampType) type).getPrecision()); + case TIMESTAMP_WITH_LOCAL_TIME_ZONE: + return timestampToByteBuffer( + (Timestamp) value, ((LocalZonedTimestampType) type).getPrecision()); default: throw new UnsupportedOperationException("Cannot serialize type: " + type); } } + private static ByteBuffer timestampToByteBuffer(Timestamp timestamp, int precision) { + Preconditions.checkArgument( + precision > 3 && precision <= 6, + "Paimon Iceberg compatibility only support timestamp type with precision from 4 to 6."); + return ByteBuffer.allocate(8) + .order(ByteOrder.LITTLE_ENDIAN) + .putLong(0, timestamp.toMicros()); + } + public static Object toPaimonObject(DataType type, byte[] bytes) { switch (type.getTypeRoot()) { case BOOLEAN: @@ -112,6 +131,15 @@ public static Object toPaimonObject(DataType type, byte[] bytes) { DecimalType decimalType = (DecimalType) type; return Decimal.fromUnscaledBytes( bytes, decimalType.getPrecision(), decimalType.getScale()); + case TIMESTAMP_WITHOUT_TIME_ZONE: + case TIMESTAMP_WITH_LOCAL_TIME_ZONE: + int timestampPrecision = ((TimestampType) type).getPrecision(); + long timestampLong = + ByteBuffer.wrap(bytes).order(ByteOrder.LITTLE_ENDIAN).getLong(); + Preconditions.checkArgument( + timestampPrecision > 3 && timestampPrecision <= 6, + "Paimon Iceberg compatibility only support timestamp type with precision from 4 to 6."); + return Timestamp.fromMicros(timestampLong); default: throw new UnsupportedOperationException("Cannot deserialize type: " + type); } diff --git a/paimon-core/src/main/java/org/apache/paimon/iceberg/manifest/IcebergDataFileMeta.java b/paimon-core/src/main/java/org/apache/paimon/iceberg/manifest/IcebergDataFileMeta.java index da13eb3188a8..d171962becad 100644 --- a/paimon-core/src/main/java/org/apache/paimon/iceberg/manifest/IcebergDataFileMeta.java +++ b/paimon-core/src/main/java/org/apache/paimon/iceberg/manifest/IcebergDataFileMeta.java @@ -22,13 +22,15 @@ import org.apache.paimon.data.GenericMap; import org.apache.paimon.data.InternalMap; import org.apache.paimon.data.InternalRow; -import org.apache.paimon.schema.TableSchema; +import org.apache.paimon.iceberg.metadata.IcebergDataField; +import org.apache.paimon.iceberg.metadata.IcebergSchema; import org.apache.paimon.stats.SimpleStats; import org.apache.paimon.types.DataField; -import org.apache.paimon.types.DataType; import org.apache.paimon.types.DataTypes; import org.apache.paimon.types.RowType; +import javax.annotation.Nullable; + import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -109,27 +111,45 @@ public static IcebergDataFileMeta create( BinaryRow partition, long recordCount, long fileSizeInBytes, - TableSchema tableSchema, - SimpleStats stats) { + IcebergSchema icebergSchema, + SimpleStats stats, + @Nullable List statsColumns) { + int numFields = icebergSchema.fields().size(); + Map indexMap = new HashMap<>(); + if (statsColumns == null) { + for (int i = 0; i < numFields; i++) { + indexMap.put(icebergSchema.fields().get(i).name(), i); + } + } else { + for (int i = 0; i < statsColumns.size(); i++) { + indexMap.put(statsColumns.get(i), i); + } + } + Map nullValueCounts = new HashMap<>(); Map lowerBounds = new HashMap<>(); Map upperBounds = new HashMap<>(); - List fieldGetters = new ArrayList<>(); - int numFields = tableSchema.fields().size(); for (int i = 0; i < numFields; i++) { - fieldGetters.add(InternalRow.createFieldGetter(tableSchema.fields().get(i).type(), i)); - } + IcebergDataField field = icebergSchema.fields().get(i); + if (!indexMap.containsKey(field.name())) { + continue; + } - for (int i = 0; i < numFields; i++) { - int fieldId = tableSchema.fields().get(i).id(); - DataType type = tableSchema.fields().get(i).type(); - nullValueCounts.put(fieldId, stats.nullCounts().getLong(i)); - Object minValue = fieldGetters.get(i).getFieldOrNull(stats.minValues()); - Object maxValue = fieldGetters.get(i).getFieldOrNull(stats.maxValues()); + int idx = indexMap.get(field.name()); + nullValueCounts.put(field.id(), stats.nullCounts().getLong(idx)); + + InternalRow.FieldGetter fieldGetter = + InternalRow.createFieldGetter(field.dataType(), idx); + Object minValue = fieldGetter.getFieldOrNull(stats.minValues()); + Object maxValue = fieldGetter.getFieldOrNull(stats.maxValues()); if (minValue != null && maxValue != null) { - lowerBounds.put(fieldId, IcebergConversions.toByteBuffer(type, minValue).array()); - upperBounds.put(fieldId, IcebergConversions.toByteBuffer(type, maxValue).array()); + lowerBounds.put( + field.id(), + IcebergConversions.toByteBuffer(field.dataType(), minValue).array()); + upperBounds.put( + field.id(), + IcebergConversions.toByteBuffer(field.dataType(), maxValue).array()); } } @@ -204,7 +224,7 @@ public static RowType schema(RowType partitionType) { 128, "upper_bounds", DataTypes.MAP(DataTypes.INT().notNull(), DataTypes.BYTES().notNull()))); - return new RowType(fields); + return new RowType(false, fields); } @Override diff --git a/paimon-core/src/main/java/org/apache/paimon/iceberg/manifest/IcebergManifestEntry.java b/paimon-core/src/main/java/org/apache/paimon/iceberg/manifest/IcebergManifestEntry.java index 213ec01316b0..1e8108a9405e 100644 --- a/paimon-core/src/main/java/org/apache/paimon/iceberg/manifest/IcebergManifestEntry.java +++ b/paimon-core/src/main/java/org/apache/paimon/iceberg/manifest/IcebergManifestEntry.java @@ -118,7 +118,7 @@ public static RowType schema(RowType partitionType) { fields.add(new DataField(4, "file_sequence_number", DataTypes.BIGINT())); fields.add( new DataField(2, "data_file", IcebergDataFileMeta.schema(partitionType).notNull())); - return new RowType(fields); + return new RowType(false, fields); } @Override diff --git a/paimon-core/src/main/java/org/apache/paimon/iceberg/manifest/IcebergManifestFile.java b/paimon-core/src/main/java/org/apache/paimon/iceberg/manifest/IcebergManifestFile.java index d04cf3576a11..5955da6220f8 100644 --- a/paimon-core/src/main/java/org/apache/paimon/iceberg/manifest/IcebergManifestFile.java +++ b/paimon-core/src/main/java/org/apache/paimon/iceberg/manifest/IcebergManifestFile.java @@ -18,7 +18,7 @@ package org.apache.paimon.iceberg.manifest; -import org.apache.paimon.CoreOptions; +import org.apache.paimon.annotation.VisibleForTesting; import org.apache.paimon.format.FileFormat; import org.apache.paimon.format.FormatReaderFactory; import org.apache.paimon.format.FormatWriterFactory; @@ -26,6 +26,7 @@ import org.apache.paimon.format.SimpleStatsCollector; import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.Path; +import org.apache.paimon.iceberg.IcebergOptions; import org.apache.paimon.iceberg.IcebergPathFactory; import org.apache.paimon.iceberg.manifest.IcebergManifestFileMeta.Content; import org.apache.paimon.iceberg.metadata.IcebergPartitionSpec; @@ -82,29 +83,34 @@ public IcebergManifestFile( this.targetFileSize = targetFileSize; } + @VisibleForTesting + public String compression() { + return compression; + } + public static IcebergManifestFile create(FileStoreTable table, IcebergPathFactory pathFactory) { RowType partitionType = table.schema().logicalPartitionType(); RowType entryType = IcebergManifestEntry.schema(partitionType); - Options manifestFileAvroOptions = Options.fromMap(table.options()); + Options avroOptions = Options.fromMap(table.options()); // https://github.com/apache/iceberg/blob/main/core/src/main/java/org/apache/iceberg/ManifestReader.java - manifestFileAvroOptions.set( + avroOptions.set( "avro.row-name-mapping", "org.apache.paimon.avro.generated.record:manifest_entry," + "manifest_entry_data_file:r2," + "r2_partition:r102"); - FileFormat manifestFileAvro = FileFormat.fromIdentifier("avro", manifestFileAvroOptions); + FileFormat manifestFileAvro = FileFormat.fromIdentifier("avro", avroOptions); return new IcebergManifestFile( table.fileIO(), partitionType, manifestFileAvro.createReaderFactory(entryType), manifestFileAvro.createWriterFactory(entryType), - table.coreOptions().manifestCompression(), + avroOptions.get(IcebergOptions.MANIFEST_COMPRESSION), pathFactory.manifestFileFactory(), table.coreOptions().manifestTargetSize()); } public List rollingWrite( - Iterator entries, long sequenceNumber) throws IOException { + Iterator entries, long sequenceNumber) { RollingFileWriter writer = new RollingFileWriter<>( () -> createWriter(sequenceNumber), targetFileSize.getBytes()); @@ -120,10 +126,7 @@ public List rollingWrite( public SingleFileWriter createWriter( long sequenceNumber) { return new IcebergManifestEntryWriter( - writerFactory, - pathFactory.newPath(), - CoreOptions.FILE_COMPRESSION.defaultValue(), - sequenceNumber); + writerFactory, pathFactory.newPath(), compression, sequenceNumber); } private class IcebergManifestEntryWriter diff --git a/paimon-core/src/main/java/org/apache/paimon/iceberg/manifest/IcebergManifestFileMeta.java b/paimon-core/src/main/java/org/apache/paimon/iceberg/manifest/IcebergManifestFileMeta.java index 01b3b96acbf6..c5fcb6005fcb 100644 --- a/paimon-core/src/main/java/org/apache/paimon/iceberg/manifest/IcebergManifestFileMeta.java +++ b/paimon-core/src/main/java/org/apache/paimon/iceberg/manifest/IcebergManifestFileMeta.java @@ -165,7 +165,11 @@ public List partitions() { return partitions; } - public static RowType schema() { + public static RowType schema(boolean legacyVersion) { + return legacyVersion ? schemaForIceberg1_4() : schemaForIcebergNew(); + } + + private static RowType schemaForIcebergNew() { List fields = new ArrayList<>(); fields.add(new DataField(500, "manifest_path", DataTypes.STRING().notNull())); fields.add(new DataField(501, "manifest_length", DataTypes.BIGINT().notNull())); @@ -183,7 +187,30 @@ public static RowType schema() { fields.add( new DataField( 508, "partitions", DataTypes.ARRAY(IcebergPartitionSummary.schema()))); - return new RowType(fields); + return new RowType(false, fields); + } + + private static RowType schemaForIceberg1_4() { + // see https://github.com/apache/iceberg/pull/5338 + // some reader still want old schema, for example, AWS athena + List fields = new ArrayList<>(); + fields.add(new DataField(500, "manifest_path", DataTypes.STRING().notNull())); + fields.add(new DataField(501, "manifest_length", DataTypes.BIGINT().notNull())); + fields.add(new DataField(502, "partition_spec_id", DataTypes.INT().notNull())); + fields.add(new DataField(517, "content", DataTypes.INT().notNull())); + fields.add(new DataField(515, "sequence_number", DataTypes.BIGINT().notNull())); + fields.add(new DataField(516, "min_sequence_number", DataTypes.BIGINT().notNull())); + fields.add(new DataField(503, "added_snapshot_id", DataTypes.BIGINT())); + fields.add(new DataField(504, "added_data_files_count", DataTypes.INT().notNull())); + fields.add(new DataField(505, "existing_data_files_count", DataTypes.INT().notNull())); + fields.add(new DataField(506, "deleted_data_files_count", DataTypes.INT().notNull())); + fields.add(new DataField(512, "added_rows_count", DataTypes.BIGINT().notNull())); + fields.add(new DataField(513, "existing_rows_count", DataTypes.BIGINT().notNull())); + fields.add(new DataField(514, "deleted_rows_count", DataTypes.BIGINT().notNull())); + fields.add( + new DataField( + 508, "partitions", DataTypes.ARRAY(IcebergPartitionSummary.schema()))); + return new RowType(false, fields); } @Override diff --git a/paimon-core/src/main/java/org/apache/paimon/iceberg/manifest/IcebergManifestFileMetaSerializer.java b/paimon-core/src/main/java/org/apache/paimon/iceberg/manifest/IcebergManifestFileMetaSerializer.java index c40a26e8fdf8..2b4c9b771c59 100644 --- a/paimon-core/src/main/java/org/apache/paimon/iceberg/manifest/IcebergManifestFileMetaSerializer.java +++ b/paimon-core/src/main/java/org/apache/paimon/iceberg/manifest/IcebergManifestFileMetaSerializer.java @@ -24,6 +24,7 @@ import org.apache.paimon.data.InternalArray; import org.apache.paimon.data.InternalRow; import org.apache.paimon.iceberg.manifest.IcebergManifestFileMeta.Content; +import org.apache.paimon.types.RowType; import org.apache.paimon.utils.ObjectSerializer; import java.util.ArrayList; @@ -36,8 +37,8 @@ public class IcebergManifestFileMetaSerializer extends ObjectSerializer { public IcebergManifestList( FileIO fileIO, - FormatReaderFactory readerFactory, - FormatWriterFactory writerFactory, + FileFormat fileFormat, + RowType manifestType, String compression, PathFactory pathFactory) { super( fileIO, - new IcebergManifestFileMetaSerializer(), - IcebergManifestFileMeta.schema(), - readerFactory, - writerFactory, + new IcebergManifestFileMetaSerializer(manifestType), + manifestType, + fileFormat.createReaderFactory(manifestType), + fileFormat.createWriterFactory(manifestType), compression, pathFactory, null); } + @VisibleForTesting + public String compression() { + return compression; + } + public static IcebergManifestList create(FileStoreTable table, IcebergPathFactory pathFactory) { - Options manifestListAvroOptions = Options.fromMap(table.options()); + Options avroOptions = Options.fromMap(table.options()); // https://github.com/apache/iceberg/blob/main/core/src/main/java/org/apache/iceberg/ManifestLists.java - manifestListAvroOptions.set( + avroOptions.set( "avro.row-name-mapping", "org.apache.paimon.avro.generated.record:manifest_file," + "manifest_file_partitions:r508"); - FileFormat manifestListAvro = FileFormat.fromIdentifier("avro", manifestListAvroOptions); + FileFormat fileFormat = FileFormat.fromIdentifier("avro", avroOptions); + RowType manifestType = + IcebergManifestFileMeta.schema( + avroOptions.get(IcebergOptions.MANIFEST_LEGACY_VERSION)); return new IcebergManifestList( table.fileIO(), - manifestListAvro.createReaderFactory(IcebergManifestFileMeta.schema()), - manifestListAvro.createWriterFactory(IcebergManifestFileMeta.schema()), - table.coreOptions().manifestCompression(), + fileFormat, + manifestType, + avroOptions.get(IcebergOptions.MANIFEST_COMPRESSION), pathFactory.manifestListFactory()); } } diff --git a/paimon-core/src/main/java/org/apache/paimon/iceberg/manifest/IcebergPartitionSummary.java b/paimon-core/src/main/java/org/apache/paimon/iceberg/manifest/IcebergPartitionSummary.java index e47a8fe00461..9d27318606af 100644 --- a/paimon-core/src/main/java/org/apache/paimon/iceberg/manifest/IcebergPartitionSummary.java +++ b/paimon-core/src/main/java/org/apache/paimon/iceberg/manifest/IcebergPartitionSummary.java @@ -69,7 +69,7 @@ public static RowType schema() { fields.add(new DataField(518, "contains_nan", DataTypes.BOOLEAN())); fields.add(new DataField(510, "lower_bound", DataTypes.BYTES())); fields.add(new DataField(511, "upper_bound", DataTypes.BYTES())); - return (RowType) new RowType(fields).notNull(); + return new RowType(false, fields); } @Override diff --git a/paimon-core/src/main/java/org/apache/paimon/iceberg/metadata/IcebergDataField.java b/paimon-core/src/main/java/org/apache/paimon/iceberg/metadata/IcebergDataField.java index fd05183b6dc9..4ecc77a13581 100644 --- a/paimon-core/src/main/java/org/apache/paimon/iceberg/metadata/IcebergDataField.java +++ b/paimon-core/src/main/java/org/apache/paimon/iceberg/metadata/IcebergDataField.java @@ -18,16 +18,25 @@ package org.apache.paimon.iceberg.metadata; +import org.apache.paimon.table.SpecialFields; +import org.apache.paimon.types.ArrayType; import org.apache.paimon.types.DataField; import org.apache.paimon.types.DataType; import org.apache.paimon.types.DecimalType; +import org.apache.paimon.types.LocalZonedTimestampType; +import org.apache.paimon.types.MapType; +import org.apache.paimon.types.RowType; +import org.apache.paimon.types.TimestampType; +import org.apache.paimon.utils.Preconditions; import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator; import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonGetter; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonIgnore; import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonIgnoreProperties; import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty; import java.util.Objects; +import java.util.stream.Collectors; /** * {@link DataField} in Iceberg. @@ -53,7 +62,9 @@ public class IcebergDataField { private final boolean required; @JsonProperty(FIELD_TYPE) - private final String type; + private final Object type; + + @JsonIgnore private final DataType dataType; @JsonProperty(FIELD_DOC) private final String doc; @@ -63,7 +74,8 @@ public IcebergDataField(DataField dataField) { dataField.id(), dataField.name(), !dataField.type().isNullable(), - toTypeString(dataField.type()), + toTypeObject(dataField.type(), dataField.id(), 0), + dataField.type(), dataField.description()); } @@ -72,12 +84,18 @@ public IcebergDataField( @JsonProperty(FIELD_ID) int id, @JsonProperty(FIELD_NAME) String name, @JsonProperty(FIELD_REQUIRED) boolean required, - @JsonProperty(FIELD_TYPE) String type, + @JsonProperty(FIELD_TYPE) Object type, @JsonProperty(FIELD_DOC) String doc) { + this(id, name, required, type, null, doc); + } + + public IcebergDataField( + int id, String name, boolean required, Object type, DataType dataType, String doc) { this.id = id; this.name = name; this.required = required; this.type = type; + this.dataType = dataType; this.doc = doc; } @@ -97,7 +115,7 @@ public boolean required() { } @JsonGetter(FIELD_TYPE) - public String type() { + public Object type() { return type; } @@ -106,7 +124,12 @@ public String doc() { return doc; } - private static String toTypeString(DataType dataType) { + @JsonIgnore + public DataType dataType() { + return Preconditions.checkNotNull(dataType); + } + + private static Object toTypeObject(DataType dataType, int fieldId, int depth) { switch (dataType.getTypeRoot()) { case BOOLEAN: return "boolean"; @@ -130,6 +153,38 @@ private static String toTypeString(DataType dataType) { DecimalType decimalType = (DecimalType) dataType; return String.format( "decimal(%d, %d)", decimalType.getPrecision(), decimalType.getScale()); + case TIMESTAMP_WITHOUT_TIME_ZONE: + int timestampPrecision = ((TimestampType) dataType).getPrecision(); + Preconditions.checkArgument( + timestampPrecision > 3 && timestampPrecision <= 6, + "Paimon Iceberg compatibility only support timestamp type with precision from 4 to 6."); + return "timestamp"; + case TIMESTAMP_WITH_LOCAL_TIME_ZONE: + int timestampLtzPrecision = ((LocalZonedTimestampType) dataType).getPrecision(); + Preconditions.checkArgument( + timestampLtzPrecision > 3 && timestampLtzPrecision <= 6, + "Paimon Iceberg compatibility only support timestamp type with precision from 4 to 6."); + return "timestamptz"; + case ARRAY: + ArrayType arrayType = (ArrayType) dataType; + return new IcebergListType( + SpecialFields.getArrayElementFieldId(fieldId, depth + 1), + !dataType.isNullable(), + toTypeObject(arrayType.getElementType(), fieldId, depth + 1)); + case MAP: + MapType mapType = (MapType) dataType; + return new IcebergMapType( + SpecialFields.getMapKeyFieldId(fieldId, depth + 1), + toTypeObject(mapType.getKeyType(), fieldId, depth + 1), + SpecialFields.getMapValueFieldId(fieldId, depth + 1), + !mapType.getValueType().isNullable(), + toTypeObject(mapType.getValueType(), fieldId, depth + 1)); + case ROW: + RowType rowType = (RowType) dataType; + return new IcebergStructType( + rowType.getFields().stream() + .map(IcebergDataField::new) + .collect(Collectors.toList())); default: throw new UnsupportedOperationException("Unsupported data type: " + dataType); } diff --git a/paimon-core/src/main/java/org/apache/paimon/iceberg/metadata/IcebergListType.java b/paimon-core/src/main/java/org/apache/paimon/iceberg/metadata/IcebergListType.java new file mode 100644 index 000000000000..d25ead64fcb5 --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/iceberg/metadata/IcebergListType.java @@ -0,0 +1,109 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.iceberg.metadata; + +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonGetter; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.Objects; + +/** + * {@link org.apache.paimon.types.ArrayType} in Iceberg. + * + *

    See Iceberg spec. + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class IcebergListType { + + private static final String FIELD_TYPE = "type"; + private static final String FIELD_ELEMENT_ID = "element-id"; + private static final String FIELD_ELEMENT_REQUIRED = "element-required"; + private static final String FIELD_ELEMENT = "element"; + + @JsonProperty(FIELD_TYPE) + private final String type; + + @JsonProperty(FIELD_ELEMENT_ID) + private final int elementId; + + @JsonProperty(FIELD_ELEMENT_REQUIRED) + private final boolean elementRequired; + + @JsonProperty(FIELD_ELEMENT) + private final Object element; + + public IcebergListType(int elementId, boolean elementRequired, Object element) { + this("list", elementId, elementRequired, element); + } + + @JsonCreator + public IcebergListType( + @JsonProperty(FIELD_TYPE) String type, + @JsonProperty(FIELD_ELEMENT_ID) int elementId, + @JsonProperty(FIELD_ELEMENT_REQUIRED) boolean elementRequired, + @JsonProperty(FIELD_ELEMENT) Object element) { + this.type = type; + this.elementId = elementId; + this.elementRequired = elementRequired; + this.element = element; + } + + @JsonGetter(FIELD_TYPE) + public String type() { + return type; + } + + @JsonGetter(FIELD_ELEMENT_ID) + public int elementId() { + return elementId; + } + + @JsonGetter(FIELD_ELEMENT_REQUIRED) + public boolean elementRequired() { + return elementRequired; + } + + @JsonGetter(FIELD_ELEMENT) + public Object element() { + return element; + } + + @Override + public int hashCode() { + return Objects.hash(type, elementId, elementRequired, element); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof IcebergListType)) { + return false; + } + + IcebergListType that = (IcebergListType) o; + return Objects.equals(type, that.type) + && elementId == that.elementId + && elementRequired == that.elementRequired + && Objects.equals(element, that.element); + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/iceberg/metadata/IcebergMapType.java b/paimon-core/src/main/java/org/apache/paimon/iceberg/metadata/IcebergMapType.java new file mode 100644 index 000000000000..81a3a04b1f41 --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/iceberg/metadata/IcebergMapType.java @@ -0,0 +1,132 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.iceberg.metadata; + +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonGetter; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.Objects; + +/** + * {@link org.apache.paimon.types.MapType} in Iceberg. + * + *

    See Iceberg spec. + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class IcebergMapType { + + private static final String FIELD_TYPE = "type"; + private static final String FIELD_KEY_ID = "key-id"; + private static final String FIELD_KEY = "key"; + private static final String FIELD_VALUE_ID = "value-id"; + private static final String FIELD_VALUE_REQUIRED = "value-required"; + private static final String FIELD_VALUE = "value"; + + @JsonProperty(FIELD_TYPE) + private final String type; + + @JsonProperty(FIELD_KEY_ID) + private final int keyId; + + @JsonProperty(FIELD_KEY) + private final Object key; + + @JsonProperty(FIELD_VALUE_ID) + private final int valueId; + + @JsonProperty(FIELD_VALUE_REQUIRED) + private final boolean valueRequired; + + @JsonProperty(FIELD_VALUE) + private final Object value; + + public IcebergMapType(int keyId, Object key, int valueId, boolean valueRequired, Object value) { + this("map", keyId, key, valueId, valueRequired, value); + } + + @JsonCreator + public IcebergMapType( + @JsonProperty(FIELD_TYPE) String type, + @JsonProperty(FIELD_KEY_ID) int keyId, + @JsonProperty(FIELD_KEY) Object key, + @JsonProperty(FIELD_VALUE_ID) int valueId, + @JsonProperty(FIELD_VALUE_REQUIRED) boolean valueRequired, + @JsonProperty(FIELD_VALUE) Object value) { + this.type = type; + this.keyId = keyId; + this.key = key; + this.valueId = valueId; + this.valueRequired = valueRequired; + this.value = value; + } + + @JsonGetter(FIELD_TYPE) + public String type() { + return type; + } + + @JsonGetter(FIELD_KEY_ID) + public int keyId() { + return keyId; + } + + @JsonGetter(FIELD_KEY) + public Object key() { + return key; + } + + @JsonGetter(FIELD_VALUE_ID) + public int valueId() { + return valueId; + } + + @JsonGetter(FIELD_VALUE_REQUIRED) + public boolean valueRequired() { + return valueRequired; + } + + @JsonGetter(FIELD_VALUE) + public Object value() { + return value; + } + + @Override + public int hashCode() { + return Objects.hash(type, keyId, key, valueId, valueRequired, value); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof IcebergMapType)) { + return false; + } + IcebergMapType that = (IcebergMapType) o; + return Objects.equals(type, that.type) + && keyId == that.keyId + && Objects.equals(key, that.key) + && valueId == that.valueId + && valueRequired == that.valueRequired + && Objects.equals(value, that.value); + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/iceberg/metadata/IcebergMetadata.java b/paimon-core/src/main/java/org/apache/paimon/iceberg/metadata/IcebergMetadata.java index a3af25a1c668..86fb4a5df75a 100644 --- a/paimon-core/src/main/java/org/apache/paimon/iceberg/metadata/IcebergMetadata.java +++ b/paimon-core/src/main/java/org/apache/paimon/iceberg/metadata/IcebergMetadata.java @@ -27,9 +27,13 @@ import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonIgnoreProperties; import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty; +import javax.annotation.Nullable; + import java.io.IOException; import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Objects; /** @@ -57,6 +61,7 @@ public class IcebergMetadata { private static final String FIELD_DEFAULT_SORT_ORDER_ID = "default-sort-order-id"; private static final String FIELD_SNAPSHOTS = "snapshots"; private static final String FIELD_CURRENT_SNAPSHOT_ID = "current-snapshot-id"; + private static final String FIELD_PROPERTIES = "properties"; @JsonProperty(FIELD_FORMAT_VERSION) private final int formatVersion; @@ -103,6 +108,10 @@ public class IcebergMetadata { @JsonProperty(FIELD_CURRENT_SNAPSHOT_ID) private final int currentSnapshotId; + @JsonProperty(FIELD_PROPERTIES) + @Nullable + private final Map properties; + public IcebergMetadata( String tableUuid, String location, @@ -129,7 +138,8 @@ public IcebergMetadata( Collections.singletonList(new IcebergSortOrder()), IcebergSortOrder.ORDER_ID, snapshots, - currentSnapshotId); + currentSnapshotId, + new HashMap<>()); } @JsonCreator @@ -148,7 +158,8 @@ public IcebergMetadata( @JsonProperty(FIELD_SORT_ORDERS) List sortOrders, @JsonProperty(FIELD_DEFAULT_SORT_ORDER_ID) int defaultSortOrderId, @JsonProperty(FIELD_SNAPSHOTS) List snapshots, - @JsonProperty(FIELD_CURRENT_SNAPSHOT_ID) int currentSnapshotId) { + @JsonProperty(FIELD_CURRENT_SNAPSHOT_ID) int currentSnapshotId, + @JsonProperty(FIELD_PROPERTIES) @Nullable Map properties) { this.formatVersion = formatVersion; this.tableUuid = tableUuid; this.location = location; @@ -164,6 +175,7 @@ public IcebergMetadata( this.defaultSortOrderId = defaultSortOrderId; this.snapshots = snapshots; this.currentSnapshotId = currentSnapshotId; + this.properties = properties; } @JsonGetter(FIELD_FORMAT_VERSION) @@ -241,6 +253,11 @@ public int currentSnapshotId() { return currentSnapshotId; } + @JsonGetter(FIELD_PROPERTIES) + public Map properties() { + return properties == null ? new HashMap<>() : properties; + } + public IcebergSnapshot currentSnapshot() { for (IcebergSnapshot snapshot : snapshots) { if (snapshot.snapshotId() == currentSnapshotId) { diff --git a/paimon-core/src/main/java/org/apache/paimon/iceberg/metadata/IcebergPartitionField.java b/paimon-core/src/main/java/org/apache/paimon/iceberg/metadata/IcebergPartitionField.java index 7be0d0493b84..5b8af183e100 100644 --- a/paimon-core/src/main/java/org/apache/paimon/iceberg/metadata/IcebergPartitionField.java +++ b/paimon-core/src/main/java/org/apache/paimon/iceberg/metadata/IcebergPartitionField.java @@ -18,8 +18,6 @@ package org.apache.paimon.iceberg.metadata; -import org.apache.paimon.types.DataField; - import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator; import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonGetter; import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonIgnoreProperties; @@ -55,7 +53,7 @@ public class IcebergPartitionField { @JsonProperty(FIELD_FIELD_ID) private final int fieldId; - public IcebergPartitionField(DataField dataField, int fieldId) { + public IcebergPartitionField(IcebergDataField dataField, int fieldId) { this( dataField.name(), // currently Paimon's partition value does not have any transformation diff --git a/paimon-core/src/main/java/org/apache/paimon/iceberg/metadata/IcebergSchema.java b/paimon-core/src/main/java/org/apache/paimon/iceberg/metadata/IcebergSchema.java index b3c82021ec95..ff28a3bfd2c4 100644 --- a/paimon-core/src/main/java/org/apache/paimon/iceberg/metadata/IcebergSchema.java +++ b/paimon-core/src/main/java/org/apache/paimon/iceberg/metadata/IcebergSchema.java @@ -22,6 +22,7 @@ import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator; import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonGetter; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonIgnore; import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonIgnoreProperties; import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty; @@ -50,8 +51,8 @@ public class IcebergSchema { @JsonProperty(FIELD_FIELDS) private final List fields; - public IcebergSchema(TableSchema tableSchema) { - this( + public static IcebergSchema create(TableSchema tableSchema) { + return new IcebergSchema( (int) tableSchema.id(), tableSchema.fields().stream() .map(IcebergDataField::new) @@ -87,6 +88,11 @@ public List fields() { return fields; } + @JsonIgnore + public int highestFieldId() { + return fields.stream().mapToInt(IcebergDataField::id).max().orElse(0); + } + @Override public int hashCode() { return Objects.hash(type, schemaId, fields); diff --git a/paimon-core/src/main/java/org/apache/paimon/iceberg/metadata/IcebergStructType.java b/paimon-core/src/main/java/org/apache/paimon/iceberg/metadata/IcebergStructType.java new file mode 100644 index 000000000000..84b0d430e438 --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/iceberg/metadata/IcebergStructType.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.iceberg.metadata; + +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonGetter; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.List; +import java.util.Objects; + +/** + * {@link org.apache.paimon.types.RowType} in Iceberg. + * + *

    See Iceberg spec. + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class IcebergStructType { + + private static final String FIELD_TYPE = "type"; + private static final String FIELD_FIELDS = "fields"; + + @JsonProperty(FIELD_TYPE) + private final String type; + + @JsonProperty(FIELD_FIELDS) + private final List fields; + + public IcebergStructType(List fields) { + this("struct", fields); + } + + @JsonCreator + public IcebergStructType( + @JsonProperty(FIELD_TYPE) String type, + @JsonProperty(FIELD_FIELDS) List fields) { + this.type = type; + this.fields = fields; + } + + @JsonGetter(FIELD_TYPE) + public String type() { + return type; + } + + @JsonGetter(FIELD_FIELDS) + public List fields() { + return fields; + } + + @Override + public int hashCode() { + return Objects.hash(type, fields); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof IcebergStructType)) { + return false; + } + + IcebergStructType that = (IcebergStructType) o; + return Objects.equals(type, that.type) && Objects.equals(fields, that.fields); + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/index/DeletionVectorMeta.java b/paimon-core/src/main/java/org/apache/paimon/index/DeletionVectorMeta.java new file mode 100644 index 000000000000..9eb38818f694 --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/index/DeletionVectorMeta.java @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.index; + +import org.apache.paimon.types.BigIntType; +import org.apache.paimon.types.DataField; +import org.apache.paimon.types.IntType; +import org.apache.paimon.types.RowType; + +import javax.annotation.Nullable; + +import java.util.Objects; + +import static org.apache.paimon.utils.SerializationUtils.newStringType; + +/** Metadata of deletion vector. */ +public class DeletionVectorMeta { + + public static final RowType SCHEMA = + RowType.of( + new DataField(0, "f0", newStringType(false)), + new DataField(1, "f1", new IntType(false)), + new DataField(2, "f2", new IntType(false)), + new DataField(3, "_CARDINALITY", new BigIntType(true))); + + private final String dataFileName; + private final int offset; + private final int length; + @Nullable private final Long cardinality; + + public DeletionVectorMeta( + String dataFileName, int start, int size, @Nullable Long cardinality) { + this.dataFileName = dataFileName; + this.offset = start; + this.length = size; + this.cardinality = cardinality; + } + + public String dataFileName() { + return dataFileName; + } + + public int offset() { + return offset; + } + + public int length() { + return length; + } + + @Nullable + public Long cardinality() { + return cardinality; + } + + @Override + public boolean equals(Object o) { + if (o == null || getClass() != o.getClass()) { + return false; + } + DeletionVectorMeta that = (DeletionVectorMeta) o; + return offset == that.offset + && length == that.length + && Objects.equals(dataFileName, that.dataFileName) + && Objects.equals(cardinality, that.cardinality); + } + + @Override + public int hashCode() { + return Objects.hash(dataFileName, offset, length, cardinality); + } + + @Override + public String toString() { + return "DeletionVectorMeta{" + + "dataFileName='" + + dataFileName + + '\'' + + ", offset=" + + offset + + ", length=" + + length + + ", cardinality=" + + cardinality + + '}'; + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/index/IndexFileHandler.java b/paimon-core/src/main/java/org/apache/paimon/index/IndexFileHandler.java index 7e5efccdd813..8b0e5c5021f6 100644 --- a/paimon-core/src/main/java/org/apache/paimon/index/IndexFileHandler.java +++ b/paimon-core/src/main/java/org/apache/paimon/index/IndexFileHandler.java @@ -100,15 +100,16 @@ public Map scanDVIndex( if (meta.indexType().equals(DELETION_VECTORS_INDEX) && file.partition().equals(partition) && file.bucket() == bucket) { - LinkedHashMap> dvRanges = - meta.deletionVectorsRanges(); - checkNotNull(dvRanges); - for (String dataFile : dvRanges.keySet()) { - Pair pair = dvRanges.get(dataFile); - DeletionFile deletionFile = + LinkedHashMap dvMetas = meta.deletionVectorMetas(); + checkNotNull(dvMetas); + for (DeletionVectorMeta dvMeta : dvMetas.values()) { + result.put( + dvMeta.dataFileName(), new DeletionFile( - filePath(meta).toString(), pair.getLeft(), pair.getRight()); - result.put(dataFile, deletionFile); + filePath(meta).toString(), + dvMeta.offset(), + dvMeta.length(), + dvMeta.cardinality())); } } } diff --git a/paimon-core/src/main/java/org/apache/paimon/index/IndexFileMeta.java b/paimon-core/src/main/java/org/apache/paimon/index/IndexFileMeta.java index 24ba6992a5d9..aae4f8c4731b 100644 --- a/paimon-core/src/main/java/org/apache/paimon/index/IndexFileMeta.java +++ b/paimon-core/src/main/java/org/apache/paimon/index/IndexFileMeta.java @@ -23,9 +23,7 @@ import org.apache.paimon.types.ArrayType; import org.apache.paimon.types.BigIntType; import org.apache.paimon.types.DataField; -import org.apache.paimon.types.IntType; import org.apache.paimon.types.RowType; -import org.apache.paimon.utils.Pair; import javax.annotation.Nullable; @@ -54,12 +52,7 @@ public class IndexFileMeta { new DataField( 4, "_DELETIONS_VECTORS_RANGES", - new ArrayType( - true, - RowType.of( - newStringType(false), - new IntType(false), - new IntType(false)))))); + new ArrayType(true, DeletionVectorMeta.SCHEMA)))); private final String indexType; private final String fileName; @@ -68,9 +61,9 @@ public class IndexFileMeta { /** * Metadata only used by {@link DeletionVectorsIndexFile}, use LinkedHashMap to ensure that the - * order of DeletionVectorRanges and the written DeletionVectors is consistent. + * order of DeletionVectorMetas and the written DeletionVectors is consistent. */ - private final @Nullable LinkedHashMap> deletionVectorsRanges; + private final @Nullable LinkedHashMap deletionVectorMetas; public IndexFileMeta(String indexType, String fileName, long fileSize, long rowCount) { this(indexType, fileName, fileSize, rowCount, null); @@ -81,12 +74,12 @@ public IndexFileMeta( String fileName, long fileSize, long rowCount, - @Nullable LinkedHashMap> deletionVectorsRanges) { + @Nullable LinkedHashMap deletionVectorMetas) { this.indexType = indexType; this.fileName = fileName; this.fileSize = fileSize; this.rowCount = rowCount; - this.deletionVectorsRanges = deletionVectorsRanges; + this.deletionVectorMetas = deletionVectorMetas; } public String indexType() { @@ -105,8 +98,8 @@ public long rowCount() { return rowCount; } - public @Nullable LinkedHashMap> deletionVectorsRanges() { - return deletionVectorsRanges; + public @Nullable LinkedHashMap deletionVectorMetas() { + return deletionVectorMetas; } @Override @@ -122,12 +115,12 @@ public boolean equals(Object o) { && Objects.equals(fileName, that.fileName) && fileSize == that.fileSize && rowCount == that.rowCount - && Objects.equals(deletionVectorsRanges, that.deletionVectorsRanges); + && Objects.equals(deletionVectorMetas, that.deletionVectorMetas); } @Override public int hashCode() { - return Objects.hash(indexType, fileName, fileSize, rowCount, deletionVectorsRanges); + return Objects.hash(indexType, fileName, fileSize, rowCount, deletionVectorMetas); } @Override @@ -142,8 +135,8 @@ public String toString() { + fileSize + ", rowCount=" + rowCount - + ", deletionVectorsRanges=" - + deletionVectorsRanges + + ", deletionVectorMetas=" + + deletionVectorMetas + '}'; } } diff --git a/paimon-core/src/main/java/org/apache/paimon/index/IndexFileMeta09Serializer.java b/paimon-core/src/main/java/org/apache/paimon/index/IndexFileMeta09Serializer.java new file mode 100644 index 000000000000..915d904569d7 --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/index/IndexFileMeta09Serializer.java @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.index; + +import org.apache.paimon.data.InternalArray; +import org.apache.paimon.data.InternalRow; +import org.apache.paimon.data.serializer.InternalRowSerializer; +import org.apache.paimon.data.serializer.InternalSerializers; +import org.apache.paimon.io.DataInputView; +import org.apache.paimon.types.ArrayType; +import org.apache.paimon.types.BigIntType; +import org.apache.paimon.types.DataField; +import org.apache.paimon.types.IntType; +import org.apache.paimon.types.RowType; + +import java.io.IOException; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.List; + +import static org.apache.paimon.utils.SerializationUtils.newStringType; + +/** Serializer for {@link IndexFileMeta} with 0.9 version. */ +public class IndexFileMeta09Serializer implements Serializable { + + private static final long serialVersionUID = 1L; + + public static final RowType SCHEMA = + new RowType( + false, + Arrays.asList( + new DataField(0, "_INDEX_TYPE", newStringType(false)), + new DataField(1, "_FILE_NAME", newStringType(false)), + new DataField(2, "_FILE_SIZE", new BigIntType(false)), + new DataField(3, "_ROW_COUNT", new BigIntType(false)), + new DataField( + 4, + "_DELETIONS_VECTORS_RANGES", + new ArrayType( + true, + RowType.of( + newStringType(false), + new IntType(false), + new IntType(false)))))); + + protected final InternalRowSerializer rowSerializer; + + public IndexFileMeta09Serializer() { + this.rowSerializer = InternalSerializers.create(SCHEMA); + } + + public IndexFileMeta fromRow(InternalRow row) { + return new IndexFileMeta( + row.getString(0).toString(), + row.getString(1).toString(), + row.getLong(2), + row.getLong(3), + row.isNullAt(4) ? null : rowArrayDataToDvMetas(row.getArray(4))); + } + + public final List deserializeList(DataInputView source) throws IOException { + int size = source.readInt(); + List records = new ArrayList<>(size); + for (int i = 0; i < size; i++) { + records.add(deserialize(source)); + } + return records; + } + + public IndexFileMeta deserialize(DataInputView in) throws IOException { + return fromRow(rowSerializer.deserialize(in)); + } + + public static LinkedHashMap rowArrayDataToDvMetas( + InternalArray arrayData) { + LinkedHashMap dvMetas = new LinkedHashMap<>(arrayData.size()); + for (int i = 0; i < arrayData.size(); i++) { + InternalRow row = arrayData.getRow(i, 3); + dvMetas.put( + row.getString(0).toString(), + new DeletionVectorMeta( + row.getString(0).toString(), row.getInt(1), row.getInt(2), null)); + } + return dvMetas; + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/index/IndexFileMetaSerializer.java b/paimon-core/src/main/java/org/apache/paimon/index/IndexFileMetaSerializer.java index 4b52932623f2..db4a44838fbf 100644 --- a/paimon-core/src/main/java/org/apache/paimon/index/IndexFileMetaSerializer.java +++ b/paimon-core/src/main/java/org/apache/paimon/index/IndexFileMetaSerializer.java @@ -24,9 +24,9 @@ import org.apache.paimon.data.InternalArray; import org.apache.paimon.data.InternalRow; import org.apache.paimon.utils.ObjectSerializer; -import org.apache.paimon.utils.Pair; import org.apache.paimon.utils.VersionedObjectSerializer; +import java.util.Collection; import java.util.LinkedHashMap; /** A {@link VersionedObjectSerializer} for {@link IndexFileMeta}. */ @@ -43,9 +43,9 @@ public InternalRow toRow(IndexFileMeta record) { BinaryString.fromString(record.fileName()), record.fileSize(), record.rowCount(), - record.deletionVectorsRanges() == null + record.deletionVectorMetas() == null ? null - : dvRangesToRowArrayData(record.deletionVectorsRanges())); + : dvMetasToRowArrayData(record.deletionVectorMetas().values())); } @Override @@ -55,30 +55,35 @@ public IndexFileMeta fromRow(InternalRow row) { row.getString(1).toString(), row.getLong(2), row.getLong(3), - row.isNullAt(4) ? null : rowArrayDataToDvRanges(row.getArray(4))); + row.isNullAt(4) ? null : rowArrayDataToDvMetas(row.getArray(4))); } - public static InternalArray dvRangesToRowArrayData( - LinkedHashMap> dvRanges) { + public static InternalArray dvMetasToRowArrayData(Collection dvMetas) { return new GenericArray( - dvRanges.entrySet().stream() + dvMetas.stream() .map( - entry -> + dvMeta -> GenericRow.of( - BinaryString.fromString(entry.getKey()), - entry.getValue().getLeft(), - entry.getValue().getRight())) + BinaryString.fromString(dvMeta.dataFileName()), + dvMeta.offset(), + dvMeta.length(), + dvMeta.cardinality())) .toArray(GenericRow[]::new)); } - public static LinkedHashMap> rowArrayDataToDvRanges( + public static LinkedHashMap rowArrayDataToDvMetas( InternalArray arrayData) { - LinkedHashMap> dvRanges = - new LinkedHashMap<>(arrayData.size()); + LinkedHashMap dvMetas = new LinkedHashMap<>(arrayData.size()); for (int i = 0; i < arrayData.size(); i++) { - InternalRow row = arrayData.getRow(i, 3); - dvRanges.put(row.getString(0).toString(), Pair.of(row.getInt(1), row.getInt(2))); + InternalRow row = arrayData.getRow(i, DeletionVectorMeta.SCHEMA.getFieldCount()); + dvMetas.put( + row.getString(0).toString(), + new DeletionVectorMeta( + row.getString(0).toString(), + row.getInt(1), + row.getInt(2), + row.isNullAt(3) ? null : row.getLong(3))); } - return dvRanges; + return dvMetas; } } diff --git a/paimon-core/src/main/java/org/apache/paimon/io/DataFileMeta.java b/paimon-core/src/main/java/org/apache/paimon/io/DataFileMeta.java index 364d4d42eb23..bb9e45ff002d 100644 --- a/paimon-core/src/main/java/org/apache/paimon/io/DataFileMeta.java +++ b/paimon-core/src/main/java/org/apache/paimon/io/DataFileMeta.java @@ -78,7 +78,11 @@ public class DataFileMeta { new DataField(12, "_CREATION_TIME", DataTypes.TIMESTAMP_MILLIS()), new DataField(13, "_DELETE_ROW_COUNT", new BigIntType(true)), new DataField(14, "_EMBEDDED_FILE_INDEX", newBytesType(true)), - new DataField(15, "_FILE_SOURCE", new TinyIntType(true)))); + new DataField(15, "_FILE_SOURCE", new TinyIntType(true)), + new DataField( + 16, + "_VALUE_STATS_COLS", + DataTypes.ARRAY(DataTypes.STRING().notNull())))); public static final BinaryRow EMPTY_MIN_KEY = EMPTY_ROW; public static final BinaryRow EMPTY_MAX_KEY = EMPTY_ROW; @@ -114,6 +118,8 @@ public class DataFileMeta { private final @Nullable FileSource fileSource; + private final @Nullable List valueStatsCols; + public static DataFileMeta forAppend( String fileName, long fileSize, @@ -122,48 +128,65 @@ public static DataFileMeta forAppend( long minSequenceNumber, long maxSequenceNumber, long schemaId, - @Nullable FileSource fileSource) { - return forAppend( + List extraFiles, + @Nullable byte[] embeddedIndex, + @Nullable FileSource fileSource, + @Nullable List valueStatsCols) { + return new DataFileMeta( fileName, fileSize, rowCount, + EMPTY_MIN_KEY, + EMPTY_MAX_KEY, + EMPTY_STATS, rowStats, minSequenceNumber, maxSequenceNumber, schemaId, - Collections.emptyList(), - null, - fileSource); + DUMMY_LEVEL, + extraFiles, + Timestamp.fromLocalDateTime(LocalDateTime.now()).toMillisTimestamp(), + 0L, + embeddedIndex, + fileSource, + valueStatsCols); } - public static DataFileMeta forAppend( + public DataFileMeta( String fileName, long fileSize, long rowCount, - SimpleStats rowStats, + BinaryRow minKey, + BinaryRow maxKey, + SimpleStats keyStats, + SimpleStats valueStats, long minSequenceNumber, long maxSequenceNumber, long schemaId, + int level, List extraFiles, + @Nullable Long deleteRowCount, @Nullable byte[] embeddedIndex, - @Nullable FileSource fileSource) { - return new DataFileMeta( + @Nullable FileSource fileSource, + @Nullable List valueStatsCols) { + this( fileName, fileSize, rowCount, - EMPTY_MIN_KEY, - EMPTY_MAX_KEY, - EMPTY_STATS, - rowStats, + minKey, + maxKey, + keyStats, + valueStats, minSequenceNumber, maxSequenceNumber, schemaId, - DUMMY_LEVEL, + level, extraFiles, Timestamp.fromLocalDateTime(LocalDateTime.now()).toMillisTimestamp(), - 0L, + deleteRowCount, embeddedIndex, - fileSource); + fileSource, + valueStatsCols); } public DataFileMeta( @@ -180,7 +203,8 @@ public DataFileMeta( int level, @Nullable Long deleteRowCount, @Nullable byte[] embeddedIndex, - @Nullable FileSource fileSource) { + @Nullable FileSource fileSource, + @Nullable List valueStatsCols) { this( fileName, fileSize, @@ -197,7 +221,8 @@ public DataFileMeta( Timestamp.fromLocalDateTime(LocalDateTime.now()).toMillisTimestamp(), deleteRowCount, embeddedIndex, - fileSource); + fileSource, + valueStatsCols); } public DataFileMeta( @@ -216,7 +241,8 @@ public DataFileMeta( Timestamp creationTime, @Nullable Long deleteRowCount, @Nullable byte[] embeddedIndex, - @Nullable FileSource fileSource) { + @Nullable FileSource fileSource, + @Nullable List valueStatsCols) { this.fileName = fileName; this.fileSize = fileSize; @@ -237,6 +263,7 @@ public DataFileMeta( this.deleteRowCount = deleteRowCount; this.fileSource = fileSource; + this.valueStatsCols = valueStatsCols; } public String fileName() { @@ -334,6 +361,11 @@ public Optional fileSource() { return Optional.ofNullable(fileSource); } + @Nullable + public List valueStatsCols() { + return valueStatsCols; + } + public DataFileMeta upgrade(int newLevel) { checkArgument(newLevel > this.level); return new DataFileMeta( @@ -352,7 +384,8 @@ public DataFileMeta upgrade(int newLevel) { creationTime, deleteRowCount, embeddedIndex, - fileSource); + fileSource, + valueStatsCols); } public DataFileMeta rename(String newFileName) { @@ -372,7 +405,29 @@ public DataFileMeta rename(String newFileName) { creationTime, deleteRowCount, embeddedIndex, - fileSource); + fileSource, + valueStatsCols); + } + + public DataFileMeta copyWithoutStats() { + return new DataFileMeta( + fileName, + fileSize, + rowCount, + minKey, + maxKey, + keyStats, + EMPTY_STATS, + minSequenceNumber, + maxSequenceNumber, + schemaId, + level, + extraFiles, + creationTime, + deleteRowCount, + embeddedIndex, + fileSource, + Collections.emptyList()); } public List collectFiles(DataFilePathFactory pathFactory) { @@ -399,7 +454,8 @@ public DataFileMeta copy(List newExtraFiles) { creationTime, deleteRowCount, embeddedIndex, - fileSource); + fileSource, + valueStatsCols); } public DataFileMeta copy(byte[] newEmbeddedIndex) { @@ -419,7 +475,8 @@ public DataFileMeta copy(byte[] newEmbeddedIndex) { creationTime, deleteRowCount, newEmbeddedIndex, - fileSource); + fileSource, + valueStatsCols); } @Override @@ -446,7 +503,8 @@ public boolean equals(Object o) { && Objects.equals(extraFiles, that.extraFiles) && Objects.equals(creationTime, that.creationTime) && Objects.equals(deleteRowCount, that.deleteRowCount) - && Objects.equals(fileSource, that.fileSource); + && Objects.equals(fileSource, that.fileSource) + && Objects.equals(valueStatsCols, that.valueStatsCols); } @Override @@ -467,7 +525,8 @@ public int hashCode() { extraFiles, creationTime, deleteRowCount, - fileSource); + fileSource, + valueStatsCols); } @Override @@ -477,7 +536,7 @@ public String toString() { + "minKey: %s, maxKey: %s, keyStats: %s, valueStats: %s, " + "minSequenceNumber: %d, maxSequenceNumber: %d, " + "schemaId: %d, level: %d, extraFiles: %s, creationTime: %s, " - + "deleteRowCount: %d, fileSource: %s}", + + "deleteRowCount: %d, fileSource: %s, valueStatsCols: %s}", fileName, fileSize, rowCount, @@ -493,7 +552,8 @@ public String toString() { extraFiles, creationTime, deleteRowCount, - fileSource); + fileSource, + valueStatsCols); } public static long getMaxSequenceNumber(List fileMetas) { diff --git a/paimon-core/src/main/java/org/apache/paimon/io/DataFileMeta08Serializer.java b/paimon-core/src/main/java/org/apache/paimon/io/DataFileMeta08Serializer.java index c65f7e78ed6d..03e4ed51f4be 100644 --- a/paimon-core/src/main/java/org/apache/paimon/io/DataFileMeta08Serializer.java +++ b/paimon-core/src/main/java/org/apache/paimon/io/DataFileMeta08Serializer.java @@ -132,6 +132,7 @@ public DataFileMeta deserialize(DataInputView in) throws IOException { row.getTimestamp(12, 3), row.isNullAt(13) ? null : row.getLong(13), row.isNullAt(14) ? null : row.getBinary(14), + null, null); } } diff --git a/paimon-core/src/main/java/org/apache/paimon/io/DataFileMeta09Serializer.java b/paimon-core/src/main/java/org/apache/paimon/io/DataFileMeta09Serializer.java new file mode 100644 index 000000000000..2f8d89f5b1ab --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/io/DataFileMeta09Serializer.java @@ -0,0 +1,144 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.io; + +import org.apache.paimon.data.BinaryString; +import org.apache.paimon.data.GenericRow; +import org.apache.paimon.data.safe.SafeBinaryRow; +import org.apache.paimon.data.serializer.InternalRowSerializer; +import org.apache.paimon.data.serializer.InternalSerializers; +import org.apache.paimon.manifest.FileSource; +import org.apache.paimon.stats.SimpleStats; +import org.apache.paimon.types.ArrayType; +import org.apache.paimon.types.BigIntType; +import org.apache.paimon.types.DataField; +import org.apache.paimon.types.DataTypes; +import org.apache.paimon.types.IntType; +import org.apache.paimon.types.RowType; +import org.apache.paimon.types.TinyIntType; + +import java.io.IOException; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import static org.apache.paimon.utils.InternalRowUtils.fromStringArrayData; +import static org.apache.paimon.utils.InternalRowUtils.toStringArrayData; +import static org.apache.paimon.utils.SerializationUtils.deserializeBinaryRow; +import static org.apache.paimon.utils.SerializationUtils.newBytesType; +import static org.apache.paimon.utils.SerializationUtils.newStringType; +import static org.apache.paimon.utils.SerializationUtils.serializeBinaryRow; + +/** Serializer for {@link DataFileMeta} with 0.9 version. */ +public class DataFileMeta09Serializer implements Serializable { + + private static final long serialVersionUID = 1L; + + public static final RowType SCHEMA = + new RowType( + false, + Arrays.asList( + new DataField(0, "_FILE_NAME", newStringType(false)), + new DataField(1, "_FILE_SIZE", new BigIntType(false)), + new DataField(2, "_ROW_COUNT", new BigIntType(false)), + new DataField(3, "_MIN_KEY", newBytesType(false)), + new DataField(4, "_MAX_KEY", newBytesType(false)), + new DataField(5, "_KEY_STATS", SimpleStats.SCHEMA), + new DataField(6, "_VALUE_STATS", SimpleStats.SCHEMA), + new DataField(7, "_MIN_SEQUENCE_NUMBER", new BigIntType(false)), + new DataField(8, "_MAX_SEQUENCE_NUMBER", new BigIntType(false)), + new DataField(9, "_SCHEMA_ID", new BigIntType(false)), + new DataField(10, "_LEVEL", new IntType(false)), + new DataField( + 11, "_EXTRA_FILES", new ArrayType(false, newStringType(false))), + new DataField(12, "_CREATION_TIME", DataTypes.TIMESTAMP_MILLIS()), + new DataField(13, "_DELETE_ROW_COUNT", new BigIntType(true)), + new DataField(14, "_EMBEDDED_FILE_INDEX", newBytesType(true)), + new DataField(15, "_FILE_SOURCE", new TinyIntType(true)))); + + protected final InternalRowSerializer rowSerializer; + + public DataFileMeta09Serializer() { + this.rowSerializer = InternalSerializers.create(SCHEMA); + } + + public final void serializeList(List records, DataOutputView target) + throws IOException { + target.writeInt(records.size()); + for (DataFileMeta t : records) { + serialize(t, target); + } + } + + public void serialize(DataFileMeta meta, DataOutputView target) throws IOException { + GenericRow row = + GenericRow.of( + BinaryString.fromString(meta.fileName()), + meta.fileSize(), + meta.rowCount(), + serializeBinaryRow(meta.minKey()), + serializeBinaryRow(meta.maxKey()), + meta.keyStats().toRow(), + meta.valueStats().toRow(), + meta.minSequenceNumber(), + meta.maxSequenceNumber(), + meta.schemaId(), + meta.level(), + toStringArrayData(meta.extraFiles()), + meta.creationTime(), + meta.deleteRowCount().orElse(null), + meta.embeddedIndex(), + meta.fileSource().map(FileSource::toByteValue).orElse(null)); + rowSerializer.serialize(row, target); + } + + public final List deserializeList(DataInputView source) throws IOException { + int size = source.readInt(); + List records = new ArrayList<>(size); + for (int i = 0; i < size; i++) { + records.add(deserialize(source)); + } + return records; + } + + public DataFileMeta deserialize(DataInputView in) throws IOException { + byte[] bytes = new byte[in.readInt()]; + in.readFully(bytes); + SafeBinaryRow row = new SafeBinaryRow(rowSerializer.getArity(), bytes, 0); + return new DataFileMeta( + row.getString(0).toString(), + row.getLong(1), + row.getLong(2), + deserializeBinaryRow(row.getBinary(3)), + deserializeBinaryRow(row.getBinary(4)), + SimpleStats.fromRow(row.getRow(5, 3)), + SimpleStats.fromRow(row.getRow(6, 3)), + row.getLong(7), + row.getLong(8), + row.getLong(9), + row.getInt(10), + fromStringArrayData(row.getArray(11)), + row.getTimestamp(12, 3), + row.isNullAt(13) ? null : row.getLong(13), + row.isNullAt(14) ? null : row.getBinary(14), + row.isNullAt(15) ? null : FileSource.fromByteValue(row.getByte(15)), + null); + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/io/DataFileMetaSerializer.java b/paimon-core/src/main/java/org/apache/paimon/io/DataFileMetaSerializer.java index 209aaafd8bf1..626201ca30ce 100644 --- a/paimon-core/src/main/java/org/apache/paimon/io/DataFileMetaSerializer.java +++ b/paimon-core/src/main/java/org/apache/paimon/io/DataFileMetaSerializer.java @@ -57,7 +57,8 @@ public InternalRow toRow(DataFileMeta meta) { meta.creationTime(), meta.deleteRowCount().orElse(null), meta.embeddedIndex(), - meta.fileSource().map(FileSource::toByteValue).orElse(null)); + meta.fileSource().map(FileSource::toByteValue).orElse(null), + toStringArrayData(meta.valueStatsCols())); } @Override @@ -78,6 +79,7 @@ public DataFileMeta fromRow(InternalRow row) { row.getTimestamp(12, 3), row.isNullAt(13) ? null : row.getLong(13), row.isNullAt(14) ? null : row.getBinary(14), - row.isNullAt(15) ? null : FileSource.fromByteValue(row.getByte(15))); + row.isNullAt(15) ? null : FileSource.fromByteValue(row.getByte(15)), + row.isNullAt(16) ? null : fromStringArrayData(row.getArray(16))); } } diff --git a/paimon-core/src/main/java/org/apache/paimon/io/DataFilePathFactory.java b/paimon-core/src/main/java/org/apache/paimon/io/DataFilePathFactory.java index 742888b36420..b632d44c9420 100644 --- a/paimon-core/src/main/java/org/apache/paimon/io/DataFilePathFactory.java +++ b/paimon-core/src/main/java/org/apache/paimon/io/DataFilePathFactory.java @@ -39,18 +39,24 @@ public class DataFilePathFactory { private final String formatIdentifier; private final String dataFilePrefix; private final String changelogFilePrefix; + private final boolean fileSuffixIncludeCompression; + private final String fileCompression; public DataFilePathFactory( Path parent, String formatIdentifier, String dataFilePrefix, - String changelogFilePrefix) { + String changelogFilePrefix, + boolean fileSuffixIncludeCompression, + String fileCompression) { this.parent = parent; this.uuid = UUID.randomUUID().toString(); this.pathCount = new AtomicInteger(0); this.formatIdentifier = formatIdentifier; this.dataFilePrefix = dataFilePrefix; this.changelogFilePrefix = changelogFilePrefix; + this.fileSuffixIncludeCompression = fileSuffixIncludeCompression; + this.fileCompression = fileCompression; } public Path newPath() { @@ -62,7 +68,13 @@ public Path newChangelogPath() { } private Path newPath(String prefix) { - String name = prefix + uuid + "-" + pathCount.getAndIncrement() + "." + formatIdentifier; + String extension; + if (fileSuffixIncludeCompression) { + extension = "." + fileCompression + "." + formatIdentifier; + } else { + extension = "." + formatIdentifier; + } + String name = prefix + uuid + "-" + pathCount.getAndIncrement() + extension; return new Path(parent, name); } diff --git a/paimon-core/src/main/java/org/apache/paimon/io/FileRecordReader.java b/paimon-core/src/main/java/org/apache/paimon/io/DataFileRecordReader.java similarity index 88% rename from paimon-core/src/main/java/org/apache/paimon/io/FileRecordReader.java rename to paimon-core/src/main/java/org/apache/paimon/io/DataFileRecordReader.java index 1e12025ba533..16fad55a49a2 100644 --- a/paimon-core/src/main/java/org/apache/paimon/io/FileRecordReader.java +++ b/paimon-core/src/main/java/org/apache/paimon/io/DataFileRecordReader.java @@ -25,7 +25,8 @@ import org.apache.paimon.data.PartitionInfo; import org.apache.paimon.data.columnar.ColumnarRowIterator; import org.apache.paimon.format.FormatReaderFactory; -import org.apache.paimon.reader.RecordReader; +import org.apache.paimon.reader.FileRecordIterator; +import org.apache.paimon.reader.FileRecordReader; import org.apache.paimon.utils.FileUtils; import org.apache.paimon.utils.ProjectedRow; @@ -34,17 +35,35 @@ import java.io.IOException; /** Reads {@link InternalRow} from data files. */ -public class FileRecordReader implements RecordReader { +public class DataFileRecordReader implements FileRecordReader { - private final RecordReader reader; + private final FileRecordReader reader; @Nullable private final int[] indexMapping; @Nullable private final PartitionInfo partitionInfo; @Nullable private final CastFieldGetter[] castMapping; + public DataFileRecordReader( + FormatReaderFactory readerFactory, + FormatReaderFactory.Context context, + @Nullable int[] indexMapping, + @Nullable CastFieldGetter[] castMapping, + @Nullable PartitionInfo partitionInfo) + throws IOException { + try { + this.reader = readerFactory.createReader(context); + } catch (Exception e) { + FileUtils.checkExists(context.fileIO(), context.filePath()); + throw e; + } + this.indexMapping = indexMapping; + this.partitionInfo = partitionInfo; + this.castMapping = castMapping; + } + @Nullable @Override - public RecordReader.RecordIterator readBatch() throws IOException { - RecordIterator iterator = reader.readBatch(); + public FileRecordIterator readBatch() throws IOException { + FileRecordIterator iterator = reader.readBatch(); if (iterator == null) { return null; } @@ -57,6 +76,7 @@ public RecordReader.RecordIterator readBatch() throws IOException { PartitionSettedRow.from(partitionInfo); iterator = iterator.transform(partitionSettedRow::replaceRow); } + if (indexMapping != null) { final ProjectedRow projectedRow = ProjectedRow.from(indexMapping); iterator = iterator.transform(projectedRow::replaceRow); @@ -71,24 +91,6 @@ public RecordReader.RecordIterator readBatch() throws IOException { return iterator; } - public FileRecordReader( - FormatReaderFactory readerFactory, - FormatReaderFactory.Context context, - @Nullable int[] indexMapping, - @Nullable CastFieldGetter[] castMapping, - @Nullable PartitionInfo partitionInfo) - throws IOException { - try { - this.reader = readerFactory.createReader(context); - } catch (Exception e) { - FileUtils.checkExists(context.fileIO(), context.filePath()); - throw e; - } - this.indexMapping = indexMapping; - this.partitionInfo = partitionInfo; - this.castMapping = castMapping; - } - @Override public void close() throws IOException { reader.close(); diff --git a/paimon-core/src/main/java/org/apache/paimon/io/FileIndexSkipper.java b/paimon-core/src/main/java/org/apache/paimon/io/FileIndexEvaluator.java similarity index 77% rename from paimon-core/src/main/java/org/apache/paimon/io/FileIndexSkipper.java rename to paimon-core/src/main/java/org/apache/paimon/io/FileIndexEvaluator.java index 0c4ac82a05c3..530b87165322 100644 --- a/paimon-core/src/main/java/org/apache/paimon/io/FileIndexSkipper.java +++ b/paimon-core/src/main/java/org/apache/paimon/io/FileIndexEvaluator.java @@ -19,6 +19,7 @@ package org.apache.paimon.io; import org.apache.paimon.fileindex.FileIndexPredicate; +import org.apache.paimon.fileindex.FileIndexResult; import org.apache.paimon.fs.FileIO; import org.apache.paimon.predicate.Predicate; import org.apache.paimon.predicate.PredicateBuilder; @@ -28,10 +29,10 @@ import java.util.List; import java.util.stream.Collectors; -/** File index reader, do the filter in the constructor. */ -public class FileIndexSkipper { +/** Evaluate file index result. */ +public class FileIndexEvaluator { - public static boolean skip( + public static FileIndexResult evaluate( FileIO fileIO, TableSchema dataSchema, List dataFilter, @@ -39,6 +40,15 @@ public static boolean skip( DataFileMeta file) throws IOException { if (dataFilter != null && !dataFilter.isEmpty()) { + byte[] embeddedIndex = file.embeddedIndex(); + if (embeddedIndex != null) { + try (FileIndexPredicate predicate = + new FileIndexPredicate(embeddedIndex, dataSchema.logicalRowType())) { + return predicate.evaluate( + PredicateBuilder.and(dataFilter.toArray(new Predicate[0]))); + } + } + List indexFiles = file.extraFiles().stream() .filter(name -> name.endsWith(DataFilePathFactory.INDEX_PATH_SUFFIX)) @@ -55,14 +65,11 @@ public static boolean skip( dataFilePathFactory.toPath(indexFiles.get(0)), fileIO, dataSchema.logicalRowType())) { - if (!predicate.testPredicate( - PredicateBuilder.and(dataFilter.toArray(new Predicate[0])))) { - return true; - } + return predicate.evaluate( + PredicateBuilder.and(dataFilter.toArray(new Predicate[0]))); } } } - - return false; + return FileIndexResult.REMAIN; } } diff --git a/paimon-core/src/main/java/org/apache/paimon/io/KeyValueDataFileRecordReader.java b/paimon-core/src/main/java/org/apache/paimon/io/KeyValueDataFileRecordReader.java index e44ad79ff53e..6cf08769703f 100644 --- a/paimon-core/src/main/java/org/apache/paimon/io/KeyValueDataFileRecordReader.java +++ b/paimon-core/src/main/java/org/apache/paimon/io/KeyValueDataFileRecordReader.java @@ -21,6 +21,8 @@ import org.apache.paimon.KeyValue; import org.apache.paimon.KeyValueSerializer; import org.apache.paimon.data.InternalRow; +import org.apache.paimon.reader.FileRecordIterator; +import org.apache.paimon.reader.FileRecordReader; import org.apache.paimon.reader.RecordReader; import org.apache.paimon.types.RowType; @@ -29,14 +31,14 @@ import java.io.IOException; /** {@link RecordReader} for reading {@link KeyValue} data files. */ -public class KeyValueDataFileRecordReader implements RecordReader { +public class KeyValueDataFileRecordReader implements FileRecordReader { - private final RecordReader reader; + private final FileRecordReader reader; private final KeyValueSerializer serializer; private final int level; public KeyValueDataFileRecordReader( - RecordReader reader, RowType keyType, RowType valueType, int level) { + FileRecordReader reader, RowType keyType, RowType valueType, int level) { this.reader = reader; this.serializer = new KeyValueSerializer(keyType, valueType); this.level = level; @@ -44,8 +46,8 @@ public KeyValueDataFileRecordReader( @Nullable @Override - public RecordIterator readBatch() throws IOException { - RecordReader.RecordIterator iterator = reader.readBatch(); + public FileRecordIterator readBatch() throws IOException { + FileRecordIterator iterator = reader.readBatch(); if (iterator == null) { return null; } diff --git a/paimon-core/src/main/java/org/apache/paimon/io/KeyValueDataFileWriter.java b/paimon-core/src/main/java/org/apache/paimon/io/KeyValueDataFileWriter.java index 064d72829141..651c6a6f7b56 100644 --- a/paimon-core/src/main/java/org/apache/paimon/io/KeyValueDataFileWriter.java +++ b/paimon-core/src/main/java/org/apache/paimon/io/KeyValueDataFileWriter.java @@ -23,6 +23,7 @@ import org.apache.paimon.data.BinaryRow; import org.apache.paimon.data.InternalRow; import org.apache.paimon.data.serializer.InternalRowSerializer; +import org.apache.paimon.fileindex.FileIndexOptions; import org.apache.paimon.format.FormatWriterFactory; import org.apache.paimon.format.SimpleColStats; import org.apache.paimon.format.SimpleStatsExtractor; @@ -32,6 +33,7 @@ import org.apache.paimon.stats.SimpleStats; import org.apache.paimon.stats.SimpleStatsConverter; import org.apache.paimon.types.RowType; +import org.apache.paimon.utils.Pair; import org.apache.paimon.utils.StatsCollectorFactories; import org.slf4j.Logger; @@ -40,9 +42,12 @@ import javax.annotation.Nullable; import java.io.IOException; -import java.util.Arrays; +import java.util.Collections; +import java.util.List; import java.util.function.Function; +import static org.apache.paimon.io.DataFilePathFactory.dataFileToFileIndexPath; + /** * A {@link StatsCollectingSingleFileWriter} to write data files containing {@link KeyValue}s. Also * produces {@link DataFileMeta} after writing a file. @@ -50,13 +55,13 @@ *

    NOTE: records given to the writer must be sorted because it does not compare the min max keys * to produce {@link DataFileMeta}. */ -public class KeyValueDataFileWriter +public abstract class KeyValueDataFileWriter extends StatsCollectingSingleFileWriter { private static final Logger LOG = LoggerFactory.getLogger(KeyValueDataFileWriter.class); - private final RowType keyType; - private final RowType valueType; + protected final RowType keyType; + protected final RowType valueType; private final long schemaId; private final int level; @@ -64,6 +69,7 @@ public class KeyValueDataFileWriter private final SimpleStatsConverter valueStatsConverter; private final InternalRowSerializer keySerializer; private final FileSource fileSource; + @Nullable private final DataFileIndexWriter dataFileIndexWriter; private BinaryRow minKey = null; private InternalRow maxKey = null; @@ -78,22 +84,24 @@ public KeyValueDataFileWriter( Function converter, RowType keyType, RowType valueType, + RowType writeRowType, @Nullable SimpleStatsExtractor simpleStatsExtractor, long schemaId, int level, String compression, CoreOptions options, - FileSource fileSource) { + FileSource fileSource, + FileIndexOptions fileIndexOptions) { super( fileIO, factory, path, converter, - KeyValue.schema(keyType, valueType), + writeRowType, simpleStatsExtractor, compression, StatsCollectorFactories.createStatsFactories( - options, KeyValue.schema(keyType, valueType).getFieldNames()), + options, writeRowType.getFieldNames(), keyType.getFieldNames()), options.asyncFileWrite()); this.keyType = keyType; @@ -102,15 +110,22 @@ public KeyValueDataFileWriter( this.level = level; this.keyStatsConverter = new SimpleStatsConverter(keyType); - this.valueStatsConverter = new SimpleStatsConverter(valueType); + this.valueStatsConverter = new SimpleStatsConverter(valueType, options.statsDenseStore()); this.keySerializer = new InternalRowSerializer(keyType); this.fileSource = fileSource; + this.dataFileIndexWriter = + DataFileIndexWriter.create( + fileIO, dataFileToFileIndexPath(path), valueType, fileIndexOptions); } @Override public void write(KeyValue kv) throws IOException { super.write(kv); + if (dataFileIndexWriter != null) { + dataFileIndexWriter.write(kv.value()); + } + updateMinKey(kv); updateMaxKey(kv); @@ -151,15 +166,16 @@ public DataFileMeta result() throws IOException { return null; } - SimpleColStats[] rowStats = fieldStats(); - int numKeyFields = keyType.getFieldCount(); + Pair keyValueStats = fetchKeyValueStats(fieldStats()); - SimpleColStats[] keyFieldStats = Arrays.copyOfRange(rowStats, 0, numKeyFields); - SimpleStats keyStats = keyStatsConverter.toBinary(keyFieldStats); + SimpleStats keyStats = keyStatsConverter.toBinaryAllMode(keyValueStats.getKey()); + Pair, SimpleStats> valueStatsPair = + valueStatsConverter.toBinary(keyValueStats.getValue()); - SimpleColStats[] valFieldStats = - Arrays.copyOfRange(rowStats, numKeyFields + 2, rowStats.length); - SimpleStats valueStats = valueStatsConverter.toBinary(valFieldStats); + DataFileIndexWriter.FileIndexResult indexResult = + dataFileIndexWriter == null + ? DataFileIndexWriter.EMPTY_RESULT + : dataFileIndexWriter.result(); return new DataFileMeta( path.getName(), @@ -168,14 +184,27 @@ public DataFileMeta result() throws IOException { minKey, keySerializer.toBinaryRow(maxKey).copy(), keyStats, - valueStats, + valueStatsPair.getValue(), minSeqNumber, maxSeqNumber, schemaId, level, + indexResult.independentIndexFile() == null + ? Collections.emptyList() + : Collections.singletonList(indexResult.independentIndexFile()), deleteRecordCount, - // TODO: enable file filter for primary key table (e.g. deletion table). - null, - fileSource); + indexResult.embeddedIndexBytes(), + fileSource, + valueStatsPair.getKey()); + } + + abstract Pair fetchKeyValueStats(SimpleColStats[] rowStats); + + @Override + public void close() throws IOException { + if (dataFileIndexWriter != null) { + dataFileIndexWriter.close(); + } + super.close(); } } diff --git a/paimon-core/src/main/java/org/apache/paimon/io/KeyValueDataFileWriterImpl.java b/paimon-core/src/main/java/org/apache/paimon/io/KeyValueDataFileWriterImpl.java new file mode 100644 index 000000000000..27a1aef64e36 --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/io/KeyValueDataFileWriterImpl.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.io; + +import org.apache.paimon.CoreOptions; +import org.apache.paimon.KeyValue; +import org.apache.paimon.data.InternalRow; +import org.apache.paimon.fileindex.FileIndexOptions; +import org.apache.paimon.format.FormatWriterFactory; +import org.apache.paimon.format.SimpleColStats; +import org.apache.paimon.format.SimpleStatsExtractor; +import org.apache.paimon.fs.FileIO; +import org.apache.paimon.fs.Path; +import org.apache.paimon.manifest.FileSource; +import org.apache.paimon.types.RowType; +import org.apache.paimon.utils.Pair; + +import javax.annotation.Nullable; + +import java.util.Arrays; +import java.util.function.Function; + +/** Write data files containing {@link KeyValue}s. */ +public class KeyValueDataFileWriterImpl extends KeyValueDataFileWriter { + + public KeyValueDataFileWriterImpl( + FileIO fileIO, + FormatWriterFactory factory, + Path path, + Function converter, + RowType keyType, + RowType valueType, + @Nullable SimpleStatsExtractor simpleStatsExtractor, + long schemaId, + int level, + String compression, + CoreOptions options, + FileSource fileSource, + FileIndexOptions fileIndexOptions) { + super( + fileIO, + factory, + path, + converter, + keyType, + valueType, + KeyValue.schema(keyType, valueType), + simpleStatsExtractor, + schemaId, + level, + compression, + options, + fileSource, + fileIndexOptions); + } + + @Override + Pair fetchKeyValueStats(SimpleColStats[] rowStats) { + int numKeyFields = keyType.getFieldCount(); + return Pair.of( + Arrays.copyOfRange(rowStats, 0, numKeyFields), + Arrays.copyOfRange(rowStats, numKeyFields + 2, rowStats.length)); + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/io/KeyValueFileReaderFactory.java b/paimon-core/src/main/java/org/apache/paimon/io/KeyValueFileReaderFactory.java index fdbb727e5674..7e272fc97c65 100644 --- a/paimon-core/src/main/java/org/apache/paimon/io/KeyValueFileReaderFactory.java +++ b/paimon-core/src/main/java/org/apache/paimon/io/KeyValueFileReaderFactory.java @@ -32,6 +32,7 @@ import org.apache.paimon.fs.Path; import org.apache.paimon.partition.PartitionUtils; import org.apache.paimon.predicate.Predicate; +import org.apache.paimon.reader.FileRecordReader; import org.apache.paimon.reader.RecordReader; import org.apache.paimon.schema.KeyValueFieldsExtractor; import org.apache.paimon.schema.SchemaManager; @@ -39,9 +40,8 @@ import org.apache.paimon.types.DataField; import org.apache.paimon.types.RowType; import org.apache.paimon.utils.AsyncRecordReader; -import org.apache.paimon.utils.BulkFormatMapping; -import org.apache.paimon.utils.BulkFormatMapping.BulkFormatMappingBuilder; import org.apache.paimon.utils.FileStorePathFactory; +import org.apache.paimon.utils.FormatReaderMapping; import javax.annotation.Nullable; @@ -63,11 +63,11 @@ public class KeyValueFileReaderFactory implements FileReaderFactory { private final RowType keyType; private final RowType valueType; - private final BulkFormatMappingBuilder bulkFormatMappingBuilder; + private final FormatReaderMapping.Builder formatReaderMappingBuilder; private final DataFilePathFactory pathFactory; private final long asyncThreshold; - private final Map bulkFormatMappings; + private final Map formatReaderMappings; private final BinaryRow partition; private final DeletionVector.Factory dvFactory; @@ -77,7 +77,7 @@ private KeyValueFileReaderFactory( TableSchema schema, RowType keyType, RowType valueType, - BulkFormatMappingBuilder bulkFormatMappingBuilder, + FormatReaderMapping.Builder formatReaderMappingBuilder, DataFilePathFactory pathFactory, long asyncThreshold, BinaryRow partition, @@ -87,11 +87,11 @@ private KeyValueFileReaderFactory( this.schema = schema; this.keyType = keyType; this.valueType = valueType; - this.bulkFormatMappingBuilder = bulkFormatMappingBuilder; + this.formatReaderMappingBuilder = formatReaderMappingBuilder; this.pathFactory = pathFactory; this.asyncThreshold = asyncThreshold; this.partition = partition; - this.bulkFormatMappings = new HashMap<>(); + this.formatReaderMappings = new HashMap<>(); this.dvFactory = dvFactory; } @@ -109,7 +109,7 @@ public RecordReader createRecordReader( return createRecordReader(schemaId, fileName, level, true, null, fileSize); } - private RecordReader createRecordReader( + private FileRecordReader createRecordReader( long schemaId, String fileName, int level, @@ -119,31 +119,31 @@ private RecordReader createRecordReader( throws IOException { String formatIdentifier = DataFilePathFactory.formatIdentifier(fileName); - Supplier formatSupplier = + Supplier formatSupplier = () -> - bulkFormatMappingBuilder.build( + formatReaderMappingBuilder.build( formatIdentifier, schema, schemaId == schema.id() ? schema : schemaManager.schema(schemaId)); - BulkFormatMapping bulkFormatMapping = + FormatReaderMapping formatReaderMapping = reuseFormat - ? bulkFormatMappings.computeIfAbsent( + ? formatReaderMappings.computeIfAbsent( new FormatKey(schemaId, formatIdentifier), key -> formatSupplier.get()) : formatSupplier.get(); Path filePath = pathFactory.toPath(fileName); - RecordReader fileRecordReader = - new FileRecordReader( - bulkFormatMapping.getReaderFactory(), + FileRecordReader fileRecordReader = + new DataFileRecordReader( + formatReaderMapping.getReaderFactory(), orcPoolSize == null ? new FormatReaderContext(fileIO, filePath, fileSize) : new OrcFormatReaderContext( fileIO, filePath, fileSize, orcPoolSize), - bulkFormatMapping.getIndexMapping(), - bulkFormatMapping.getCastMapping(), - PartitionUtils.create(bulkFormatMapping.getPartitionPair(), partition)); + formatReaderMapping.getIndexMapping(), + formatReaderMapping.getCastMapping(), + PartitionUtils.create(formatReaderMapping.getPartitionPair(), partition)); Optional deletionVector = dvFactory.create(fileName); if (deletionVector.isPresent() && !deletionVector.get().isEmpty()) { @@ -275,7 +275,7 @@ public KeyValueFileReaderFactory build( schema, finalReadKeyType, readValueType, - new BulkFormatMappingBuilder( + new FormatReaderMapping.Builder( formatDiscover, readTableFields, fieldsExtractor, filters), pathFactory.createDataFilePathFactory(partition, bucket), options.fileReaderAsyncThreshold().getBytes(), diff --git a/paimon-core/src/main/java/org/apache/paimon/io/KeyValueFileWriterFactory.java b/paimon-core/src/main/java/org/apache/paimon/io/KeyValueFileWriterFactory.java index 922b06ee8229..a6aae3985bd4 100644 --- a/paimon-core/src/main/java/org/apache/paimon/io/KeyValueFileWriterFactory.java +++ b/paimon-core/src/main/java/org/apache/paimon/io/KeyValueFileWriterFactory.java @@ -21,8 +21,10 @@ import org.apache.paimon.CoreOptions; import org.apache.paimon.KeyValue; import org.apache.paimon.KeyValueSerializer; +import org.apache.paimon.KeyValueThinSerializer; import org.apache.paimon.annotation.VisibleForTesting; import org.apache.paimon.data.BinaryRow; +import org.apache.paimon.fileindex.FileIndexOptions; import org.apache.paimon.format.FileFormat; import org.apache.paimon.format.FormatWriterFactory; import org.apache.paimon.format.SimpleStatsExtractor; @@ -30,6 +32,8 @@ import org.apache.paimon.fs.Path; import org.apache.paimon.manifest.FileSource; import org.apache.paimon.statistics.SimpleColStatsCollector; +import org.apache.paimon.table.SpecialFields; +import org.apache.paimon.types.DataField; import org.apache.paimon.types.RowType; import org.apache.paimon.utils.FileStorePathFactory; import org.apache.paimon.utils.StatsCollectorFactories; @@ -37,10 +41,13 @@ import javax.annotation.Nullable; import java.io.IOException; +import java.util.Collections; import java.util.HashMap; import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.function.Function; +import java.util.stream.Collectors; /** A factory to create {@link FileWriter}s for writing {@link KeyValue} files. */ public class KeyValueFileWriterFactory { @@ -52,22 +59,22 @@ public class KeyValueFileWriterFactory { private final WriteFormatContext formatContext; private final long suggestedFileSize; private final CoreOptions options; + private final FileIndexOptions fileIndexOptions; private KeyValueFileWriterFactory( FileIO fileIO, long schemaId, - RowType keyType, - RowType valueType, WriteFormatContext formatContext, long suggestedFileSize, CoreOptions options) { this.fileIO = fileIO; this.schemaId = schemaId; - this.keyType = keyType; - this.valueType = valueType; + this.keyType = formatContext.keyType; + this.valueType = formatContext.valueType; this.formatContext = formatContext; this.suggestedFileSize = suggestedFileSize; this.options = options; + this.fileIndexOptions = options.indexColumnsOptions(); } public RowType keyType() { @@ -104,20 +111,35 @@ public RollingFileWriter createRollingChangelogFileWrite private KeyValueDataFileWriter createDataFileWriter( Path path, int level, FileSource fileSource) { - KeyValueSerializer kvSerializer = new KeyValueSerializer(keyType, valueType); - return new KeyValueDataFileWriter( - fileIO, - formatContext.writerFactory(level), - path, - kvSerializer::toRow, - keyType, - valueType, - formatContext.extractor(level), - schemaId, - level, - formatContext.compression(level), - options, - fileSource); + return formatContext.thinModeEnabled() + ? new KeyValueThinDataFileWriterImpl( + fileIO, + formatContext.writerFactory(level), + path, + new KeyValueThinSerializer(keyType, valueType)::toRow, + keyType, + valueType, + formatContext.extractor(level), + schemaId, + level, + formatContext.compression(level), + options, + fileSource, + fileIndexOptions) + : new KeyValueDataFileWriterImpl( + fileIO, + formatContext.writerFactory(level), + path, + new KeyValueSerializer(keyType, valueType)::toRow, + keyType, + valueType, + formatContext.extractor(level), + schemaId, + level, + formatContext.compression(level), + options, + fileSource, + fileIndexOptions); } public void deleteFile(String filename, int level) { @@ -187,17 +209,17 @@ private Builder( public KeyValueFileWriterFactory build( BinaryRow partition, int bucket, CoreOptions options) { - RowType fileRowType = KeyValue.schema(keyType, valueType); WriteFormatContext context = new WriteFormatContext( partition, bucket, - fileRowType, + keyType, + valueType, fileFormat, format2PathFactory, options); return new KeyValueFileWriterFactory( - fileIO, schemaId, keyType, valueType, context, suggestedFileSize, options); + fileIO, schemaId, context, suggestedFileSize, options); } } @@ -210,13 +232,24 @@ private static class WriteFormatContext { private final Map format2PathFactory; private final Map format2WriterFactory; + private final RowType keyType; + private final RowType valueType; + private final boolean thinModeEnabled; + private WriteFormatContext( BinaryRow partition, int bucket, - RowType rowType, + RowType keyType, + RowType valueType, FileFormat defaultFormat, Map parentFactories, CoreOptions options) { + this.keyType = keyType; + this.valueType = valueType; + this.thinModeEnabled = + options.dataFileThinMode() && supportsThinMode(keyType, valueType); + RowType writeRowType = + KeyValue.schema(thinModeEnabled ? RowType.of() : keyType, valueType); Map fileFormatPerLevel = options.fileFormatPerLevel(); this.level2Format = level -> @@ -232,7 +265,10 @@ private WriteFormatContext( this.format2PathFactory = new HashMap<>(); this.format2WriterFactory = new HashMap<>(); SimpleColStatsCollector.Factory[] statsCollectorFactories = - StatsCollectorFactories.createStatsFactories(options, rowType.getFieldNames()); + StatsCollectorFactories.createStatsFactories( + options, + writeRowType.getFieldNames(), + thinModeEnabled ? keyType.getFieldNames() : Collections.emptyList()); for (String format : parentFactories.keySet()) { format2PathFactory.put( format, @@ -248,11 +284,30 @@ private WriteFormatContext( format.equals("avro") ? Optional.empty() : fileFormat.createStatsExtractor( - rowType, statsCollectorFactories)); - format2WriterFactory.put(format, fileFormat.createWriterFactory(rowType)); + writeRowType, statsCollectorFactories)); + format2WriterFactory.put(format, fileFormat.createWriterFactory(writeRowType)); } } + private boolean supportsThinMode(RowType keyType, RowType valueType) { + Set keyFieldIds = + valueType.getFields().stream().map(DataField::id).collect(Collectors.toSet()); + + for (DataField field : keyType.getFields()) { + if (!SpecialFields.isKeyField(field.name())) { + return false; + } + if (!keyFieldIds.contains(field.id() - SpecialFields.KEY_FIELD_ID_START)) { + return false; + } + } + return true; + } + + private boolean thinModeEnabled() { + return thinModeEnabled; + } + @Nullable private SimpleStatsExtractor extractor(int level) { return format2Extractor.get(level2Format.apply(level)).orElse(null); diff --git a/paimon-core/src/main/java/org/apache/paimon/io/KeyValueThinDataFileWriterImpl.java b/paimon-core/src/main/java/org/apache/paimon/io/KeyValueThinDataFileWriterImpl.java new file mode 100644 index 000000000000..dd7ebb006764 --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/io/KeyValueThinDataFileWriterImpl.java @@ -0,0 +1,128 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.io; + +import org.apache.paimon.CoreOptions; +import org.apache.paimon.KeyValue; +import org.apache.paimon.data.InternalRow; +import org.apache.paimon.fileindex.FileIndexOptions; +import org.apache.paimon.format.FormatWriterFactory; +import org.apache.paimon.format.SimpleColStats; +import org.apache.paimon.format.SimpleStatsExtractor; +import org.apache.paimon.fs.FileIO; +import org.apache.paimon.fs.Path; +import org.apache.paimon.manifest.FileSource; +import org.apache.paimon.table.SpecialFields; +import org.apache.paimon.types.RowType; +import org.apache.paimon.utils.Pair; + +import javax.annotation.Nullable; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; +import java.util.function.Function; + +/** + * Implementation of KeyValueDataFileWriter for thin data files. Thin data files only contain + * _SEQUENCE_NUMBER_, _ROW_KIND_ and value fields. + */ +public class KeyValueThinDataFileWriterImpl extends KeyValueDataFileWriter { + + private final int[] keyStatMapping; + + /** + * Constructs a KeyValueThinDataFileWriterImpl. + * + * @param fileIO The file IO interface. + * @param factory The format writer factory. + * @param path The path to the file. + * @param converter The function to convert KeyValue to InternalRow. + * @param keyType The row type of the key. + * @param valueType The row type of the value. + * @param simpleStatsExtractor The simple stats extractor, can be null. + * @param schemaId The schema ID. + * @param level The level. + * @param compression The compression type. + * @param options The core options. + * @param fileSource The file source. + * @param fileIndexOptions The file index options. + */ + public KeyValueThinDataFileWriterImpl( + FileIO fileIO, + FormatWriterFactory factory, + Path path, + Function converter, + RowType keyType, + RowType valueType, + @Nullable SimpleStatsExtractor simpleStatsExtractor, + long schemaId, + int level, + String compression, + CoreOptions options, + FileSource fileSource, + FileIndexOptions fileIndexOptions) { + super( + fileIO, + factory, + path, + converter, + keyType, + valueType, + KeyValue.schema(RowType.of(), valueType), + simpleStatsExtractor, + schemaId, + level, + compression, + options, + fileSource, + fileIndexOptions); + Map idToIndex = new HashMap<>(valueType.getFieldCount()); + for (int i = 0; i < valueType.getFieldCount(); i++) { + idToIndex.put(valueType.getFields().get(i).id(), i); + } + this.keyStatMapping = new int[keyType.getFieldCount()]; + for (int i = 0; i < keyType.getFieldCount(); i++) { + keyStatMapping[i] = + idToIndex.get( + keyType.getFields().get(i).id() - SpecialFields.KEY_FIELD_ID_START); + } + } + + /** + * Fetches the key and value statistics. + * + * @param rowStats The row statistics. + * @return A pair of key statistics and value statistics. + */ + @Override + Pair fetchKeyValueStats(SimpleColStats[] rowStats) { + int numKeyFields = keyType.getFieldCount(); + // In thin mode, there is no key stats in rowStats, so we only jump + // _SEQUNCE_NUMBER_ and _ROW_KIND_ stats. Therefore, the 'from' value is 2. + SimpleColStats[] valFieldStats = Arrays.copyOfRange(rowStats, 2, rowStats.length); + // Thin mode on, so need to map value stats to key stats. + SimpleColStats[] keyStats = new SimpleColStats[numKeyFields]; + for (int i = 0; i < keyStatMapping.length; i++) { + keyStats[i] = valFieldStats[keyStatMapping[i]]; + } + + return Pair.of(keyStats, valFieldStats); + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/io/RecordLevelExpire.java b/paimon-core/src/main/java/org/apache/paimon/io/RecordLevelExpire.java index c1fef547b861..e43a9d03d9b0 100644 --- a/paimon-core/src/main/java/org/apache/paimon/io/RecordLevelExpire.java +++ b/paimon-core/src/main/java/org/apache/paimon/io/RecordLevelExpire.java @@ -20,24 +20,28 @@ import org.apache.paimon.CoreOptions; import org.apache.paimon.KeyValue; +import org.apache.paimon.data.InternalRow; import org.apache.paimon.reader.RecordReader; import org.apache.paimon.types.BigIntType; -import org.apache.paimon.types.DataField; +import org.apache.paimon.types.DataType; +import org.apache.paimon.types.DataTypeChecks; import org.apache.paimon.types.IntType; +import org.apache.paimon.types.LocalZonedTimestampType; import org.apache.paimon.types.RowType; +import org.apache.paimon.types.TimestampType; import javax.annotation.Nullable; import java.time.Duration; +import java.util.function.Function; import static org.apache.paimon.utils.Preconditions.checkArgument; /** A factory to create {@link RecordReader} expires records by time. */ public class RecordLevelExpire { - private final int timeField; private final int expireTime; - private final CoreOptions.TimeFieldType timeFieldType; + private final Function fieldGetter; @Nullable public static RecordLevelExpire create(CoreOptions options, RowType rowType) { @@ -46,43 +50,28 @@ public static RecordLevelExpire create(CoreOptions options, RowType rowType) { return null; } - String timeField = options.recordLevelTimeField(); - if (timeField == null) { + String timeFieldName = options.recordLevelTimeField(); + if (timeFieldName == null) { throw new IllegalArgumentException( "You should set time field for record-level expire."); } // should no project here, record level expire only works in compaction - int fieldIndex = rowType.getFieldIndex(timeField); + int fieldIndex = rowType.getFieldIndex(timeFieldName); if (fieldIndex == -1) { throw new IllegalArgumentException( String.format( - "Can not find time field %s for record level expire.", timeField)); + "Can not find time field %s for record level expire.", timeFieldName)); } - CoreOptions.TimeFieldType timeFieldType = options.recordLevelTimeFieldType(); - DataField field = rowType.getField(timeField); - if (!((timeFieldType == CoreOptions.TimeFieldType.SECONDS_INT - && field.type() instanceof IntType) - || (timeFieldType == CoreOptions.TimeFieldType.SECONDS_LONG - && field.type() instanceof BigIntType) - || (timeFieldType == CoreOptions.TimeFieldType.MILLIS_LONG - && field.type() instanceof BigIntType))) { - throw new IllegalArgumentException( - String.format( - "The record level time field type should be one of SECONDS_INT,SECONDS_LONG or MILLIS_LONG, " - + "but time field type is %s, field type is %s.", - timeFieldType, field.type())); - } - - return new RecordLevelExpire(fieldIndex, (int) expireTime.getSeconds(), timeFieldType); + DataType dataType = rowType.getField(timeFieldName).type(); + Function fieldGetter = createFieldGetter(dataType, fieldIndex); + return new RecordLevelExpire((int) expireTime.getSeconds(), fieldGetter); } - private RecordLevelExpire( - int timeField, int expireTime, CoreOptions.TimeFieldType timeFieldType) { - this.timeField = timeField; + private RecordLevelExpire(int expireTime, Function fieldGetter) { this.expireTime = expireTime; - this.timeFieldType = timeFieldType; + this.fieldGetter = fieldGetter; } public FileReaderFactory wrap(FileReaderFactory readerFactory) { @@ -91,31 +80,38 @@ public FileReaderFactory wrap(FileReaderFactory readerFactor private RecordReader wrap(RecordReader reader) { int currentTime = (int) (System.currentTimeMillis() / 1000); - return reader.filter( - kv -> { - checkArgument( - !kv.value().isNullAt(timeField), - "Time field for record-level expire should not be null."); - final int recordTime; - switch (timeFieldType) { - case SECONDS_INT: - recordTime = kv.value().getInt(timeField); - break; - case SECONDS_LONG: - recordTime = (int) kv.value().getLong(timeField); - break; - case MILLIS_LONG: - recordTime = (int) (kv.value().getLong(timeField) / 1000); - break; - default: - String msg = - String.format( - "type %s not support in %s", - timeFieldType, - CoreOptions.TimeFieldType.class.getName()); - throw new IllegalArgumentException(msg); - } - return currentTime <= recordTime + expireTime; - }); + return reader.filter(kv -> currentTime <= fieldGetter.apply(kv.value()) + expireTime); + } + + private static Function createFieldGetter( + DataType dataType, int fieldIndex) { + final Function fieldGetter; + if (dataType instanceof IntType) { + fieldGetter = row -> row.getInt(fieldIndex); + } else if (dataType instanceof BigIntType) { + fieldGetter = + row -> { + long value = row.getLong(fieldIndex); + // If it is milliseconds, convert it to seconds. + return (int) (value >= 1_000_000_000_000L ? value / 1000 : value); + }; + } else if (dataType instanceof TimestampType + || dataType instanceof LocalZonedTimestampType) { + int precision = DataTypeChecks.getPrecision(dataType); + fieldGetter = + row -> (int) (row.getTimestamp(fieldIndex, precision).getMillisecond() / 1000); + } else { + throw new IllegalArgumentException( + String.format( + "The record level time field type should be one of INT, BIGINT, or TIMESTAMP, but field type is %s.", + dataType)); + } + + return row -> { + checkArgument( + !row.isNullAt(fieldIndex), + "Time field for record-level expire should not be null."); + return fieldGetter.apply(row); + }; } } diff --git a/paimon-core/src/main/java/org/apache/paimon/io/RollingFileWriter.java b/paimon-core/src/main/java/org/apache/paimon/io/RollingFileWriter.java index 109b7574304e..29b9223b9a37 100644 --- a/paimon-core/src/main/java/org/apache/paimon/io/RollingFileWriter.java +++ b/paimon-core/src/main/java/org/apache/paimon/io/RollingFileWriter.java @@ -64,10 +64,9 @@ public long targetFileSize() { return targetFileSize; } - @VisibleForTesting - boolean rollingFile() throws IOException { + private boolean rollingFile(boolean forceCheck) throws IOException { return currentWriter.reachTargetSize( - recordCount % CHECK_ROLLING_RECORD_CNT == 0, targetFileSize); + forceCheck || recordCount % CHECK_ROLLING_RECORD_CNT == 0, targetFileSize); } @Override @@ -81,7 +80,7 @@ public void write(T row) throws IOException { currentWriter.write(row); recordCount += 1; - if (rollingFile()) { + if (rollingFile(false)) { closeCurrentWriter(); } } catch (Throwable e) { @@ -105,7 +104,7 @@ public void writeBundle(BundleRecords bundle) throws IOException { currentWriter.writeBundle(bundle); recordCount += bundle.rowCount(); - if (rollingFile()) { + if (rollingFile(true)) { closeCurrentWriter(); } } catch (Throwable e) { diff --git a/paimon-core/src/main/java/org/apache/paimon/io/RowDataFileWriter.java b/paimon-core/src/main/java/org/apache/paimon/io/RowDataFileWriter.java index f9c9b950214f..8c2e8ec9498c 100644 --- a/paimon-core/src/main/java/org/apache/paimon/io/RowDataFileWriter.java +++ b/paimon-core/src/main/java/org/apache/paimon/io/RowDataFileWriter.java @@ -30,11 +30,13 @@ import org.apache.paimon.stats.SimpleStatsConverter; import org.apache.paimon.types.RowType; import org.apache.paimon.utils.LongCounter; +import org.apache.paimon.utils.Pair; import javax.annotation.Nullable; import java.io.IOException; import java.util.Collections; +import java.util.List; import java.util.function.Function; import static org.apache.paimon.io.DataFilePathFactory.dataFileToFileIndexPath; @@ -63,7 +65,8 @@ public RowDataFileWriter( SimpleColStatsCollector.Factory[] statsCollectors, FileIndexOptions fileIndexOptions, FileSource fileSource, - boolean asyncFileWrite) { + boolean asyncFileWrite, + boolean statsDenseStore) { super( fileIO, factory, @@ -76,7 +79,7 @@ public RowDataFileWriter( asyncFileWrite); this.schemaId = schemaId; this.seqNumCounter = seqNumCounter; - this.statsArraySerializer = new SimpleStatsConverter(writeSchema); + this.statsArraySerializer = new SimpleStatsConverter(writeSchema, statsDenseStore); this.dataFileIndexWriter = DataFileIndexWriter.create( fileIO, dataFileToFileIndexPath(path), writeSchema, fileIndexOptions); @@ -103,7 +106,7 @@ public void close() throws IOException { @Override public DataFileMeta result() throws IOException { - SimpleStats stats = statsArraySerializer.toBinary(fieldStats()); + Pair, SimpleStats> statsPair = statsArraySerializer.toBinary(fieldStats()); DataFileIndexWriter.FileIndexResult indexResult = dataFileIndexWriter == null ? DataFileIndexWriter.EMPTY_RESULT @@ -112,7 +115,7 @@ public DataFileMeta result() throws IOException { path.getName(), fileIO.getFileSize(path), recordCount(), - stats, + statsPair.getRight(), seqNumCounter.getValue() - super.recordCount(), seqNumCounter.getValue() - 1, schemaId, @@ -120,6 +123,7 @@ public DataFileMeta result() throws IOException { ? Collections.emptyList() : Collections.singletonList(indexResult.independentIndexFile()), indexResult.embeddedIndexBytes(), - fileSource); + fileSource, + statsPair.getKey()); } } diff --git a/paimon-core/src/main/java/org/apache/paimon/io/RowDataRollingFileWriter.java b/paimon-core/src/main/java/org/apache/paimon/io/RowDataRollingFileWriter.java index e60913d25f87..b929a4ae22af 100644 --- a/paimon-core/src/main/java/org/apache/paimon/io/RowDataRollingFileWriter.java +++ b/paimon-core/src/main/java/org/apache/paimon/io/RowDataRollingFileWriter.java @@ -43,7 +43,8 @@ public RowDataRollingFileWriter( SimpleColStatsCollector.Factory[] statsCollectors, FileIndexOptions fileIndexOptions, FileSource fileSource, - boolean asyncFileWrite) { + boolean asyncFileWrite, + boolean statsDenseStore) { super( () -> new RowDataFileWriter( @@ -62,7 +63,8 @@ public RowDataRollingFileWriter( statsCollectors, fileIndexOptions, fileSource, - asyncFileWrite), + asyncFileWrite, + statsDenseStore), targetFileSize); } } diff --git a/paimon-core/src/main/java/org/apache/paimon/io/SingleFileWriter.java b/paimon-core/src/main/java/org/apache/paimon/io/SingleFileWriter.java index d41040e05bb7..f303e8597870 100644 --- a/paimon-core/src/main/java/org/apache/paimon/io/SingleFileWriter.java +++ b/paimon-core/src/main/java/org/apache/paimon/io/SingleFileWriter.java @@ -49,7 +49,7 @@ public abstract class SingleFileWriter implements FileWriter { protected final Path path; private final Function converter; - private final FormatWriter writer; + private FormatWriter writer; private PositionOutputStream out; private long recordCount; @@ -144,7 +144,14 @@ public boolean reachTargetSize(boolean suggestedCheck, long targetSize) throws I @Override public void abort() { - IOUtils.closeQuietly(out); + if (writer != null) { + IOUtils.closeQuietly(writer); + writer = null; + } + if (out != null) { + IOUtils.closeQuietly(out); + out = null; + } fileIO.deleteQuietly(path); } @@ -167,9 +174,15 @@ public void close() throws IOException { } try { - writer.close(); - out.flush(); - out.close(); + if (writer != null) { + writer.close(); + writer = null; + } + if (out != null) { + out.flush(); + out.close(); + out = null; + } } catch (IOException e) { LOG.warn("Exception occurs when closing file {}. Cleaning up.", path, e); abort(); diff --git a/paimon-core/src/main/java/org/apache/paimon/io/StatsCollectingSingleFileWriter.java b/paimon-core/src/main/java/org/apache/paimon/io/StatsCollectingSingleFileWriter.java index 2f4190a049dc..67a3fa6d1ace 100644 --- a/paimon-core/src/main/java/org/apache/paimon/io/StatsCollectingSingleFileWriter.java +++ b/paimon-core/src/main/java/org/apache/paimon/io/StatsCollectingSingleFileWriter.java @@ -25,6 +25,7 @@ import org.apache.paimon.format.SimpleStatsExtractor; import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.Path; +import org.apache.paimon.statistics.NoneSimpleColStatsCollector; import org.apache.paimon.statistics.SimpleColStatsCollector; import org.apache.paimon.types.RowType; import org.apache.paimon.utils.Preconditions; @@ -32,7 +33,9 @@ import javax.annotation.Nullable; import java.io.IOException; +import java.util.Arrays; import java.util.function.Function; +import java.util.stream.IntStream; /** * A {@link SingleFileWriter} which also produces statistics for each written field. @@ -44,6 +47,8 @@ public abstract class StatsCollectingSingleFileWriter extends SingleFileWr @Nullable private final SimpleStatsExtractor simpleStatsExtractor; @Nullable private SimpleStatsCollector simpleStatsCollector = null; + @Nullable private SimpleColStats[] noneStats = null; + private final boolean isStatsDisabled; public StatsCollectingSingleFileWriter( FileIO fileIO, @@ -63,6 +68,15 @@ public StatsCollectingSingleFileWriter( Preconditions.checkArgument( statsCollectors.length == writeSchema.getFieldCount(), "The stats collector is not aligned to write schema."); + this.isStatsDisabled = + Arrays.stream(SimpleColStatsCollector.create(statsCollectors)) + .allMatch(p -> p instanceof NoneSimpleColStatsCollector); + if (isStatsDisabled) { + this.noneStats = + IntStream.range(0, statsCollectors.length) + .mapToObj(i -> SimpleColStats.NONE) + .toArray(SimpleColStats[]::new); + } } @Override @@ -85,7 +99,11 @@ public void writeBundle(BundleRecords bundle) throws IOException { public SimpleColStats[] fieldStats() throws IOException { Preconditions.checkState(closed, "Cannot access metric unless the writer is closed."); if (simpleStatsExtractor != null) { - return simpleStatsExtractor.extract(fileIO, path); + if (isStatsDisabled) { + return noneStats; + } else { + return simpleStatsExtractor.extract(fileIO, path); + } } else { return simpleStatsCollector.extract(); } diff --git a/paimon-core/src/main/java/org/apache/paimon/jdbc/JdbcCatalog.java b/paimon-core/src/main/java/org/apache/paimon/jdbc/JdbcCatalog.java index e52bc9d6b173..778bc591fe89 100644 --- a/paimon-core/src/main/java/org/apache/paimon/jdbc/JdbcCatalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/jdbc/JdbcCatalog.java @@ -22,6 +22,7 @@ import org.apache.paimon.catalog.AbstractCatalog; import org.apache.paimon.catalog.CatalogLockContext; import org.apache.paimon.catalog.CatalogLockFactory; +import org.apache.paimon.catalog.Database; import org.apache.paimon.catalog.Identifier; import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.Path; @@ -51,6 +52,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.stream.Collectors; import static org.apache.paimon.jdbc.JdbcCatalogLock.acquireTimeout; import static org.apache.paimon.jdbc.JdbcCatalogLock.checkMaxSleep; @@ -155,22 +157,21 @@ public List listDatabases() { row -> row.getString(JdbcUtils.DATABASE_NAME), JdbcUtils.LIST_ALL_PROPERTY_DATABASES_SQL, catalogKey)); - return databases; + return databases.stream().distinct().collect(Collectors.toList()); } @Override - protected Map loadDatabasePropertiesImpl(String databaseName) - throws DatabaseNotExistException { + protected Database getDatabaseImpl(String databaseName) throws DatabaseNotExistException { if (!JdbcUtils.databaseExists(connections, catalogKey, databaseName)) { throw new DatabaseNotExistException(databaseName); } - Map properties = Maps.newHashMap(); - properties.putAll(fetchProperties(databaseName)); - if (!properties.containsKey(DB_LOCATION_PROP)) { - properties.put(DB_LOCATION_PROP, newDatabasePath(databaseName).getName()); + Map options = Maps.newHashMap(); + options.putAll(fetchProperties(databaseName)); + if (!options.containsKey(DB_LOCATION_PROP)) { + options.put(DB_LOCATION_PROP, newDatabasePath(databaseName).getName()); } - properties.remove(DATABASE_EXISTS_PROPERTY); - return ImmutableMap.copyOf(properties); + options.remove(DATABASE_EXISTS_PROPERTY); + return Database.of(databaseName, options, null); } @Override diff --git a/paimon-core/src/main/java/org/apache/paimon/lookup/RocksDBState.java b/paimon-core/src/main/java/org/apache/paimon/lookup/RocksDBState.java index 25e58984edbe..0181917a7a26 100644 --- a/paimon-core/src/main/java/org/apache/paimon/lookup/RocksDBState.java +++ b/paimon-core/src/main/java/org/apache/paimon/lookup/RocksDBState.java @@ -113,7 +113,8 @@ public static BinaryExternalSortBuffer createBulkLoadSorter( options.pageSize(), options.localSortMaxNumFileHandles(), options.spillCompressOptions(), - options.writeBufferSpillDiskSize()); + options.writeBufferSpillDiskSize(), + options.sequenceFieldSortOrderIsAscending()); } /** A class wraps byte[] to implement equals and hashCode. */ diff --git a/paimon-core/src/main/java/org/apache/paimon/manifest/ExpireFileEntry.java b/paimon-core/src/main/java/org/apache/paimon/manifest/ExpireFileEntry.java new file mode 100644 index 000000000000..060360623cd0 --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/manifest/ExpireFileEntry.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.manifest; + +import org.apache.paimon.data.BinaryRow; + +import javax.annotation.Nullable; + +import java.util.List; +import java.util.Objects; +import java.util.Optional; + +/** A {@link SimpleFileEntry} with {@link #fileSource}. */ +public class ExpireFileEntry extends SimpleFileEntry { + + @Nullable private final FileSource fileSource; + + public ExpireFileEntry( + FileKind kind, + BinaryRow partition, + int bucket, + int level, + String fileName, + List extraFiles, + @Nullable byte[] embeddedIndex, + BinaryRow minKey, + BinaryRow maxKey, + @Nullable FileSource fileSource) { + super(kind, partition, bucket, level, fileName, extraFiles, embeddedIndex, minKey, maxKey); + this.fileSource = fileSource; + } + + public Optional fileSource() { + return Optional.ofNullable(fileSource); + } + + public static ExpireFileEntry from(ManifestEntry entry) { + return new ExpireFileEntry( + entry.kind(), + entry.partition(), + entry.bucket(), + entry.level(), + entry.fileName(), + entry.file().extraFiles(), + entry.file().embeddedIndex(), + entry.minKey(), + entry.maxKey(), + entry.file().fileSource().orElse(null)); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + if (!super.equals(o)) { + return false; + } + ExpireFileEntry that = (ExpireFileEntry) o; + return fileSource == that.fileSource; + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), fileSource); + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/manifest/FileEntry.java b/paimon-core/src/main/java/org/apache/paimon/manifest/FileEntry.java index 3b3e514e0a1e..a2569beac61c 100644 --- a/paimon-core/src/main/java/org/apache/paimon/manifest/FileEntry.java +++ b/paimon-core/src/main/java/org/apache/paimon/manifest/FileEntry.java @@ -28,15 +28,17 @@ import java.util.Arrays; import java.util.Collection; -import java.util.HashSet; +import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; import java.util.function.Function; import java.util.stream.Collectors; +import static org.apache.paimon.utils.ManifestReadThreadPool.randomlyExecuteSequentialReturn; import static org.apache.paimon.utils.ManifestReadThreadPool.sequentialBatchedExecute; /** Entry representing a file. */ @@ -58,6 +60,8 @@ public interface FileEntry { BinaryRow maxKey(); + List extraFiles(); + /** * The same {@link Identifier} indicates that the {@link ManifestEntry} refers to the same data * file. @@ -214,7 +218,11 @@ static Set readDeletedEntries( return readDeletedEntries( m -> manifestFile.read( - m.fileName(), m.fileSize(), Filter.alwaysTrue(), deletedFilter()), + m.fileName(), + m.fileSize(), + Filter.alwaysTrue(), + deletedFilter(), + Filter.alwaysTrue()), manifestFiles, manifestReadParallelism); } @@ -234,11 +242,11 @@ static Set readDeletedEntries( .filter(e -> e.kind() == FileKind.DELETE) .map(FileEntry::identifier) .collect(Collectors.toList()); - Iterable identifiers = - sequentialBatchedExecute(processor, manifestFiles, manifestReadParallelism); - Set result = new HashSet<>(); - for (Identifier identifier : identifiers) { - result.add(identifier); + Iterator identifiers = + randomlyExecuteSequentialReturn(processor, manifestFiles, manifestReadParallelism); + Set result = ConcurrentHashMap.newKeySet(); + while (identifiers.hasNext()) { + result.add(identifiers.next()); } return result; } @@ -247,4 +255,9 @@ static Filter deletedFilter() { Function getter = ManifestEntrySerializer.kindGetter(); return row -> getter.apply(row) == FileKind.DELETE; } + + static Filter addFilter() { + Function getter = ManifestEntrySerializer.kindGetter(); + return row -> getter.apply(row) == FileKind.ADD; + } } diff --git a/paimon-core/src/main/java/org/apache/paimon/manifest/FilteredManifestEntry.java b/paimon-core/src/main/java/org/apache/paimon/manifest/FilteredManifestEntry.java new file mode 100644 index 000000000000..29ae6f6389c6 --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/manifest/FilteredManifestEntry.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.manifest; + +/** Wrap a {@link ManifestEntry} to contain {@link #selected}. */ +public class FilteredManifestEntry extends ManifestEntry { + + private final boolean selected; + + public FilteredManifestEntry(ManifestEntry entry, boolean selected) { + super(entry.kind(), entry.partition(), entry.bucket(), entry.totalBuckets(), entry.file()); + this.selected = selected; + } + + public boolean selected() { + return selected; + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/manifest/IndexManifestEntry.java b/paimon-core/src/main/java/org/apache/paimon/manifest/IndexManifestEntry.java index a52d9e8af40f..2431a1c26412 100644 --- a/paimon-core/src/main/java/org/apache/paimon/manifest/IndexManifestEntry.java +++ b/paimon-core/src/main/java/org/apache/paimon/manifest/IndexManifestEntry.java @@ -20,6 +20,7 @@ import org.apache.paimon.annotation.Public; import org.apache.paimon.data.BinaryRow; +import org.apache.paimon.index.DeletionVectorMeta; import org.apache.paimon.index.IndexFileMeta; import org.apache.paimon.types.ArrayType; import org.apache.paimon.types.BigIntType; @@ -57,12 +58,7 @@ public class IndexManifestEntry { new DataField( 7, "_DELETIONS_VECTORS_RANGES", - new ArrayType( - true, - RowType.of( - newStringType(false), - new IntType(false), - new IntType(false)))))); + new ArrayType(true, DeletionVectorMeta.SCHEMA)))); private final FileKind kind; private final BinaryRow partition; diff --git a/paimon-core/src/main/java/org/apache/paimon/manifest/IndexManifestEntrySerializer.java b/paimon-core/src/main/java/org/apache/paimon/manifest/IndexManifestEntrySerializer.java index 574e935550eb..6f2ec17dda8c 100644 --- a/paimon-core/src/main/java/org/apache/paimon/manifest/IndexManifestEntrySerializer.java +++ b/paimon-core/src/main/java/org/apache/paimon/manifest/IndexManifestEntrySerializer.java @@ -22,10 +22,9 @@ import org.apache.paimon.data.GenericRow; import org.apache.paimon.data.InternalRow; import org.apache.paimon.index.IndexFileMeta; +import org.apache.paimon.index.IndexFileMetaSerializer; import org.apache.paimon.utils.VersionedObjectSerializer; -import static org.apache.paimon.index.IndexFileMetaSerializer.dvRangesToRowArrayData; -import static org.apache.paimon.index.IndexFileMetaSerializer.rowArrayDataToDvRanges; import static org.apache.paimon.utils.SerializationUtils.deserializeBinaryRow; import static org.apache.paimon.utils.SerializationUtils.serializeBinaryRow; @@ -52,9 +51,10 @@ public InternalRow convertTo(IndexManifestEntry record) { BinaryString.fromString(indexFile.fileName()), indexFile.fileSize(), indexFile.rowCount(), - record.indexFile().deletionVectorsRanges() == null + record.indexFile().deletionVectorMetas() == null ? null - : dvRangesToRowArrayData(record.indexFile().deletionVectorsRanges())); + : IndexFileMetaSerializer.dvMetasToRowArrayData( + record.indexFile().deletionVectorMetas().values())); } @Override @@ -72,6 +72,8 @@ public IndexManifestEntry convertFrom(int version, InternalRow row) { row.getString(4).toString(), row.getLong(5), row.getLong(6), - row.isNullAt(7) ? null : rowArrayDataToDvRanges(row.getArray(7)))); + row.isNullAt(7) + ? null + : IndexFileMetaSerializer.rowArrayDataToDvMetas(row.getArray(7)))); } } diff --git a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestCommittable.java b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestCommittable.java index 61c4619bd6d6..b4abd0e9ec0e 100644 --- a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestCommittable.java +++ b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestCommittable.java @@ -62,13 +62,14 @@ public void addFileCommittable(CommitMessage commitMessage) { commitMessages.add(commitMessage); } - public void addLogOffset(int bucket, long offset) { - if (logOffsets.containsKey(bucket)) { + public void addLogOffset(int bucket, long offset, boolean allowDuplicate) { + if (!allowDuplicate && logOffsets.containsKey(bucket)) { throw new RuntimeException( String.format( "bucket-%d appears multiple times, which is not possible.", bucket)); } - logOffsets.put(bucket, offset); + long newOffset = Math.max(logOffsets.getOrDefault(bucket, offset), offset); + logOffsets.put(bucket, newOffset); } public long identifier() { diff --git a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestEntry.java b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestEntry.java index f7c5c4639a6f..626e0a5d468f 100644 --- a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestEntry.java +++ b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestEntry.java @@ -102,6 +102,11 @@ public BinaryRow maxKey() { return file.maxKey(); } + @Override + public List extraFiles() { + return file.extraFiles(); + } + public int totalBuckets() { return totalBuckets; } @@ -121,6 +126,10 @@ public Identifier identifier() { file.embeddedIndex()); } + public ManifestEntry copyWithoutStats() { + return new ManifestEntry(kind, partition, bucket, totalBuckets, file.copyWithoutStats()); + } + @Override public boolean equals(Object o) { if (!(o instanceof ManifestEntry)) { diff --git a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestFile.java b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestFile.java index 48216b0c474b..1aba2ef19561 100644 --- a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestFile.java +++ b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestFile.java @@ -39,6 +39,7 @@ import javax.annotation.Nullable; import java.io.IOException; +import java.util.ArrayList; import java.util.List; /** @@ -84,6 +85,15 @@ public long suggestedFileSize() { return suggestedFileSize; } + public List readExpireFileEntries(String fileName, @Nullable Long fileSize) { + List entries = read(fileName, fileSize); + List result = new ArrayList<>(entries.size()); + for (ManifestEntry entry : entries) { + result.add(ExpireFileEntry.from(entry)); + } + return result; + } + /** * Write several {@link ManifestEntry}s into manifest files. * @@ -154,7 +164,7 @@ public ManifestFileMeta result() throws IOException { fileIO.getFileSize(path), numAddedFiles, numDeletedFiles, - partitionStatsSerializer.toBinary(partitionStatsCollector.extract()), + partitionStatsSerializer.toBinaryAllMode(partitionStatsCollector.extract()), numAddedFiles + numDeletedFiles > 0 ? schemaId : schemaManager.latest().get().id()); @@ -211,18 +221,5 @@ public ManifestFile create() { suggestedFileSize, cache); } - - public ObjectsFile createSimpleFileEntryReader() { - RowType entryType = VersionedObjectSerializer.versionType(ManifestEntry.SCHEMA); - return new ObjectsFile<>( - fileIO, - new SimpleFileEntrySerializer(), - entryType, - fileFormat.createReaderFactory(entryType), - fileFormat.createWriterFactory(entryType), - compression, - pathFactory.manifestFileFactory(), - cache); - } } } diff --git a/paimon-core/src/main/java/org/apache/paimon/manifest/SimpleFileEntry.java b/paimon-core/src/main/java/org/apache/paimon/manifest/SimpleFileEntry.java index 8d33ede0c4a1..fdaed2b85aaf 100644 --- a/paimon-core/src/main/java/org/apache/paimon/manifest/SimpleFileEntry.java +++ b/paimon-core/src/main/java/org/apache/paimon/manifest/SimpleFileEntry.java @@ -117,6 +117,11 @@ public BinaryRow maxKey() { return maxKey; } + @Override + public List extraFiles() { + return extraFiles; + } + @Override public boolean equals(Object o) { if (this == o) { diff --git a/paimon-core/src/main/java/org/apache/paimon/mergetree/MergeTreeWriter.java b/paimon-core/src/main/java/org/apache/paimon/mergetree/MergeTreeWriter.java index de0a28c33417..f2a964bae16a 100644 --- a/paimon-core/src/main/java/org/apache/paimon/mergetree/MergeTreeWriter.java +++ b/paimon-core/src/main/java/org/apache/paimon/mergetree/MergeTreeWriter.java @@ -279,6 +279,7 @@ public CommitIncrement prepareCommit(boolean waitCompaction) throws Exception { @Override public boolean isCompacting() { + compactManager.triggerCompaction(false); return compactManager.isCompacting(); } @@ -289,6 +290,9 @@ public void sync() throws Exception { @Override public void withInsertOnly(boolean insertOnly) { + if (this.isInsertOnly == insertOnly) { + return; + } if (insertOnly && writeBuffer != null && writeBuffer.size() > 0) { throw new IllegalStateException( "Insert-only can only be set before any record is received."); diff --git a/paimon-core/src/main/java/org/apache/paimon/mergetree/SortBufferWriteBuffer.java b/paimon-core/src/main/java/org/apache/paimon/mergetree/SortBufferWriteBuffer.java index 76c84fd4c937..433ff2158bc9 100644 --- a/paimon-core/src/main/java/org/apache/paimon/mergetree/SortBufferWriteBuffer.java +++ b/paimon-core/src/main/java/org/apache/paimon/mergetree/SortBufferWriteBuffer.java @@ -101,7 +101,7 @@ public SortBufferWriteBuffer( NormalizedKeyComputer normalizedKeyComputer = CodeGenUtils.newNormalizedKeyComputer(fieldTypes, sortFieldArray); RecordComparator keyComparator = - CodeGenUtils.newRecordComparator(fieldTypes, sortFieldArray); + CodeGenUtils.newRecordComparator(fieldTypes, sortFieldArray, true); if (memoryPool.freePages() < 3) { throw new IllegalArgumentException( diff --git a/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/MergeTreeCompactManager.java b/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/MergeTreeCompactManager.java index 28853a12381f..8ae29201301c 100644 --- a/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/MergeTreeCompactManager.java +++ b/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/MergeTreeCompactManager.java @@ -211,6 +211,9 @@ private void submitCompaction(CompactUnit unit, boolean dropDelete) { .collect(Collectors.joining(", "))); } taskFuture = executor.submit(task); + if (metricsReporter != null) { + metricsReporter.increaseCompactionsQueuedCount(); + } } /** Finish current task, and update result files to {@link Levels}. */ diff --git a/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/PartialUpdateMergeFunction.java b/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/PartialUpdateMergeFunction.java index 17d2d937a90e..ab25794129ba 100644 --- a/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/PartialUpdateMergeFunction.java +++ b/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/PartialUpdateMergeFunction.java @@ -23,6 +23,7 @@ import org.apache.paimon.data.GenericRow; import org.apache.paimon.data.InternalRow; import org.apache.paimon.mergetree.compact.aggregate.FieldAggregator; +import org.apache.paimon.mergetree.compact.aggregate.factory.FieldAggregatorFactory; import org.apache.paimon.options.Options; import org.apache.paimon.types.DataField; import org.apache.paimon.types.DataType; @@ -51,7 +52,7 @@ import static org.apache.paimon.CoreOptions.FIELDS_PREFIX; import static org.apache.paimon.CoreOptions.FIELDS_SEPARATOR; import static org.apache.paimon.CoreOptions.PARTIAL_UPDATE_REMOVE_RECORD_ON_DELETE; -import static org.apache.paimon.mergetree.compact.aggregate.FieldAggregator.createFieldAggregator; +import static org.apache.paimon.CoreOptions.PARTIAL_UPDATE_REMOVE_RECORD_ON_SEQUENCE_GROUP; import static org.apache.paimon.utils.InternalRowUtils.createFieldGetters; /** @@ -68,6 +69,7 @@ public class PartialUpdateMergeFunction implements MergeFunction { private final boolean fieldSequenceEnabled; private final Map fieldAggregators; private final boolean removeRecordOnDelete; + private final Set sequenceGroupPartialDelete; private InternalRow currentKey; private long latestSequenceNumber; @@ -81,13 +83,15 @@ protected PartialUpdateMergeFunction( Map fieldSeqComparators, Map fieldAggregators, boolean fieldSequenceEnabled, - boolean removeRecordOnDelete) { + boolean removeRecordOnDelete, + Set sequenceGroupPartialDelete) { this.getters = getters; this.ignoreDelete = ignoreDelete; this.fieldSeqComparators = fieldSeqComparators; this.fieldAggregators = fieldAggregators; this.fieldSequenceEnabled = fieldSequenceEnabled; this.removeRecordOnDelete = removeRecordOnDelete; + this.sequenceGroupPartialDelete = sequenceGroupPartialDelete; } @Override @@ -120,7 +124,6 @@ public void add(KeyValue kv) { currentDeleteRow = true; row = new GenericRow(getters.length); } - // ignore -U records return; } @@ -221,8 +224,15 @@ private void retractWithSequenceGroup(KeyValue kv) { .anyMatch(field -> field == index)) { for (int field : seqComparator.compareFields()) { if (!updatedSequenceFields.contains(field)) { - row.setField(field, getters[field].getFieldOrNull(kv.value())); - updatedSequenceFields.add(field); + if (kv.valueKind() == RowKind.DELETE + && sequenceGroupPartialDelete.contains(field)) { + currentDeleteRow = true; + row = new GenericRow(getters.length); + return; + } else { + row.setField(field, getters[field].getFieldOrNull(kv.value())); + updatedSequenceFields.add(field); + } } } } else { @@ -254,10 +264,9 @@ public KeyValue getResult() { if (reused == null) { reused = new KeyValue(); } - if (currentDeleteRow) { - return null; - } - return reused.replace(currentKey, latestSequenceNumber, RowKind.INSERT, row); + + RowKind rowKind = currentDeleteRow ? RowKind.DELETE : RowKind.INSERT; + return reused.replace(currentKey, latestSequenceNumber, rowKind, row); } public static MergeFunctionFactory factory( @@ -280,13 +289,21 @@ private static class Factory implements MergeFunctionFactory { private final boolean removeRecordOnDelete; + private final String removeRecordOnSequenceGroup; + + private Set sequenceGroupPartialDelete; + private Factory(Options options, RowType rowType, List primaryKeys) { this.ignoreDelete = options.get(CoreOptions.IGNORE_DELETE); this.rowType = rowType; this.tableTypes = rowType.getFieldTypes(); + this.removeRecordOnSequenceGroup = + options.get(PARTIAL_UPDATE_REMOVE_RECORD_ON_SEQUENCE_GROUP); + this.sequenceGroupPartialDelete = new HashSet<>(); List fieldNames = rowType.getFieldNames(); this.fieldSeqComparators = new HashMap<>(); + Map sequenceGroupMap = new HashMap<>(); for (Map.Entry entry : options.toMap().entrySet()) { String k = entry.getKey(); String v = entry.getValue(); @@ -303,7 +320,7 @@ private Factory(Options options, RowType rowType, List primaryKeys) { .collect(Collectors.toList()); Supplier userDefinedSeqComparator = - () -> UserDefinedSeqComparator.create(rowType, sequenceFields); + () -> UserDefinedSeqComparator.create(rowType, sequenceFields, true); Arrays.stream(v.split(FIELDS_SEPARATOR)) .map( fieldName -> @@ -325,6 +342,7 @@ private Factory(Options options, RowType rowType, List primaryKeys) { fieldName -> { int index = fieldNames.indexOf(fieldName); fieldSeqComparators.put(index, userDefinedSeqComparator); + sequenceGroupMap.put(fieldName, index); }); } } @@ -347,6 +365,21 @@ private Factory(Options options, RowType rowType, List primaryKeys) { String.format( "sequence group and %s have conflicting behavior so should not be enabled at the same time.", PARTIAL_UPDATE_REMOVE_RECORD_ON_DELETE)); + + if (removeRecordOnSequenceGroup != null) { + String[] sequenceGroupArr = removeRecordOnSequenceGroup.split(FIELDS_SEPARATOR); + Preconditions.checkState( + sequenceGroupMap.keySet().containsAll(Arrays.asList(sequenceGroupArr)), + String.format( + "field '%s' defined in '%s' option must be part of sequence groups", + removeRecordOnSequenceGroup, + PARTIAL_UPDATE_REMOVE_RECORD_ON_SEQUENCE_GROUP.key())); + sequenceGroupPartialDelete = + Arrays.stream(sequenceGroupArr) + .filter(sequenceGroupMap::containsKey) + .map(sequenceGroupMap::get) + .collect(Collectors.toSet()); + } } @Override @@ -392,7 +425,7 @@ public MergeFunction create(@Nullable int[][] projection) { projectedSeqComparators.put( newField, UserDefinedSeqComparator.create( - newRowType, newSequenceFields)); + newRowType, newSequenceFields, true)); } }); for (int i = 0; i < projects.length; i++) { @@ -407,7 +440,8 @@ public MergeFunction create(@Nullable int[][] projection) { projectedSeqComparators, projectedAggregators, !fieldSeqComparators.isEmpty(), - removeRecordOnDelete); + removeRecordOnDelete, + sequenceGroupPartialDelete); } else { Map fieldSeqComparators = new HashMap<>(); this.fieldSeqComparators.forEach( @@ -421,7 +455,8 @@ public MergeFunction create(@Nullable int[][] projection) { fieldSeqComparators, fieldAggregators, !fieldSeqComparators.isEmpty(), - removeRecordOnDelete); + removeRecordOnDelete, + sequenceGroupPartialDelete); } } @@ -497,7 +532,7 @@ private Map> createFieldAggregators( fieldAggregators.put( i, () -> - createFieldAggregator( + FieldAggregatorFactory.create( fieldType, strAggFunc, ignoreRetract, @@ -508,7 +543,7 @@ private Map> createFieldAggregators( fieldAggregators.put( i, () -> - createFieldAggregator( + FieldAggregatorFactory.create( fieldType, defaultAggFunc, ignoreRetract, diff --git a/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/AggregateMergeFunction.java b/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/AggregateMergeFunction.java index e73bfe8e9acc..bad77ba91da5 100644 --- a/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/AggregateMergeFunction.java +++ b/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/AggregateMergeFunction.java @@ -24,6 +24,7 @@ import org.apache.paimon.data.InternalRow; import org.apache.paimon.mergetree.compact.MergeFunction; import org.apache.paimon.mergetree.compact.MergeFunctionFactory; +import org.apache.paimon.mergetree.compact.aggregate.factory.FieldAggregatorFactory; import org.apache.paimon.options.Options; import org.apache.paimon.types.DataType; import org.apache.paimon.types.RowKind; @@ -142,7 +143,7 @@ public MergeFunction create(@Nullable int[][] projection) { boolean ignoreRetract = options.fieldAggIgnoreRetract(fieldName); fieldAggregators[i] = - FieldAggregator.createFieldAggregator( + FieldAggregatorFactory.create( fieldType, strAggFunc, ignoreRetract, diff --git a/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/FieldAggregator.java b/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/FieldAggregator.java index 2737c691dbdb..cd368a818bdd 100644 --- a/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/FieldAggregator.java +++ b/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/FieldAggregator.java @@ -18,164 +18,23 @@ package org.apache.paimon.mergetree.compact.aggregate; -import org.apache.paimon.CoreOptions; -import org.apache.paimon.types.ArrayType; import org.apache.paimon.types.DataType; -import org.apache.paimon.types.MapType; -import org.apache.paimon.types.RowType; -import org.apache.paimon.types.VarBinaryType; - -import javax.annotation.Nullable; import java.io.Serializable; -import java.util.Collections; -import java.util.List; - -import static org.apache.paimon.utils.Preconditions.checkArgument; /** abstract class of aggregating a field of a row. */ public abstract class FieldAggregator implements Serializable { - protected DataType fieldType; private static final long serialVersionUID = 1L; - public FieldAggregator(DataType dataType) { - this.fieldType = dataType; - } - - public static FieldAggregator createFieldAggregator( - DataType fieldType, - @Nullable String strAgg, - boolean ignoreRetract, - boolean isPrimaryKey, - CoreOptions options, - String field) { - FieldAggregator fieldAggregator; - if (isPrimaryKey) { - fieldAggregator = new FieldPrimaryKeyAgg(fieldType); - } else { - // If the field has no aggregate function, use last_non_null_value. - if (strAgg == null) { - fieldAggregator = new FieldLastNonNullValueAgg(fieldType); - } else { - // ordered by type root definition - switch (strAgg) { - case FieldSumAgg.NAME: - fieldAggregator = new FieldSumAgg(fieldType); - break; - case FieldMaxAgg.NAME: - fieldAggregator = new FieldMaxAgg(fieldType); - break; - case FieldMinAgg.NAME: - fieldAggregator = new FieldMinAgg(fieldType); - break; - case FieldLastNonNullValueAgg.NAME: - fieldAggregator = new FieldLastNonNullValueAgg(fieldType); - break; - case FieldLastValueAgg.NAME: - fieldAggregator = new FieldLastValueAgg(fieldType); - break; - case FieldListaggAgg.NAME: - fieldAggregator = new FieldListaggAgg(fieldType, options, field); - break; - case FieldBoolOrAgg.NAME: - fieldAggregator = new FieldBoolOrAgg(fieldType); - break; - case FieldBoolAndAgg.NAME: - fieldAggregator = new FieldBoolAndAgg(fieldType); - break; - case FieldFirstValueAgg.NAME: - fieldAggregator = new FieldFirstValueAgg(fieldType); - break; - case FieldFirstNonNullValueAgg.NAME: - case FieldFirstNonNullValueAgg.LEGACY_NAME: - fieldAggregator = new FieldFirstNonNullValueAgg(fieldType); - break; - case FieldProductAgg.NAME: - fieldAggregator = new FieldProductAgg(fieldType); - break; - case FieldNestedUpdateAgg.NAME: - fieldAggregator = - createFieldNestedUpdateAgg( - fieldType, options.fieldNestedUpdateAggNestedKey(field)); - break; - case FieldCollectAgg.NAME: - checkArgument( - fieldType instanceof ArrayType, - "Data type for collect column must be 'Array' but was '%s'.", - fieldType); - fieldAggregator = - new FieldCollectAgg( - (ArrayType) fieldType, - options.fieldCollectAggDistinct(field)); - break; - case FieldMergeMapAgg.NAME: - checkArgument( - fieldType instanceof MapType, - "Data type of merge map column must be 'MAP' but was '%s'", - fieldType); - fieldAggregator = new FieldMergeMapAgg((MapType) fieldType); - break; - case FieldThetaSketchAgg.NAME: - checkArgument( - fieldType instanceof VarBinaryType, - "Data type for theta sketch column must be 'VarBinaryType' but was '%s'.", - fieldType); - fieldAggregator = new FieldThetaSketchAgg((VarBinaryType) fieldType); - break; - case FieldHllSketchAgg.NAME: - checkArgument( - fieldType instanceof VarBinaryType, - "Data type for hll sketch column must be 'VarBinaryType' but was '%s'.", - fieldType); - fieldAggregator = new FieldHllSketchAgg((VarBinaryType) fieldType); - break; - case FieldRoaringBitmap32Agg.NAME: - checkArgument( - fieldType instanceof VarBinaryType, - "Data type for roaring bitmap column must be 'VarBinaryType' but was '%s'.", - fieldType); - fieldAggregator = new FieldRoaringBitmap32Agg((VarBinaryType) fieldType); - break; - case FieldRoaringBitmap64Agg.NAME: - checkArgument( - fieldType instanceof VarBinaryType, - "Data type for roaring bitmap column must be 'VarBinaryType' but was '%s'.", - fieldType); - fieldAggregator = new FieldRoaringBitmap64Agg((VarBinaryType) fieldType); - break; - default: - throw new RuntimeException( - String.format( - "Use unsupported aggregation: %s or spell aggregate function incorrectly!", - strAgg)); - } - } - } - - if (ignoreRetract) { - fieldAggregator = new FieldIgnoreRetractAgg(fieldAggregator); - } + protected final DataType fieldType; + protected final String name; - return fieldAggregator; - } - - private static FieldAggregator createFieldNestedUpdateAgg( - DataType fieldType, List nestedKey) { - if (nestedKey == null) { - nestedKey = Collections.emptyList(); - } - - String typeErrorMsg = "Data type of nested table column must be 'Array' but was '%s'."; - checkArgument(fieldType instanceof ArrayType, typeErrorMsg, fieldType); - ArrayType arrayType = (ArrayType) fieldType; - checkArgument(arrayType.getElementType() instanceof RowType, typeErrorMsg, fieldType); - - return new FieldNestedUpdateAgg(arrayType, nestedKey); + public FieldAggregator(String name, DataType dataType) { + this.name = name; + this.fieldType = dataType; } - abstract String name(); - public abstract Object agg(Object accumulator, Object inputField); public Object aggReversed(Object accumulator, Object inputField) { @@ -191,6 +50,6 @@ public Object retract(Object accumulator, Object retractField) { "Aggregate function '%s' does not support retraction," + " If you allow this function to ignore retraction messages," + " you can configure 'fields.${field_name}.ignore-retract'='true'.", - name())); + name)); } } diff --git a/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/FieldBoolAndAgg.java b/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/FieldBoolAndAgg.java index 0b6371309168..cc44ce5ed9f6 100644 --- a/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/FieldBoolAndAgg.java +++ b/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/FieldBoolAndAgg.java @@ -18,43 +18,22 @@ package org.apache.paimon.mergetree.compact.aggregate; -import org.apache.paimon.types.DataType; +import org.apache.paimon.types.BooleanType; /** bool_and aggregate a field of a row. */ public class FieldBoolAndAgg extends FieldAggregator { - public static final String NAME = "bool_and"; - private static final long serialVersionUID = 1L; - public FieldBoolAndAgg(DataType dataType) { - super(dataType); - } - - @Override - String name() { - return NAME; + public FieldBoolAndAgg(String name, BooleanType dataType) { + super(name, dataType); } @Override public Object agg(Object accumulator, Object inputField) { - Object boolAnd; - if (accumulator == null || inputField == null) { - boolAnd = (inputField == null) ? accumulator : inputField; - } else { - switch (fieldType.getTypeRoot()) { - case BOOLEAN: - boolAnd = (boolean) accumulator && (boolean) inputField; - break; - default: - String msg = - String.format( - "type %s not support in %s", - fieldType.getTypeRoot().toString(), this.getClass().getName()); - throw new IllegalArgumentException(msg); - } + return accumulator == null ? inputField : accumulator; } - return boolAnd; + return (boolean) accumulator && (boolean) inputField; } } diff --git a/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/FieldBoolOrAgg.java b/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/FieldBoolOrAgg.java index ae385b3cc221..105f4219118d 100644 --- a/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/FieldBoolOrAgg.java +++ b/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/FieldBoolOrAgg.java @@ -18,43 +18,22 @@ package org.apache.paimon.mergetree.compact.aggregate; -import org.apache.paimon.types.DataType; +import org.apache.paimon.types.BooleanType; /** bool_or aggregate a field of a row. */ public class FieldBoolOrAgg extends FieldAggregator { - public static final String NAME = "bool_or"; - private static final long serialVersionUID = 1L; - public FieldBoolOrAgg(DataType dataType) { - super(dataType); - } - - @Override - String name() { - return NAME; + public FieldBoolOrAgg(String name, BooleanType dataType) { + super(name, dataType); } @Override public Object agg(Object accumulator, Object inputField) { - Object boolOr; - if (accumulator == null || inputField == null) { - boolOr = (inputField == null) ? accumulator : inputField; - } else { - switch (fieldType.getTypeRoot()) { - case BOOLEAN: - boolOr = (boolean) accumulator || (boolean) inputField; - break; - default: - String msg = - String.format( - "type %s not support in %s", - fieldType.getTypeRoot().toString(), this.getClass().getName()); - throw new IllegalArgumentException(msg); - } + return accumulator == null ? inputField : accumulator; } - return boolOr; + return (boolean) accumulator || (boolean) inputField; } } diff --git a/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/FieldCollectAgg.java b/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/FieldCollectAgg.java index ddfae4de2697..afe5e05e70c0 100644 --- a/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/FieldCollectAgg.java +++ b/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/FieldCollectAgg.java @@ -43,16 +43,14 @@ /** Collect elements into an ARRAY. */ public class FieldCollectAgg extends FieldAggregator { - public static final String NAME = "collect"; - private static final long serialVersionUID = 1L; private final boolean distinct; private final InternalArray.ElementGetter elementGetter; @Nullable private final BiFunction equaliser; - public FieldCollectAgg(ArrayType dataType, boolean distinct) { - super(dataType); + public FieldCollectAgg(String name, ArrayType dataType, boolean distinct) { + super(name, dataType); this.distinct = distinct; this.elementGetter = InternalArray.createElementGetter(dataType.getElementType()); @@ -84,11 +82,6 @@ public FieldCollectAgg(ArrayType dataType, boolean distinct) { } } - @Override - String name() { - return NAME; - } - @Override public Object aggReversed(Object accumulator, Object inputField) { // we don't need to actually do the reverse here for this agg diff --git a/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/FieldFirstNonNullValueAgg.java b/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/FieldFirstNonNullValueAgg.java index 3d758d8f5489..273af1a957f6 100644 --- a/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/FieldFirstNonNullValueAgg.java +++ b/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/FieldFirstNonNullValueAgg.java @@ -23,20 +23,12 @@ /** first non-null value aggregate a field of a row. */ public class FieldFirstNonNullValueAgg extends FieldAggregator { - public static final String NAME = "first_non_null_value"; - public static final String LEGACY_NAME = "first_not_null_value"; - private static final long serialVersionUID = 1L; private boolean initialized; - public FieldFirstNonNullValueAgg(DataType dataType) { - super(dataType); - } - - @Override - String name() { - return NAME; + public FieldFirstNonNullValueAgg(String name, DataType dataType) { + super(name, dataType); } @Override diff --git a/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/FieldFirstValueAgg.java b/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/FieldFirstValueAgg.java index d31a6e0ae144..436f841d95dc 100644 --- a/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/FieldFirstValueAgg.java +++ b/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/FieldFirstValueAgg.java @@ -23,19 +23,12 @@ /** first value aggregate a field of a row. */ public class FieldFirstValueAgg extends FieldAggregator { - public static final String NAME = "first_value"; - private static final long serialVersionUID = 1L; private boolean initialized; - public FieldFirstValueAgg(DataType dataType) { - super(dataType); - } - - @Override - public String name() { - return NAME; + public FieldFirstValueAgg(String name, DataType dataType) { + super(name, dataType); } @Override diff --git a/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/FieldHllSketchAgg.java b/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/FieldHllSketchAgg.java index 0ccf4af6497c..aa399ac37e32 100644 --- a/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/FieldHllSketchAgg.java +++ b/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/FieldHllSketchAgg.java @@ -24,25 +24,14 @@ /** HllSketch aggregate a field of a row. */ public class FieldHllSketchAgg extends FieldAggregator { - public static final String NAME = "hll_sketch"; - private static final long serialVersionUID = 1L; - public FieldHllSketchAgg(VarBinaryType dataType) { - super(dataType); - } - - @Override - public String name() { - return NAME; + public FieldHllSketchAgg(String name, VarBinaryType dataType) { + super(name, dataType); } @Override public Object agg(Object accumulator, Object inputField) { - if (accumulator == null && inputField == null) { - return null; - } - if (accumulator == null || inputField == null) { return accumulator == null ? inputField : accumulator; } diff --git a/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/FieldIgnoreRetractAgg.java b/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/FieldIgnoreRetractAgg.java index 83f3f72a2d6b..e98e64852b6f 100644 --- a/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/FieldIgnoreRetractAgg.java +++ b/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/FieldIgnoreRetractAgg.java @@ -21,20 +21,15 @@ /** An aggregator which ignores retraction messages. */ public class FieldIgnoreRetractAgg extends FieldAggregator { - private final FieldAggregator aggregator; - private static final long serialVersionUID = 1L; + private final FieldAggregator aggregator; + public FieldIgnoreRetractAgg(FieldAggregator aggregator) { - super(aggregator.fieldType); + super(aggregator.name, aggregator.fieldType); this.aggregator = aggregator; } - @Override - String name() { - return aggregator.name(); - } - @Override public Object agg(Object accumulator, Object inputField) { return aggregator.agg(accumulator, inputField); diff --git a/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/FieldLastNonNullValueAgg.java b/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/FieldLastNonNullValueAgg.java index f069a914d64d..cc5383739861 100644 --- a/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/FieldLastNonNullValueAgg.java +++ b/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/FieldLastNonNullValueAgg.java @@ -23,17 +23,10 @@ /** last non-null value aggregate a field of a row. */ public class FieldLastNonNullValueAgg extends FieldAggregator { - public static final String NAME = "last_non_null_value"; - private static final long serialVersionUID = 1L; - public FieldLastNonNullValueAgg(DataType dataType) { - super(dataType); - } - - @Override - String name() { - return NAME; + public FieldLastNonNullValueAgg(String name, DataType dataType) { + super(name, dataType); } @Override diff --git a/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/FieldLastValueAgg.java b/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/FieldLastValueAgg.java index 9a4a5d4de6d5..592f080fbd77 100644 --- a/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/FieldLastValueAgg.java +++ b/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/FieldLastValueAgg.java @@ -23,17 +23,10 @@ /** last value aggregate a field of a row. */ public class FieldLastValueAgg extends FieldAggregator { - public static final String NAME = "last_value"; - private static final long serialVersionUID = 1L; - public FieldLastValueAgg(DataType dataType) { - super(dataType); - } - - @Override - String name() { - return NAME; + public FieldLastValueAgg(String name, DataType dataType) { + super(name, dataType); } @Override diff --git a/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/FieldListaggAgg.java b/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/FieldListaggAgg.java index 25ee8cc24cbf..a01891501119 100644 --- a/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/FieldListaggAgg.java +++ b/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/FieldListaggAgg.java @@ -20,53 +20,32 @@ import org.apache.paimon.CoreOptions; import org.apache.paimon.data.BinaryString; -import org.apache.paimon.types.DataType; +import org.apache.paimon.types.VarCharType; import org.apache.paimon.utils.StringUtils; /** listagg aggregate a field of a row. */ public class FieldListaggAgg extends FieldAggregator { - public static final String NAME = "listagg"; - private static final long serialVersionUID = 1L; private final String delimiter; - public FieldListaggAgg(DataType dataType, CoreOptions options, String field) { - super(dataType); + public FieldListaggAgg(String name, VarCharType dataType, CoreOptions options, String field) { + super(name, dataType); this.delimiter = options.fieldListAggDelimiter(field); } - @Override - String name() { - return NAME; - } - @Override public Object agg(Object accumulator, Object inputField) { - Object concatenate; - - if (inputField == null || accumulator == null) { - concatenate = (inputField == null) ? accumulator : inputField; - } else { - // ordered by type root definition - switch (fieldType.getTypeRoot()) { - case VARCHAR: - // TODO: ensure not VARCHAR(n) - BinaryString mergeFieldSD = (BinaryString) accumulator; - BinaryString inFieldSD = (BinaryString) inputField; - concatenate = - StringUtils.concat( - mergeFieldSD, BinaryString.fromString(delimiter), inFieldSD); - break; - default: - String msg = - String.format( - "type %s not support in %s", - fieldType.getTypeRoot().toString(), this.getClass().getName()); - throw new IllegalArgumentException(msg); - } + if (accumulator == null || inputField == null) { + return accumulator == null ? inputField : accumulator; } - return concatenate; + // ordered by type root definition + + // TODO: ensure not VARCHAR(n) + BinaryString mergeFieldSD = (BinaryString) accumulator; + BinaryString inFieldSD = (BinaryString) inputField; + + return StringUtils.concat(mergeFieldSD, BinaryString.fromString(delimiter), inFieldSD); } } diff --git a/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/FieldMaxAgg.java b/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/FieldMaxAgg.java index 292c29510a4d..06628244e44d 100644 --- a/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/FieldMaxAgg.java +++ b/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/FieldMaxAgg.java @@ -25,33 +25,20 @@ /** max aggregate a field of a row. */ public class FieldMaxAgg extends FieldAggregator { - public static final String NAME = "max"; - private static final long serialVersionUID = 1L; - public FieldMaxAgg(DataType dataType) { - super(dataType); - } - - @Override - String name() { - return NAME; + public FieldMaxAgg(String name, DataType dataType) { + super(name, dataType); } @Override public Object agg(Object accumulator, Object inputField) { - Object max; - if (accumulator == null || inputField == null) { - max = (accumulator == null ? inputField : accumulator); - } else { - DataTypeRoot type = fieldType.getTypeRoot(); - if (InternalRowUtils.compare(accumulator, inputField, type) < 0) { - max = inputField; - } else { - max = accumulator; - } + return accumulator == null ? inputField : accumulator; } - return max; + DataTypeRoot type = fieldType.getTypeRoot(); + return InternalRowUtils.compare(accumulator, inputField, type) < 0 + ? inputField + : accumulator; } } diff --git a/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/FieldMergeMapAgg.java b/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/FieldMergeMapAgg.java index 8ba78ad5e4ff..9965339afd2b 100644 --- a/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/FieldMergeMapAgg.java +++ b/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/FieldMergeMapAgg.java @@ -31,25 +31,18 @@ /** Merge two maps. */ public class FieldMergeMapAgg extends FieldAggregator { - public static final String NAME = "merge_map"; - private static final long serialVersionUID = 1L; private final InternalArray.ElementGetter keyGetter; private final InternalArray.ElementGetter valueGetter; - public FieldMergeMapAgg(MapType dataType) { - super(dataType); + public FieldMergeMapAgg(String name, MapType dataType) { + super(name, dataType); this.keyGetter = InternalArray.createElementGetter(dataType.getKeyType()); this.valueGetter = InternalArray.createElementGetter(dataType.getValueType()); } - @Override - String name() { - return NAME; - } - @Override public Object agg(Object accumulator, Object inputField) { if (accumulator == null || inputField == null) { diff --git a/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/FieldMinAgg.java b/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/FieldMinAgg.java index 403724f25d73..01b0403baec3 100644 --- a/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/FieldMinAgg.java +++ b/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/FieldMinAgg.java @@ -25,33 +25,21 @@ /** min aggregate a field of a row. */ public class FieldMinAgg extends FieldAggregator { - public static final String NAME = "min"; - private static final long serialVersionUID = 1L; - public FieldMinAgg(DataType dataType) { - super(dataType); - } - - @Override - String name() { - return NAME; + public FieldMinAgg(String name, DataType dataType) { + super(name, dataType); } @Override public Object agg(Object accumulator, Object inputField) { - Object min; - if (accumulator == null || inputField == null) { - min = (accumulator == null ? inputField : accumulator); - } else { - DataTypeRoot type = fieldType.getTypeRoot(); - if (InternalRowUtils.compare(accumulator, inputField, type) < 0) { - min = accumulator; - } else { - min = inputField; - } + return accumulator == null ? inputField : accumulator; } - return min; + + DataTypeRoot type = fieldType.getTypeRoot(); + return InternalRowUtils.compare(accumulator, inputField, type) < 0 + ? accumulator + : inputField; } } diff --git a/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/FieldNestedUpdateAgg.java b/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/FieldNestedUpdateAgg.java index 1f725bb7dcfb..005bf7b17f1f 100644 --- a/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/FieldNestedUpdateAgg.java +++ b/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/FieldNestedUpdateAgg.java @@ -44,8 +44,6 @@ */ public class FieldNestedUpdateAgg extends FieldAggregator { - public static final String NAME = "nested_update"; - private static final long serialVersionUID = 1L; private final int nestedFields; @@ -53,8 +51,8 @@ public class FieldNestedUpdateAgg extends FieldAggregator { @Nullable private final Projection keyProjection; @Nullable private final RecordEqualiser elementEqualiser; - public FieldNestedUpdateAgg(ArrayType dataType, List nestedKey) { - super(dataType); + public FieldNestedUpdateAgg(String name, ArrayType dataType, List nestedKey) { + super(name, dataType); RowType nestedType = (RowType) dataType.getElementType(); this.nestedFields = nestedType.getFieldCount(); if (nestedKey.isEmpty()) { @@ -66,18 +64,10 @@ public FieldNestedUpdateAgg(ArrayType dataType, List nestedKey) { } } - @Override - String name() { - return NAME; - } - @Override public Object agg(Object accumulator, Object inputField) { - if (accumulator == null) { - return inputField; - } - if (inputField == null) { - return accumulator; + if (accumulator == null || inputField == null) { + return accumulator == null ? inputField : accumulator; } InternalArray acc = (InternalArray) accumulator; diff --git a/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/FieldPrimaryKeyAgg.java b/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/FieldPrimaryKeyAgg.java index 58961bc5a603..3db4e9b3246d 100644 --- a/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/FieldPrimaryKeyAgg.java +++ b/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/FieldPrimaryKeyAgg.java @@ -23,17 +23,10 @@ /** primary key aggregate a field of a row. */ public class FieldPrimaryKeyAgg extends FieldAggregator { - public static final String NAME = "primary-key"; - private static final long serialVersionUID = 1L; - public FieldPrimaryKeyAgg(DataType dataType) { - super(dataType); - } - - @Override - String name() { - return NAME; + public FieldPrimaryKeyAgg(String name, DataType dataType) { + super(name, dataType); } @Override diff --git a/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/FieldProductAgg.java b/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/FieldProductAgg.java index 1a02002460df..26a0c0c52e14 100644 --- a/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/FieldProductAgg.java +++ b/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/FieldProductAgg.java @@ -28,65 +28,58 @@ /** product value aggregate a field of a row. */ public class FieldProductAgg extends FieldAggregator { - public static final String NAME = "product"; - private static final long serialVersionUID = 1L; - public FieldProductAgg(DataType dataType) { - super(dataType); - } - - @Override - String name() { - return NAME; + public FieldProductAgg(String name, DataType dataType) { + super(name, dataType); } @Override public Object agg(Object accumulator, Object inputField) { + if (accumulator == null || inputField == null) { + return accumulator == null ? inputField : accumulator; + } + Object product; - if (accumulator == null || inputField == null) { - product = (accumulator == null ? inputField : accumulator); - } else { - // ordered by type root definition - switch (fieldType.getTypeRoot()) { - case DECIMAL: - Decimal mergeFieldDD = (Decimal) accumulator; - Decimal inFieldDD = (Decimal) inputField; - assert mergeFieldDD.scale() == inFieldDD.scale() - : "Inconsistent scale of aggregate Decimal!"; - assert mergeFieldDD.precision() == inFieldDD.precision() - : "Inconsistent precision of aggregate Decimal!"; - BigDecimal bigDecimal = mergeFieldDD.toBigDecimal(); - BigDecimal bigDecimal1 = inFieldDD.toBigDecimal(); - BigDecimal mul = bigDecimal.multiply(bigDecimal1); - product = fromBigDecimal(mul, mergeFieldDD.precision(), mergeFieldDD.scale()); - break; - case TINYINT: - product = (byte) ((byte) accumulator * (byte) inputField); - break; - case SMALLINT: - product = (short) ((short) accumulator * (short) inputField); - break; - case INTEGER: - product = (int) accumulator * (int) inputField; - break; - case BIGINT: - product = (long) accumulator * (long) inputField; - break; - case FLOAT: - product = (float) accumulator * (float) inputField; - break; - case DOUBLE: - product = (double) accumulator * (double) inputField; - break; - default: - String msg = - String.format( - "type %s not support in %s", - fieldType.getTypeRoot().toString(), this.getClass().getName()); - throw new IllegalArgumentException(msg); - } + // ordered by type root definition + switch (fieldType.getTypeRoot()) { + case DECIMAL: + Decimal mergeFieldDD = (Decimal) accumulator; + Decimal inFieldDD = (Decimal) inputField; + assert mergeFieldDD.scale() == inFieldDD.scale() + : "Inconsistent scale of aggregate Decimal!"; + assert mergeFieldDD.precision() == inFieldDD.precision() + : "Inconsistent precision of aggregate Decimal!"; + BigDecimal bigDecimal = mergeFieldDD.toBigDecimal(); + BigDecimal bigDecimal1 = inFieldDD.toBigDecimal(); + BigDecimal mul = bigDecimal.multiply(bigDecimal1); + product = fromBigDecimal(mul, mergeFieldDD.precision(), mergeFieldDD.scale()); + break; + case TINYINT: + product = (byte) ((byte) accumulator * (byte) inputField); + break; + case SMALLINT: + product = (short) ((short) accumulator * (short) inputField); + break; + case INTEGER: + product = (int) accumulator * (int) inputField; + break; + case BIGINT: + product = (long) accumulator * (long) inputField; + break; + case FLOAT: + product = (float) accumulator * (float) inputField; + break; + case DOUBLE: + product = (double) accumulator * (double) inputField; + break; + default: + String msg = + String.format( + "type %s not support in %s", + fieldType.getTypeRoot().toString(), this.getClass().getName()); + throw new IllegalArgumentException(msg); } return product; } diff --git a/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/FieldRoaringBitmap32Agg.java b/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/FieldRoaringBitmap32Agg.java index 15cbc2b96e37..ef7ac20e839a 100644 --- a/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/FieldRoaringBitmap32Agg.java +++ b/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/FieldRoaringBitmap32Agg.java @@ -26,29 +26,18 @@ /** roaring bitmap aggregate a field of a row. */ public class FieldRoaringBitmap32Agg extends FieldAggregator { - public static final String NAME = "rbm32"; - private static final long serialVersionUID = 1L; private final RoaringBitmap32 roaringBitmapAcc; private final RoaringBitmap32 roaringBitmapInput; - public FieldRoaringBitmap32Agg(VarBinaryType dataType) { - super(dataType); + public FieldRoaringBitmap32Agg(String name, VarBinaryType dataType) { + super(name, dataType); this.roaringBitmapAcc = new RoaringBitmap32(); this.roaringBitmapInput = new RoaringBitmap32(); } - @Override - public String name() { - return NAME; - } - @Override public Object agg(Object accumulator, Object inputField) { - if (accumulator == null && inputField == null) { - return null; - } - if (accumulator == null || inputField == null) { return accumulator == null ? inputField : accumulator; } diff --git a/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/FieldRoaringBitmap64Agg.java b/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/FieldRoaringBitmap64Agg.java index aa9cff1fe120..b1d096497465 100644 --- a/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/FieldRoaringBitmap64Agg.java +++ b/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/FieldRoaringBitmap64Agg.java @@ -26,29 +26,18 @@ /** roaring bitmap aggregate a field of a row. */ public class FieldRoaringBitmap64Agg extends FieldAggregator { - public static final String NAME = "rbm64"; - private static final long serialVersionUID = 1L; private final RoaringBitmap64 roaringBitmapAcc; private final RoaringBitmap64 roaringBitmapInput; - public FieldRoaringBitmap64Agg(VarBinaryType dataType) { - super(dataType); + public FieldRoaringBitmap64Agg(String name, VarBinaryType dataType) { + super(name, dataType); this.roaringBitmapAcc = new RoaringBitmap64(); this.roaringBitmapInput = new RoaringBitmap64(); } - @Override - public String name() { - return NAME; - } - @Override public Object agg(Object accumulator, Object inputField) { - if (accumulator == null && inputField == null) { - return null; - } - if (accumulator == null || inputField == null) { return accumulator == null ? inputField : accumulator; } diff --git a/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/FieldSumAgg.java b/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/FieldSumAgg.java index 38081e20e222..4b3ad12aea17 100644 --- a/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/FieldSumAgg.java +++ b/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/FieldSumAgg.java @@ -25,118 +25,109 @@ /** sum aggregate a field of a row. */ public class FieldSumAgg extends FieldAggregator { - public static final String NAME = "sum"; - private static final long serialVersionUID = 1L; - public FieldSumAgg(DataType dataType) { - super(dataType); - } - - @Override - String name() { - return NAME; + public FieldSumAgg(String name, DataType dataType) { + super(name, dataType); } @Override public Object agg(Object accumulator, Object inputField) { + if (accumulator == null || inputField == null) { + return accumulator == null ? inputField : accumulator; + } Object sum; - if (accumulator == null || inputField == null) { - sum = (accumulator == null ? inputField : accumulator); - } else { - // ordered by type root definition - switch (fieldType.getTypeRoot()) { - case DECIMAL: - Decimal mergeFieldDD = (Decimal) accumulator; - Decimal inFieldDD = (Decimal) inputField; - assert mergeFieldDD.scale() == inFieldDD.scale() - : "Inconsistent scale of aggregate Decimal!"; - assert mergeFieldDD.precision() == inFieldDD.precision() - : "Inconsistent precision of aggregate Decimal!"; - sum = - DecimalUtils.add( - mergeFieldDD, - inFieldDD, - mergeFieldDD.precision(), - mergeFieldDD.scale()); - break; - case TINYINT: - sum = (byte) ((byte) accumulator + (byte) inputField); - break; - case SMALLINT: - sum = (short) ((short) accumulator + (short) inputField); - break; - case INTEGER: - sum = (int) accumulator + (int) inputField; - break; - case BIGINT: - sum = (long) accumulator + (long) inputField; - break; - case FLOAT: - sum = (float) accumulator + (float) inputField; - break; - case DOUBLE: - sum = (double) accumulator + (double) inputField; - break; - default: - String msg = - String.format( - "type %s not support in %s", - fieldType.getTypeRoot().toString(), this.getClass().getName()); - throw new IllegalArgumentException(msg); - } + // ordered by type root definition + switch (fieldType.getTypeRoot()) { + case DECIMAL: + Decimal mergeFieldDD = (Decimal) accumulator; + Decimal inFieldDD = (Decimal) inputField; + assert mergeFieldDD.scale() == inFieldDD.scale() + : "Inconsistent scale of aggregate Decimal!"; + assert mergeFieldDD.precision() == inFieldDD.precision() + : "Inconsistent precision of aggregate Decimal!"; + sum = + DecimalUtils.add( + mergeFieldDD, + inFieldDD, + mergeFieldDD.precision(), + mergeFieldDD.scale()); + break; + case TINYINT: + sum = (byte) ((byte) accumulator + (byte) inputField); + break; + case SMALLINT: + sum = (short) ((short) accumulator + (short) inputField); + break; + case INTEGER: + sum = (int) accumulator + (int) inputField; + break; + case BIGINT: + sum = (long) accumulator + (long) inputField; + break; + case FLOAT: + sum = (float) accumulator + (float) inputField; + break; + case DOUBLE: + sum = (double) accumulator + (double) inputField; + break; + default: + String msg = + String.format( + "type %s not support in %s", + fieldType.getTypeRoot().toString(), this.getClass().getName()); + throw new IllegalArgumentException(msg); } return sum; } @Override public Object retract(Object accumulator, Object inputField) { - Object sum; if (accumulator == null || inputField == null) { - sum = (accumulator == null ? negative(inputField) : accumulator); - } else { - switch (fieldType.getTypeRoot()) { - case DECIMAL: - Decimal mergeFieldDD = (Decimal) accumulator; - Decimal inFieldDD = (Decimal) inputField; - assert mergeFieldDD.scale() == inFieldDD.scale() - : "Inconsistent scale of aggregate Decimal!"; - assert mergeFieldDD.precision() == inFieldDD.precision() - : "Inconsistent precision of aggregate Decimal!"; - sum = - DecimalUtils.subtract( - mergeFieldDD, - inFieldDD, - mergeFieldDD.precision(), - mergeFieldDD.scale()); - break; - case TINYINT: - sum = (byte) ((byte) accumulator - (byte) inputField); - break; - case SMALLINT: - sum = (short) ((short) accumulator - (short) inputField); - break; - case INTEGER: - sum = (int) accumulator - (int) inputField; - break; - case BIGINT: - sum = (long) accumulator - (long) inputField; - break; - case FLOAT: - sum = (float) accumulator - (float) inputField; - break; - case DOUBLE: - sum = (double) accumulator - (double) inputField; - break; - default: - String msg = - String.format( - "type %s not support in %s", - fieldType.getTypeRoot().toString(), this.getClass().getName()); - throw new IllegalArgumentException(msg); - } + return (accumulator == null ? negative(inputField) : accumulator); + } + Object sum; + switch (fieldType.getTypeRoot()) { + case DECIMAL: + Decimal mergeFieldDD = (Decimal) accumulator; + Decimal inFieldDD = (Decimal) inputField; + assert mergeFieldDD.scale() == inFieldDD.scale() + : "Inconsistent scale of aggregate Decimal!"; + assert mergeFieldDD.precision() == inFieldDD.precision() + : "Inconsistent precision of aggregate Decimal!"; + sum = + DecimalUtils.subtract( + mergeFieldDD, + inFieldDD, + mergeFieldDD.precision(), + mergeFieldDD.scale()); + break; + case TINYINT: + sum = (byte) ((byte) accumulator - (byte) inputField); + break; + case SMALLINT: + sum = (short) ((short) accumulator - (short) inputField); + break; + case INTEGER: + sum = (int) accumulator - (int) inputField; + break; + case BIGINT: + sum = (long) accumulator - (long) inputField; + break; + case FLOAT: + sum = (float) accumulator - (float) inputField; + break; + case DOUBLE: + sum = (double) accumulator - (double) inputField; + break; + default: + String msg = + String.format( + "type %s not support in %s", + fieldType.getTypeRoot().toString(), this.getClass().getName()); + throw new IllegalArgumentException(msg); } return sum; } diff --git a/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/FieldThetaSketchAgg.java b/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/FieldThetaSketchAgg.java index 7182a6744317..9622b4aff0b2 100644 --- a/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/FieldThetaSketchAgg.java +++ b/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/FieldThetaSketchAgg.java @@ -24,25 +24,14 @@ /** ThetaSketch aggregate a field of a row. */ public class FieldThetaSketchAgg extends FieldAggregator { - public static final String NAME = "theta_sketch"; - private static final long serialVersionUID = 1L; - public FieldThetaSketchAgg(VarBinaryType dataType) { - super(dataType); - } - - @Override - public String name() { - return NAME; + public FieldThetaSketchAgg(String name, VarBinaryType dataType) { + super(name, dataType); } @Override public Object agg(Object accumulator, Object inputField) { - if (accumulator == null && inputField == null) { - return null; - } - if (accumulator == null || inputField == null) { return accumulator == null ? inputField : accumulator; } diff --git a/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/factory/FieldAggregatorFactory.java b/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/factory/FieldAggregatorFactory.java new file mode 100644 index 000000000000..d2ce0e4760ad --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/factory/FieldAggregatorFactory.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.mergetree.compact.aggregate.factory; + +import org.apache.paimon.CoreOptions; +import org.apache.paimon.factories.Factory; +import org.apache.paimon.factories.FactoryUtil; +import org.apache.paimon.mergetree.compact.aggregate.FieldAggregator; +import org.apache.paimon.mergetree.compact.aggregate.FieldIgnoreRetractAgg; +import org.apache.paimon.types.DataType; + +import javax.annotation.Nullable; + +/** Factory for {@link FieldAggregator}. */ +public interface FieldAggregatorFactory extends Factory { + + FieldAggregator create(DataType fieldType, CoreOptions options, String field); + + String identifier(); + + static FieldAggregator create( + DataType fieldType, + @Nullable String strAgg, + boolean ignoreRetract, + boolean isPrimaryKey, + CoreOptions options, + String field) { + FieldAggregator fieldAggregator; + if (isPrimaryKey) { + strAgg = FieldPrimaryKeyAggFactory.NAME; + } else if (strAgg == null) { + strAgg = FieldLastNonNullValueAggFactory.NAME; + } + + FieldAggregatorFactory fieldAggregatorFactory = + FactoryUtil.discoverFactory( + FieldAggregator.class.getClassLoader(), + FieldAggregatorFactory.class, + strAgg); + if (fieldAggregatorFactory == null) { + throw new RuntimeException( + String.format( + "Use unsupported aggregation: %s or spell aggregate function incorrectly!", + strAgg)); + } + + fieldAggregator = fieldAggregatorFactory.create(fieldType, options, field); + + if (ignoreRetract) { + fieldAggregator = new FieldIgnoreRetractAgg(fieldAggregator); + } + + return fieldAggregator; + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/factory/FieldBoolAndAggFactory.java b/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/factory/FieldBoolAndAggFactory.java new file mode 100644 index 000000000000..45bb8708da39 --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/factory/FieldBoolAndAggFactory.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.mergetree.compact.aggregate.factory; + +import org.apache.paimon.CoreOptions; +import org.apache.paimon.mergetree.compact.aggregate.FieldBoolAndAgg; +import org.apache.paimon.types.BooleanType; +import org.apache.paimon.types.DataType; + +import static org.apache.paimon.utils.Preconditions.checkArgument; + +/** Factory for #{@link FieldBoolAndAgg}. */ +public class FieldBoolAndAggFactory implements FieldAggregatorFactory { + + public static final String NAME = "bool_and"; + + @Override + public FieldBoolAndAgg create(DataType fieldType, CoreOptions options, String field) { + checkArgument( + fieldType instanceof BooleanType, + "Data type for bool and column must be 'BooleanType' but was '%s'.", + fieldType); + return new FieldBoolAndAgg(identifier(), (BooleanType) fieldType); + } + + @Override + public String identifier() { + return NAME; + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/factory/FieldBoolOrAggFactory.java b/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/factory/FieldBoolOrAggFactory.java new file mode 100644 index 000000000000..266ccad6a215 --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/factory/FieldBoolOrAggFactory.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.mergetree.compact.aggregate.factory; + +import org.apache.paimon.CoreOptions; +import org.apache.paimon.mergetree.compact.aggregate.FieldBoolOrAgg; +import org.apache.paimon.types.BooleanType; +import org.apache.paimon.types.DataType; + +import static org.apache.paimon.utils.Preconditions.checkArgument; + +/** Factory for #{@link FieldBoolOrAgg}. */ +public class FieldBoolOrAggFactory implements FieldAggregatorFactory { + + public static final String NAME = "bool_or"; + + @Override + public FieldBoolOrAgg create(DataType fieldType, CoreOptions options, String field) { + checkArgument( + fieldType instanceof BooleanType, + "Data type for bool or column must be 'BooleanType' but was '%s'.", + fieldType); + return new FieldBoolOrAgg(identifier(), (BooleanType) fieldType); + } + + @Override + public String identifier() { + return NAME; + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/factory/FieldCollectAggFactory.java b/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/factory/FieldCollectAggFactory.java new file mode 100644 index 000000000000..a4325d165bea --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/factory/FieldCollectAggFactory.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.mergetree.compact.aggregate.factory; + +import org.apache.paimon.CoreOptions; +import org.apache.paimon.mergetree.compact.aggregate.FieldCollectAgg; +import org.apache.paimon.types.ArrayType; +import org.apache.paimon.types.DataType; + +import static org.apache.paimon.utils.Preconditions.checkArgument; + +/** Factory for #{@link FieldCollectAgg}. */ +public class FieldCollectAggFactory implements FieldAggregatorFactory { + + public static final String NAME = "collect"; + + @Override + public FieldCollectAgg create(DataType fieldType, CoreOptions options, String field) { + checkArgument( + fieldType instanceof ArrayType, + "Data type for collect column must be 'Array' but was '%s'.", + fieldType); + return new FieldCollectAgg( + identifier(), (ArrayType) fieldType, options.fieldCollectAggDistinct(field)); + } + + @Override + public String identifier() { + return NAME; + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/factory/FieldFirstNonNullValueAggFactory.java b/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/factory/FieldFirstNonNullValueAggFactory.java new file mode 100644 index 000000000000..51e3a6d62e46 --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/factory/FieldFirstNonNullValueAggFactory.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.mergetree.compact.aggregate.factory; + +import org.apache.paimon.CoreOptions; +import org.apache.paimon.mergetree.compact.aggregate.FieldFirstNonNullValueAgg; +import org.apache.paimon.types.DataType; + +/** Factory for #{@link FieldFirstNonNullValueAgg}. */ +public class FieldFirstNonNullValueAggFactory implements FieldAggregatorFactory { + + public static final String NAME = "first_non_null_value"; + + @Override + public FieldFirstNonNullValueAgg create(DataType fieldType, CoreOptions options, String field) { + return new FieldFirstNonNullValueAgg(identifier(), fieldType); + } + + @Override + public String identifier() { + return NAME; + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/factory/FieldFirstNonNullValueAggLegacyFactory.java b/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/factory/FieldFirstNonNullValueAggLegacyFactory.java new file mode 100644 index 000000000000..507ecbb5c7c6 --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/factory/FieldFirstNonNullValueAggLegacyFactory.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.mergetree.compact.aggregate.factory; + +import org.apache.paimon.CoreOptions; +import org.apache.paimon.mergetree.compact.aggregate.FieldAggregator; +import org.apache.paimon.mergetree.compact.aggregate.FieldFirstNonNullValueAgg; +import org.apache.paimon.types.DataType; + +/** Factory for legacy name of #{@link FieldFirstNonNullValueAgg}. */ +public class FieldFirstNonNullValueAggLegacyFactory implements FieldAggregatorFactory { + + public static final String LEGACY_NAME = "first_not_null_value"; + + @Override + public FieldAggregator create(DataType fieldType, CoreOptions options, String field) { + return new FieldFirstNonNullValueAgg(identifier(), fieldType); + } + + @Override + public String identifier() { + return LEGACY_NAME; + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/factory/FieldFirstValueAggFactory.java b/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/factory/FieldFirstValueAggFactory.java new file mode 100644 index 000000000000..84db12ffcdf6 --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/factory/FieldFirstValueAggFactory.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.mergetree.compact.aggregate.factory; + +import org.apache.paimon.CoreOptions; +import org.apache.paimon.mergetree.compact.aggregate.FieldFirstValueAgg; +import org.apache.paimon.types.DataType; + +/** Factory for #{@link FieldFirstValueAgg}. */ +public class FieldFirstValueAggFactory implements FieldAggregatorFactory { + + public static final String NAME = "first_value"; + + @Override + public FieldFirstValueAgg create(DataType fieldType, CoreOptions options, String field) { + return new FieldFirstValueAgg(identifier(), fieldType); + } + + @Override + public String identifier() { + return NAME; + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/factory/FieldHllSketchAggFactory.java b/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/factory/FieldHllSketchAggFactory.java new file mode 100644 index 000000000000..5777c6a416c4 --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/factory/FieldHllSketchAggFactory.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.mergetree.compact.aggregate.factory; + +import org.apache.paimon.CoreOptions; +import org.apache.paimon.mergetree.compact.aggregate.FieldHllSketchAgg; +import org.apache.paimon.types.DataType; +import org.apache.paimon.types.VarBinaryType; + +import static org.apache.paimon.utils.Preconditions.checkArgument; + +/** Factory for #{@link FieldHllSketchAgg}. */ +public class FieldHllSketchAggFactory implements FieldAggregatorFactory { + + public static final String NAME = "hll_sketch"; + + @Override + public FieldHllSketchAgg create(DataType fieldType, CoreOptions options, String field) { + checkArgument( + fieldType instanceof VarBinaryType, + "Data type for hll sketch column must be 'VarBinaryType' but was '%s'.", + fieldType); + return new FieldHllSketchAgg(identifier(), (VarBinaryType) fieldType); + } + + @Override + public String identifier() { + return NAME; + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/factory/FieldLastNonNullValueAggFactory.java b/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/factory/FieldLastNonNullValueAggFactory.java new file mode 100644 index 000000000000..bbc6402bb118 --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/factory/FieldLastNonNullValueAggFactory.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.mergetree.compact.aggregate.factory; + +import org.apache.paimon.CoreOptions; +import org.apache.paimon.mergetree.compact.aggregate.FieldLastNonNullValueAgg; +import org.apache.paimon.types.DataType; + +/** Factory for #{@link FieldLastNonNullValueAgg}. */ +public class FieldLastNonNullValueAggFactory implements FieldAggregatorFactory { + + public static final String NAME = "last_non_null_value"; + + @Override + public FieldLastNonNullValueAgg create(DataType fieldType, CoreOptions options, String field) { + return new FieldLastNonNullValueAgg(identifier(), fieldType); + } + + @Override + public String identifier() { + return NAME; + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/factory/FieldLastValueAggFactory.java b/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/factory/FieldLastValueAggFactory.java new file mode 100644 index 000000000000..c825825a159c --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/factory/FieldLastValueAggFactory.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.mergetree.compact.aggregate.factory; + +import org.apache.paimon.CoreOptions; +import org.apache.paimon.mergetree.compact.aggregate.FieldLastValueAgg; +import org.apache.paimon.types.DataType; + +/** Factory for #{@link FieldLastValueAgg}. */ +public class FieldLastValueAggFactory implements FieldAggregatorFactory { + + public static final String NAME = "last_value"; + + @Override + public FieldLastValueAgg create(DataType fieldType, CoreOptions options, String field) { + return new FieldLastValueAgg(identifier(), fieldType); + } + + @Override + public String identifier() { + return NAME; + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/factory/FieldListaggAggFactory.java b/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/factory/FieldListaggAggFactory.java new file mode 100644 index 000000000000..cdb9c128a67a --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/factory/FieldListaggAggFactory.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.mergetree.compact.aggregate.factory; + +import org.apache.paimon.CoreOptions; +import org.apache.paimon.mergetree.compact.aggregate.FieldListaggAgg; +import org.apache.paimon.types.DataType; +import org.apache.paimon.types.VarCharType; + +import static org.apache.paimon.utils.Preconditions.checkArgument; + +/** Factory for #{@link FieldListaggAgg}. */ +public class FieldListaggAggFactory implements FieldAggregatorFactory { + + public static final String NAME = "listagg"; + + @Override + public FieldListaggAgg create(DataType fieldType, CoreOptions options, String field) { + checkArgument( + fieldType instanceof VarCharType, + "Data type for list agg column must be 'VarCharType' but was '%s'.", + fieldType); + return new FieldListaggAgg(identifier(), (VarCharType) fieldType, options, field); + } + + @Override + public String identifier() { + return NAME; + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/factory/FieldMaxAggFactory.java b/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/factory/FieldMaxAggFactory.java new file mode 100644 index 000000000000..4e3c33171a89 --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/factory/FieldMaxAggFactory.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.mergetree.compact.aggregate.factory; + +import org.apache.paimon.CoreOptions; +import org.apache.paimon.mergetree.compact.aggregate.FieldMaxAgg; +import org.apache.paimon.types.DataType; + +/** Factory for #{@link FieldMaxAgg}. */ +public class FieldMaxAggFactory implements FieldAggregatorFactory { + + public static final String NAME = "max"; + + @Override + public FieldMaxAgg create(DataType fieldType, CoreOptions options, String field) { + return new FieldMaxAgg(identifier(), fieldType); + } + + @Override + public String identifier() { + return NAME; + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/factory/FieldMergeMapAggFactory.java b/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/factory/FieldMergeMapAggFactory.java new file mode 100644 index 000000000000..e10602f61810 --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/factory/FieldMergeMapAggFactory.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.mergetree.compact.aggregate.factory; + +import org.apache.paimon.CoreOptions; +import org.apache.paimon.mergetree.compact.aggregate.FieldMergeMapAgg; +import org.apache.paimon.types.DataType; +import org.apache.paimon.types.MapType; + +import static org.apache.paimon.utils.Preconditions.checkArgument; + +/** Factory for #{@link FieldMergeMapAgg}. */ +public class FieldMergeMapAggFactory implements FieldAggregatorFactory { + + public static final String NAME = "merge_map"; + + @Override + public FieldMergeMapAgg create(DataType fieldType, CoreOptions options, String field) { + checkArgument( + fieldType instanceof MapType, + "Data type for merge map column must be 'MAP' but was '%s'", + fieldType); + return new FieldMergeMapAgg(identifier(), (MapType) fieldType); + } + + @Override + public String identifier() { + return NAME; + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/factory/FieldMinAggFactory.java b/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/factory/FieldMinAggFactory.java new file mode 100644 index 000000000000..4ac7c08b1904 --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/factory/FieldMinAggFactory.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.mergetree.compact.aggregate.factory; + +import org.apache.paimon.CoreOptions; +import org.apache.paimon.mergetree.compact.aggregate.FieldMinAgg; +import org.apache.paimon.types.DataType; + +/** Factory for #{@link FieldMinAgg}. */ +public class FieldMinAggFactory implements FieldAggregatorFactory { + + public static final String NAME = "min"; + + @Override + public FieldMinAgg create(DataType fieldType, CoreOptions options, String field) { + return new FieldMinAgg(identifier(), fieldType); + } + + @Override + public String identifier() { + return NAME; + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/factory/FieldNestedUpdateAggFactory.java b/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/factory/FieldNestedUpdateAggFactory.java new file mode 100644 index 000000000000..b92df641448b --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/factory/FieldNestedUpdateAggFactory.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.mergetree.compact.aggregate.factory; + +import org.apache.paimon.CoreOptions; +import org.apache.paimon.mergetree.compact.aggregate.FieldAggregator; +import org.apache.paimon.mergetree.compact.aggregate.FieldNestedUpdateAgg; +import org.apache.paimon.types.ArrayType; +import org.apache.paimon.types.DataType; +import org.apache.paimon.types.RowType; + +import java.util.Collections; +import java.util.List; + +import static org.apache.paimon.utils.Preconditions.checkArgument; + +/** Factory for #{@link FieldNestedUpdateAgg}. */ +public class FieldNestedUpdateAggFactory implements FieldAggregatorFactory { + + public static final String NAME = "nested_update"; + + @Override + public FieldAggregator create(DataType fieldType, CoreOptions options, String field) { + return createFieldNestedUpdateAgg(fieldType, options.fieldNestedUpdateAggNestedKey(field)); + } + + @Override + public String identifier() { + return NAME; + } + + private FieldAggregator createFieldNestedUpdateAgg(DataType fieldType, List nestedKey) { + if (nestedKey == null) { + nestedKey = Collections.emptyList(); + } + + String typeErrorMsg = + "Data type for nested table column must be 'Array' but was '%s'."; + checkArgument(fieldType instanceof ArrayType, typeErrorMsg, fieldType); + ArrayType arrayType = (ArrayType) fieldType; + checkArgument(arrayType.getElementType() instanceof RowType, typeErrorMsg, fieldType); + + return new FieldNestedUpdateAgg(identifier(), arrayType, nestedKey); + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/factory/FieldPrimaryKeyAggFactory.java b/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/factory/FieldPrimaryKeyAggFactory.java new file mode 100644 index 000000000000..0e293bcf78b2 --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/factory/FieldPrimaryKeyAggFactory.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.mergetree.compact.aggregate.factory; + +import org.apache.paimon.CoreOptions; +import org.apache.paimon.mergetree.compact.aggregate.FieldAggregator; +import org.apache.paimon.mergetree.compact.aggregate.FieldPrimaryKeyAgg; +import org.apache.paimon.types.DataType; + +/** Factory for #{@link FieldPrimaryKeyAgg}. */ +public class FieldPrimaryKeyAggFactory implements FieldAggregatorFactory { + + public static final String NAME = "primary-key"; + + @Override + public FieldAggregator create(DataType fieldType, CoreOptions options, String field) { + return new FieldPrimaryKeyAgg(identifier(), fieldType); + } + + @Override + public String identifier() { + return NAME; + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/factory/FieldProductAggFactory.java b/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/factory/FieldProductAggFactory.java new file mode 100644 index 000000000000..7dbdd9f5af5a --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/factory/FieldProductAggFactory.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.mergetree.compact.aggregate.factory; + +import org.apache.paimon.CoreOptions; +import org.apache.paimon.mergetree.compact.aggregate.FieldProductAgg; +import org.apache.paimon.types.DataType; +import org.apache.paimon.types.DataTypeFamily; + +import static org.apache.paimon.utils.Preconditions.checkArgument; + +/** Factory for #{@link FieldProductAgg}. */ +public class FieldProductAggFactory implements FieldAggregatorFactory { + + public static final String NAME = "product"; + + @Override + public FieldProductAgg create(DataType fieldType, CoreOptions options, String field) { + checkArgument( + fieldType.getTypeRoot().getFamilies().contains(DataTypeFamily.NUMERIC), + "Data type for product column must be 'NumericType' but was '%s'.", + fieldType); + return new FieldProductAgg(identifier(), fieldType); + } + + @Override + public String identifier() { + return NAME; + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/factory/FieldRoaringBitmap32AggFactory.java b/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/factory/FieldRoaringBitmap32AggFactory.java new file mode 100644 index 000000000000..91103791f984 --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/factory/FieldRoaringBitmap32AggFactory.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.mergetree.compact.aggregate.factory; + +import org.apache.paimon.CoreOptions; +import org.apache.paimon.mergetree.compact.aggregate.FieldRoaringBitmap32Agg; +import org.apache.paimon.types.DataType; +import org.apache.paimon.types.VarBinaryType; + +import static org.apache.paimon.utils.Preconditions.checkArgument; + +/** Factory for #{@link FieldRoaringBitmap32Agg}. */ +public class FieldRoaringBitmap32AggFactory implements FieldAggregatorFactory { + + public static final String NAME = "rbm32"; + + @Override + public FieldRoaringBitmap32Agg create(DataType fieldType, CoreOptions options, String field) { + checkArgument( + fieldType instanceof VarBinaryType, + "Data type for roaring bitmap column must be 'VarBinaryType' but was '%s'.", + fieldType); + return new FieldRoaringBitmap32Agg(identifier(), (VarBinaryType) fieldType); + } + + @Override + public String identifier() { + return NAME; + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/factory/FieldRoaringBitmap64AggFactory.java b/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/factory/FieldRoaringBitmap64AggFactory.java new file mode 100644 index 000000000000..56f5554af1a9 --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/factory/FieldRoaringBitmap64AggFactory.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.mergetree.compact.aggregate.factory; + +import org.apache.paimon.CoreOptions; +import org.apache.paimon.mergetree.compact.aggregate.FieldRoaringBitmap64Agg; +import org.apache.paimon.types.DataType; +import org.apache.paimon.types.VarBinaryType; + +import static org.apache.paimon.utils.Preconditions.checkArgument; + +/** Factory for #{@link FieldRoaringBitmap64Agg}. */ +public class FieldRoaringBitmap64AggFactory implements FieldAggregatorFactory { + + public static final String NAME = "rbm64"; + + @Override + public FieldRoaringBitmap64Agg create(DataType fieldType, CoreOptions options, String field) { + checkArgument( + fieldType instanceof VarBinaryType, + "Data type for roaring bitmap column must be 'VarBinaryType' but was '%s'.", + fieldType); + return new FieldRoaringBitmap64Agg(identifier(), (VarBinaryType) fieldType); + } + + @Override + public String identifier() { + return NAME; + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/factory/FieldSumAggFactory.java b/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/factory/FieldSumAggFactory.java new file mode 100644 index 000000000000..5343f67b6ad7 --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/factory/FieldSumAggFactory.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.mergetree.compact.aggregate.factory; + +import org.apache.paimon.CoreOptions; +import org.apache.paimon.mergetree.compact.aggregate.FieldSumAgg; +import org.apache.paimon.types.DataType; +import org.apache.paimon.types.DataTypeFamily; + +import static org.apache.paimon.utils.Preconditions.checkArgument; + +/** Factory for #{@link FieldSumAgg}. */ +public class FieldSumAggFactory implements FieldAggregatorFactory { + + public static final String NAME = "sum"; + + @Override + public FieldSumAgg create(DataType fieldType, CoreOptions options, String field) { + checkArgument( + fieldType.getTypeRoot().getFamilies().contains(DataTypeFamily.NUMERIC), + "Data type for sum column must be 'NumericType' but was '%s'.", + fieldType); + return new FieldSumAgg(identifier(), fieldType); + } + + @Override + public String identifier() { + return NAME; + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/factory/FieldThetaSketchAggFactory.java b/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/factory/FieldThetaSketchAggFactory.java new file mode 100644 index 000000000000..c30fb7df7fb9 --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/aggregate/factory/FieldThetaSketchAggFactory.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.mergetree.compact.aggregate.factory; + +import org.apache.paimon.CoreOptions; +import org.apache.paimon.mergetree.compact.aggregate.FieldThetaSketchAgg; +import org.apache.paimon.types.DataType; +import org.apache.paimon.types.VarBinaryType; + +import static org.apache.paimon.utils.Preconditions.checkArgument; + +/** Factory for #{@link FieldThetaSketchAgg}. */ +public class FieldThetaSketchAggFactory implements FieldAggregatorFactory { + + public static final String NAME = "theta_sketch"; + + @Override + public FieldThetaSketchAgg create(DataType fieldType, CoreOptions options, String field) { + checkArgument( + fieldType instanceof VarBinaryType, + "Data type for theta sketch column must be 'VarBinaryType' but was '%s'.", + fieldType); + return new FieldThetaSketchAgg(identifier(), (VarBinaryType) fieldType); + } + + @Override + public String identifier() { + return NAME; + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/mergetree/localmerge/HashMapLocalMerger.java b/paimon-core/src/main/java/org/apache/paimon/mergetree/localmerge/HashMapLocalMerger.java new file mode 100644 index 000000000000..1a395fb36c56 --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/mergetree/localmerge/HashMapLocalMerger.java @@ -0,0 +1,133 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.mergetree.localmerge; + +import org.apache.paimon.KeyValue; +import org.apache.paimon.data.BinaryRow; +import org.apache.paimon.data.InternalRow; +import org.apache.paimon.data.InternalRow.FieldSetter; +import org.apache.paimon.data.serializer.BinaryRowSerializer; +import org.apache.paimon.data.serializer.InternalRowSerializer; +import org.apache.paimon.hash.BytesHashMap; +import org.apache.paimon.hash.BytesMap.LookupInfo; +import org.apache.paimon.memory.MemorySegmentPool; +import org.apache.paimon.mergetree.compact.MergeFunction; +import org.apache.paimon.types.DataField; +import org.apache.paimon.types.RowKind; +import org.apache.paimon.types.RowType; +import org.apache.paimon.utils.FieldsComparator; +import org.apache.paimon.utils.KeyValueIterator; + +import javax.annotation.Nullable; + +import java.io.EOFException; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.function.Consumer; + +import static org.apache.paimon.data.InternalRow.createFieldSetter; + +/** A {@link LocalMerger} which stores records in {@link BytesHashMap}. */ +public class HashMapLocalMerger implements LocalMerger { + + private final InternalRowSerializer valueSerializer; + private final MergeFunction mergeFunction; + @Nullable private final FieldsComparator udsComparator; + private final BytesHashMap buffer; + private final List nonKeySetters; + + public HashMapLocalMerger( + RowType rowType, + List primaryKeys, + MemorySegmentPool memoryPool, + MergeFunction mergeFunction, + @Nullable FieldsComparator userDefinedSeqComparator) { + this.valueSerializer = new InternalRowSerializer(rowType); + this.mergeFunction = mergeFunction; + this.udsComparator = userDefinedSeqComparator; + this.buffer = + new BytesHashMap<>( + memoryPool, + new BinaryRowSerializer(primaryKeys.size()), + rowType.getFieldCount()); + + this.nonKeySetters = new ArrayList<>(); + for (int i = 0; i < rowType.getFieldCount(); i++) { + DataField field = rowType.getFields().get(i); + if (primaryKeys.contains(field.name())) { + continue; + } + nonKeySetters.add(createFieldSetter(field.type(), i)); + } + } + + @Override + public boolean put(RowKind rowKind, BinaryRow key, InternalRow value) throws IOException { + // we store row kind in value + value.setRowKind(rowKind); + + LookupInfo lookup = buffer.lookup(key); + if (!lookup.isFound()) { + try { + buffer.append(lookup, valueSerializer.toBinaryRow(value)); + return true; + } catch (EOFException eof) { + return false; + } + } + + mergeFunction.reset(); + BinaryRow stored = lookup.getValue(); + KeyValue previousKv = new KeyValue().replace(key, stored.getRowKind(), stored); + KeyValue newKv = new KeyValue().replace(key, value.getRowKind(), value); + if (udsComparator != null && udsComparator.compare(stored, value) > 0) { + mergeFunction.add(newKv); + mergeFunction.add(previousKv); + } else { + mergeFunction.add(previousKv); + mergeFunction.add(newKv); + } + + KeyValue result = mergeFunction.getResult(); + stored.setRowKind(result.valueKind()); + for (FieldSetter setter : nonKeySetters) { + setter.setFieldFrom(result.value(), stored); + } + return true; + } + + @Override + public int size() { + return buffer.getNumElements(); + } + + @Override + public void forEach(Consumer consumer) throws IOException { + KeyValueIterator iterator = buffer.getEntryIterator(false); + while (iterator.advanceNext()) { + consumer.accept(iterator.getValue()); + } + } + + @Override + public void clear() { + buffer.reset(); + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/mergetree/localmerge/LocalMerger.java b/paimon-core/src/main/java/org/apache/paimon/mergetree/localmerge/LocalMerger.java new file mode 100644 index 000000000000..bec71808a75e --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/mergetree/localmerge/LocalMerger.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.mergetree.localmerge; + +import org.apache.paimon.data.BinaryRow; +import org.apache.paimon.data.InternalRow; +import org.apache.paimon.types.RowKind; + +import java.io.IOException; +import java.util.function.Consumer; + +/** Local merger to merge in memory. */ +public interface LocalMerger { + + boolean put(RowKind rowKind, BinaryRow key, InternalRow value) throws IOException; + + int size(); + + void forEach(Consumer consumer) throws IOException; + + void clear(); +} diff --git a/paimon-core/src/main/java/org/apache/paimon/mergetree/localmerge/SortBufferLocalMerger.java b/paimon-core/src/main/java/org/apache/paimon/mergetree/localmerge/SortBufferLocalMerger.java new file mode 100644 index 000000000000..198e6c67d304 --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/mergetree/localmerge/SortBufferLocalMerger.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.mergetree.localmerge; + +import org.apache.paimon.KeyValue; +import org.apache.paimon.codegen.RecordComparator; +import org.apache.paimon.data.BinaryRow; +import org.apache.paimon.data.InternalRow; +import org.apache.paimon.mergetree.SortBufferWriteBuffer; +import org.apache.paimon.mergetree.compact.MergeFunction; +import org.apache.paimon.types.RowKind; + +import java.io.IOException; +import java.util.function.Consumer; + +/** A {@link LocalMerger} which stores records in {@link SortBufferWriteBuffer}. */ +public class SortBufferLocalMerger implements LocalMerger { + + private final SortBufferWriteBuffer sortBuffer; + private final RecordComparator keyComparator; + private final MergeFunction mergeFunction; + + private long recordCount; + + public SortBufferLocalMerger( + SortBufferWriteBuffer sortBuffer, + RecordComparator keyComparator, + MergeFunction mergeFunction) { + this.sortBuffer = sortBuffer; + this.keyComparator = keyComparator; + this.mergeFunction = mergeFunction; + this.recordCount = 0; + } + + @Override + public boolean put(RowKind rowKind, BinaryRow key, InternalRow value) throws IOException { + return sortBuffer.put(recordCount++, rowKind, key, value); + } + + @Override + public int size() { + return sortBuffer.size(); + } + + @Override + public void forEach(Consumer consumer) throws IOException { + sortBuffer.forEach( + keyComparator, + mergeFunction, + null, + kv -> { + InternalRow row = kv.value(); + row.setRowKind(kv.valueKind()); + consumer.accept(row); + }); + } + + @Override + public void clear() { + sortBuffer.clear(); + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/metastore/AddPartitionCommitCallback.java b/paimon-core/src/main/java/org/apache/paimon/metastore/AddPartitionCommitCallback.java index 4f7d3d554ae2..599f88e512c0 100644 --- a/paimon-core/src/main/java/org/apache/paimon/metastore/AddPartitionCommitCallback.java +++ b/paimon-core/src/main/java/org/apache/paimon/metastore/AddPartitionCommitCallback.java @@ -30,7 +30,10 @@ import org.apache.paimon.shade.guava30.com.google.common.cache.CacheBuilder; import java.time.Duration; +import java.util.ArrayList; import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; /** A {@link CommitCallback} to add newly created partitions to metastore. */ public class AddPartitionCommitCallback implements CommitCallback { @@ -52,30 +55,35 @@ public AddPartitionCommitCallback(MetastoreClient client) { @Override public void call(List committedEntries, Snapshot snapshot) { - committedEntries.stream() - .filter(e -> FileKind.ADD.equals(e.kind())) - .map(ManifestEntry::partition) - .distinct() - .forEach(this::addPartition); + Set partitions = + committedEntries.stream() + .filter(e -> FileKind.ADD.equals(e.kind())) + .map(ManifestEntry::partition) + .collect(Collectors.toSet()); + addPartitions(partitions); } @Override public void retry(ManifestCommittable committable) { - committable.fileCommittables().stream() - .map(CommitMessage::partition) - .distinct() - .forEach(this::addPartition); + Set partitions = + committable.fileCommittables().stream() + .map(CommitMessage::partition) + .collect(Collectors.toSet()); + addPartitions(partitions); } - private void addPartition(BinaryRow partition) { + private void addPartitions(Set partitions) { try { - boolean added = cache.get(partition, () -> false); - if (added) { - return; + List newPartitions = new ArrayList<>(); + for (BinaryRow partition : partitions) { + if (!cache.get(partition, () -> false)) { + newPartitions.add(partition); + } + } + if (!newPartitions.isEmpty()) { + client.addPartitions(newPartitions); + newPartitions.forEach(partition -> cache.put(partition, true)); } - - client.addPartition(partition); - cache.put(partition, true); } catch (Exception e) { throw new RuntimeException(e); } diff --git a/paimon-core/src/main/java/org/apache/paimon/metastore/MetastoreClient.java b/paimon-core/src/main/java/org/apache/paimon/metastore/MetastoreClient.java index de185155d08e..75f7af5abbdc 100644 --- a/paimon-core/src/main/java/org/apache/paimon/metastore/MetastoreClient.java +++ b/paimon-core/src/main/java/org/apache/paimon/metastore/MetastoreClient.java @@ -22,6 +22,8 @@ import java.io.Serializable; import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; /** * A metastore client related to a table. All methods of this interface operate on the same specific @@ -31,12 +33,34 @@ public interface MetastoreClient extends AutoCloseable { void addPartition(BinaryRow partition) throws Exception; + default void addPartitions(List partitions) throws Exception { + for (BinaryRow partition : partitions) { + addPartition(partition); + } + } + void addPartition(LinkedHashMap partitionSpec) throws Exception; + default void addPartitionsSpec(List> partitionSpecsList) + throws Exception { + for (LinkedHashMap partitionSpecs : partitionSpecsList) { + addPartition(partitionSpecs); + } + } + void deletePartition(LinkedHashMap partitionSpec) throws Exception; void markDone(LinkedHashMap partitionSpec) throws Exception; + default void alterPartition( + LinkedHashMap partitionSpec, + Map parameters, + long modifyTime, + boolean ignoreIfNotExist) + throws Exception { + throw new UnsupportedOperationException(); + } + /** Factory to create {@link MetastoreClient}. */ interface Factory extends Serializable { diff --git a/paimon-core/src/main/java/org/apache/paimon/migrate/FileMetaUtils.java b/paimon-core/src/main/java/org/apache/paimon/migrate/FileMetaUtils.java index 4c299bd6074b..391c5f9bb615 100644 --- a/paimon-core/src/main/java/org/apache/paimon/migrate/FileMetaUtils.java +++ b/paimon-core/src/main/java/org/apache/paimon/migrate/FileMetaUtils.java @@ -156,7 +156,7 @@ private static DataFileMeta constructFileMeta( Pair fileInfo = simpleStatsExtractor.extractWithFileInfo(fileIO, path); - SimpleStats stats = statsArraySerializer.toBinary(fileInfo.getLeft()); + SimpleStats stats = statsArraySerializer.toBinaryAllMode(fileInfo.getLeft()); return DataFileMeta.forAppend( fileName, @@ -166,7 +166,10 @@ private static DataFileMeta constructFileMeta( 0, 0, ((FileStoreTable) table).schema().id(), - FileSource.APPEND); + Collections.emptyList(), + null, + FileSource.APPEND, + null); } public static BinaryRow writePartitionValue( diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/AbstractFileStoreScan.java b/paimon-core/src/main/java/org/apache/paimon/operation/AbstractFileStoreScan.java index 683e6ffda481..c73a92062b80 100644 --- a/paimon-core/src/main/java/org/apache/paimon/operation/AbstractFileStoreScan.java +++ b/paimon-core/src/main/java/org/apache/paimon/operation/AbstractFileStoreScan.java @@ -23,7 +23,7 @@ import org.apache.paimon.data.InternalRow; import org.apache.paimon.manifest.BucketEntry; import org.apache.paimon.manifest.FileEntry; -import org.apache.paimon.manifest.FileKind; +import org.apache.paimon.manifest.FileEntry.Identifier; import org.apache.paimon.manifest.ManifestCacheFilter; import org.apache.paimon.manifest.ManifestEntry; import org.apache.paimon.manifest.ManifestEntrySerializer; @@ -43,8 +43,6 @@ import org.apache.paimon.utils.Pair; import org.apache.paimon.utils.SnapshotManager; -import org.apache.paimon.shade.guava30.com.google.common.collect.Iterators; - import javax.annotation.Nullable; import java.util.ArrayList; @@ -62,9 +60,9 @@ import java.util.stream.Collectors; import static org.apache.paimon.utils.ManifestReadThreadPool.getExecutorService; +import static org.apache.paimon.utils.ManifestReadThreadPool.randomlyExecuteSequentialReturn; import static org.apache.paimon.utils.ManifestReadThreadPool.sequentialBatchedExecute; import static org.apache.paimon.utils.Preconditions.checkArgument; -import static org.apache.paimon.utils.Preconditions.checkState; import static org.apache.paimon.utils.ThreadPoolUtils.randomlyOnlyExecute; /** Default implementation of {@link FileStoreScan}. */ @@ -82,7 +80,6 @@ public abstract class AbstractFileStoreScan implements FileStoreScan { private Snapshot specifiedSnapshot = null; private Filter bucketFilter = null; private BiFilter totalAwareBucketFilter = null; - private List specifiedManifests = null; protected ScanMode scanMode = ScanMode.ALL; private Filter levelFilter = null; private Filter manifestEntryFilter = null; @@ -90,6 +87,7 @@ public abstract class AbstractFileStoreScan implements FileStoreScan { private ManifestCacheFilter manifestCacheFilter = null; private ScanMetrics scanMetrics = null; + private boolean dropStats; public AbstractFileStoreScan( ManifestsReader manifestsReader, @@ -105,6 +103,7 @@ public AbstractFileStoreScan( this.manifestFileFactory = manifestFileFactory; this.tableSchemas = new ConcurrentHashMap<>(); this.parallelism = parallelism; + this.dropStats = false; } @Override @@ -160,25 +159,16 @@ public FileStoreScan withPartitionBucket(BinaryRow partition, int bucket) { @Override public FileStoreScan withSnapshot(long snapshotId) { - checkState(specifiedManifests == null, "Cannot set both snapshot and manifests."); this.specifiedSnapshot = snapshotManager.snapshot(snapshotId); return this; } @Override public FileStoreScan withSnapshot(Snapshot snapshot) { - checkState(specifiedManifests == null, "Cannot set both snapshot and manifests."); this.specifiedSnapshot = snapshot; return this; } - @Override - public FileStoreScan withManifestList(List manifests) { - checkState(specifiedSnapshot == null, "Cannot set both snapshot and manifests."); - this.specifiedManifests = manifests; - return this; - } - @Override public FileStoreScan withKind(ScanMode scanMode) { this.scanMode = scanMode; @@ -191,6 +181,11 @@ public FileStoreScan withLevelFilter(Filter levelFilter) { return this; } + @Override + public FileStoreScan enableValueFilter() { + return this; + } + @Override public FileStoreScan withManifestEntryFilter(Filter filter) { this.manifestEntryFilter = filter; @@ -215,6 +210,12 @@ public FileStoreScan withMetrics(ScanMetrics metrics) { return this; } + @Override + public FileStoreScan dropStats() { + this.dropStats = true; + return this; + } + @Nullable @Override public Integer parallelism() { @@ -233,47 +234,46 @@ public Plan plan() { Snapshot snapshot = manifestsResult.snapshot; List manifests = manifestsResult.filteredManifests; - long startDataFiles = - manifestsResult.allManifests.stream() - .mapToLong(f -> f.numAddedFiles() - f.numDeletedFiles()) - .sum(); - - Collection mergedEntries = - readAndMergeFileEntries(manifests, this::readManifest); - - long skippedByPartitionAndStats = startDataFiles - mergedEntries.size(); - - // We group files by bucket here, and filter them by the whole bucket filter. - // Why do this: because in primary key table, we can't just filter the value - // by the stat in files (see `PrimaryKeyFileStoreTable.nonPartitionFilterConsumer`), - // but we can do this by filter the whole bucket files - List files = - mergedEntries.stream() - .collect( - Collectors.groupingBy( - // we use LinkedHashMap to avoid disorder - file -> Pair.of(file.partition(), file.bucket()), - LinkedHashMap::new, - Collectors.toList())) - .values() - .stream() - .map(this::filterWholeBucketByStats) - .flatMap(Collection::stream) - .collect(Collectors.toList()); + Iterator iterator = readManifestEntries(manifests, false); + List files = new ArrayList<>(); + while (iterator.hasNext()) { + files.add(iterator.next()); + } + + if (wholeBucketFilterEnabled()) { + // We group files by bucket here, and filter them by the whole bucket filter. + // Why do this: because in primary key table, we can't just filter the value + // by the stat in files (see `PrimaryKeyFileStoreTable.nonPartitionFilterConsumer`), + // but we can do this by filter the whole bucket files + files = + files.stream() + .collect( + Collectors.groupingBy( + // we use LinkedHashMap to avoid disorder + file -> Pair.of(file.partition(), file.bucket()), + LinkedHashMap::new, + Collectors.toList())) + .values() + .stream() + .map(this::filterWholeBucketByStats) + .flatMap(Collection::stream) + .collect(Collectors.toList()); + } + + List result = files; - long skippedByWholeBucketFiles = mergedEntries.size() - files.size(); long scanDuration = (System.nanoTime() - started) / 1_000_000; - checkState( - startDataFiles - skippedByPartitionAndStats - skippedByWholeBucketFiles - == files.size()); if (scanMetrics != null) { + long allDataFiles = + manifestsResult.allManifests.stream() + .mapToLong(f -> f.numAddedFiles() - f.numDeletedFiles()) + .sum(); scanMetrics.reportScan( new ScanStats( scanDuration, manifests.size(), - skippedByPartitionAndStats, - skippedByWholeBucketFiles, - files.size())); + allDataFiles - result.size(), + result.size())); } return new Plan() { @@ -291,7 +291,7 @@ public Snapshot snapshot() { @Override public List files() { - return files; + return result; } }; } @@ -299,9 +299,15 @@ public List files() { @Override public List readSimpleEntries() { List manifests = readManifests().filteredManifests; - Collection mergedEntries = - readAndMergeFileEntries(manifests, this::readSimpleEntries); - return new ArrayList<>(mergedEntries); + Iterator iterator = + scanMode == ScanMode.ALL + ? readAndMergeFileEntries(manifests, SimpleFileEntry::from, false) + : readAndNoMergeFileEntries(manifests, SimpleFileEntry::from, false); + List result = new ArrayList<>(); + while (iterator.hasNext()) { + result.add(iterator.next()); + } + return result; } @Override @@ -330,30 +336,60 @@ public List readBucketEntries() { @Override public Iterator readFileIterator() { - List manifests = readManifests().filteredManifests; - Set deleteEntries = - FileEntry.readDeletedEntries(this::readSimpleEntries, manifests, parallelism); - Iterator iterator = - sequentialBatchedExecute(this::readManifest, manifests, parallelism).iterator(); - return Iterators.filter( - iterator, - entry -> - entry != null - && entry.kind() == FileKind.ADD - && !deleteEntries.contains(entry.identifier())); + // useSequential: reduce memory and iterator can be stopping + return readManifestEntries(readManifests().filteredManifests, true); } - public Collection readAndMergeFileEntries( - List manifests, Function> manifestReader) { - return FileEntry.mergeEntries( - sequentialBatchedExecute(manifestReader, manifests, parallelism)); + private Iterator readManifestEntries( + List manifests, boolean useSequential) { + return scanMode == ScanMode.ALL + ? readAndMergeFileEntries(manifests, Function.identity(), useSequential) + : readAndNoMergeFileEntries(manifests, Function.identity(), useSequential); } - private ManifestsReader.Result readManifests() { - if (specifiedManifests != null) { - return new ManifestsReader.Result(null, specifiedManifests, specifiedManifests); + private Iterator readAndMergeFileEntries( + List manifests, + Function, List> converter, + boolean useSequential) { + Set deletedEntries = + FileEntry.readDeletedEntries( + manifest -> readManifest(manifest, FileEntry.deletedFilter(), null), + manifests, + parallelism); + + manifests = + manifests.stream() + .filter(file -> file.numAddedFiles() > 0) + .collect(Collectors.toList()); + + Function> processor = + manifest -> + converter.apply( + readManifest( + manifest, + FileEntry.addFilter(), + entry -> !deletedEntries.contains(entry.identifier()))); + if (useSequential) { + return sequentialBatchedExecute(processor, manifests, parallelism).iterator(); + } else { + return randomlyExecuteSequentialReturn(processor, manifests, parallelism); } + } + + private Iterator readAndNoMergeFileEntries( + List manifests, + Function, List> converter, + boolean useSequential) { + Function> reader = + manifest -> converter.apply(readManifest(manifest)); + if (useSequential) { + return sequentialBatchedExecute(reader, manifests, parallelism).iterator(); + } else { + return randomlyExecuteSequentialReturn(reader, manifests, parallelism); + } + } + private ManifestsReader.Result readManifests() { return manifestsReader.read(specifiedSnapshot, scanMode); } @@ -371,12 +407,24 @@ protected TableSchema scanTableSchema(long id) { /** Note: Keep this thread-safe. */ protected abstract boolean filterByStats(ManifestEntry entry); - /** Note: Keep this thread-safe. */ - protected abstract List filterWholeBucketByStats(List entries); + protected boolean wholeBucketFilterEnabled() { + return false; + } + + protected List filterWholeBucketByStats(List entries) { + return entries; + } /** Note: Keep this thread-safe. */ @Override public List readManifest(ManifestFileMeta manifest) { + return readManifest(manifest, null, null); + } + + private List readManifest( + ManifestFileMeta manifest, + @Nullable Filter additionalFilter, + @Nullable Filter additionalTFilter) { List entries = manifestFileFactory .create() @@ -384,29 +432,24 @@ public List readManifest(ManifestFileMeta manifest) { manifest.fileName(), manifest.fileSize(), createCacheRowFilter(), - createEntryRowFilter()); - List filteredEntries = new ArrayList<>(entries.size()); - for (ManifestEntry entry : entries) { - if ((manifestEntryFilter == null || manifestEntryFilter.test(entry)) - && filterByStats(entry)) { - filteredEntries.add(entry); + createEntryRowFilter().and(additionalFilter), + entry -> + (additionalTFilter == null || additionalTFilter.test(entry)) + && (manifestEntryFilter == null + || manifestEntryFilter.test(entry)) + && filterByStats(entry)); + if (dropStats) { + List copied = new ArrayList<>(entries.size()); + for (ManifestEntry entry : entries) { + copied.add(dropStats(entry)); } + entries = copied; } - return filteredEntries; + return entries; } - /** Note: Keep this thread-safe. */ - private List readSimpleEntries(ManifestFileMeta manifest) { - return manifestFileFactory - .createSimpleFileEntryReader() - .read( - manifest.fileName(), - manifest.fileSize(), - // use filter for ManifestEntry - // currently, projection is not pushed down to file format - // see SimpleFileEntrySerializer - createCacheRowFilter(), - createEntryRowFilter()); + protected ManifestEntry dropStats(ManifestEntry entry) { + return entry.copyWithoutStats(); } /** diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/AbstractFileStoreWrite.java b/paimon-core/src/main/java/org/apache/paimon/operation/AbstractFileStoreWrite.java index dab20d642cb9..14dfe75a6e35 100644 --- a/paimon-core/src/main/java/org/apache/paimon/operation/AbstractFileStoreWrite.java +++ b/paimon-core/src/main/java/org/apache/paimon/operation/AbstractFileStoreWrite.java @@ -18,6 +18,7 @@ package org.apache.paimon.operation; +import org.apache.paimon.CoreOptions; import org.apache.paimon.Snapshot; import org.apache.paimon.annotation.VisibleForTesting; import org.apache.paimon.compact.CompactDeletionFile; @@ -88,6 +89,7 @@ public abstract class AbstractFileStoreWrite implements FileStoreWrite { protected CompactionMetrics compactionMetrics = null; protected final String tableName; private boolean isInsertOnly; + private boolean legacyPartitionName; protected AbstractFileStoreWrite( SnapshotManager snapshotManager, @@ -95,11 +97,19 @@ protected AbstractFileStoreWrite( @Nullable IndexMaintainer.Factory indexFactory, @Nullable DeletionVectorsMaintainer.Factory dvMaintainerFactory, String tableName, + CoreOptions options, int totalBuckets, RowType partitionType, - int writerNumberMax) { + int writerNumberMax, + boolean legacyPartitionName) { this.snapshotManager = snapshotManager; this.scan = scan; + // Statistic is useless in writer + if (options.manifestDeleteFileDropStats()) { + if (this.scan != null) { + this.scan.dropStats(); + } + } this.indexFactory = indexFactory; this.dvMaintainerFactory = dvMaintainerFactory; this.totalBuckets = totalBuckets; @@ -107,6 +117,7 @@ protected AbstractFileStoreWrite( this.writers = new HashMap<>(); this.tableName = tableName; this.writerNumberMax = writerNumberMax; + this.legacyPartitionName = legacyPartitionName; } @Override @@ -469,7 +480,8 @@ private List scanExistingFileMetas( ? "partition " + getPartitionComputer( partitionType, - PARTITION_DEFAULT_NAME.defaultValue()) + PARTITION_DEFAULT_NAME.defaultValue(), + legacyPartitionName) .generatePartValues(partition) : "table"; throw new RuntimeException( diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/AppendOnlyFileStoreScan.java b/paimon-core/src/main/java/org/apache/paimon/operation/AppendOnlyFileStoreScan.java index a3bc9c22dc53..d2ca5da42249 100644 --- a/paimon-core/src/main/java/org/apache/paimon/operation/AppendOnlyFileStoreScan.java +++ b/paimon-core/src/main/java/org/apache/paimon/operation/AppendOnlyFileStoreScan.java @@ -25,9 +25,8 @@ import org.apache.paimon.predicate.Predicate; import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.schema.TableSchema; -import org.apache.paimon.stats.SimpleStats; -import org.apache.paimon.stats.SimpleStatsConverter; -import org.apache.paimon.stats.SimpleStatsConverters; +import org.apache.paimon.stats.SimpleStatsEvolution; +import org.apache.paimon.stats.SimpleStatsEvolutions; import org.apache.paimon.types.RowType; import org.apache.paimon.utils.SnapshotManager; @@ -35,14 +34,13 @@ import java.io.IOException; import java.util.HashMap; -import java.util.List; import java.util.Map; /** {@link FileStoreScan} for {@link AppendOnlyFileStore}. */ public class AppendOnlyFileStoreScan extends AbstractFileStoreScan { private final BucketSelectConverter bucketSelectConverter; - private final SimpleStatsConverters simpleStatsConverters; + private final SimpleStatsEvolutions simpleStatsEvolutions; private final boolean fileIndexReadEnabled; @@ -68,8 +66,8 @@ public AppendOnlyFileStoreScan( manifestFileFactory, scanManifestParallelism); this.bucketSelectConverter = bucketSelectConverter; - this.simpleStatsConverters = - new SimpleStatsConverters(sid -> scanTableSchema(sid).fields(), schema.id()); + this.simpleStatsEvolutions = + new SimpleStatsEvolutions(sid -> scanTableSchema(sid).fields(), schema.id()); this.fileIndexReadEnabled = fileIndexReadEnabled; } @@ -86,24 +84,21 @@ protected boolean filterByStats(ManifestEntry entry) { return true; } - SimpleStatsConverter serializer = - simpleStatsConverters.getOrCreate(entry.file().schemaId()); - SimpleStats stats = entry.file().valueStats(); + SimpleStatsEvolution evolution = simpleStatsEvolutions.getOrCreate(entry.file().schemaId()); + SimpleStatsEvolution.Result stats = + evolution.evolution( + entry.file().valueStats(), + entry.file().rowCount(), + entry.file().valueStatsCols()); return filter.test( entry.file().rowCount(), - serializer.evolution(stats.minValues()), - serializer.evolution(stats.maxValues()), - serializer.evolution(stats.nullCounts(), entry.file().rowCount())) + stats.minValues(), + stats.maxValues(), + stats.nullCounts()) && (!fileIndexReadEnabled || testFileIndex(entry.file().embeddedIndex(), entry)); } - @Override - protected List filterWholeBucketByStats(List entries) { - // We don't need to filter per-bucket entries here - return entries; - } - private boolean testFileIndex(@Nullable byte[] embeddedIndexBytes, ManifestEntry entry) { if (embeddedIndexBytes == null) { return true; @@ -114,11 +109,11 @@ private boolean testFileIndex(@Nullable byte[] embeddedIndexBytes, ManifestEntry Predicate dataPredicate = dataFilterMapping.computeIfAbsent( entry.file().schemaId(), - id -> simpleStatsConverters.convertFilter(entry.file().schemaId(), filter)); + id -> simpleStatsEvolutions.convertFilter(entry.file().schemaId(), filter)); try (FileIndexPredicate predicate = new FileIndexPredicate(embeddedIndexBytes, dataRowType)) { - return predicate.testPredicate(dataPredicate); + return predicate.evaluate(dataPredicate).remain(); } catch (IOException e) { throw new RuntimeException("Exception happens while checking predicate.", e); } diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/AppendOnlyFileStoreWrite.java b/paimon-core/src/main/java/org/apache/paimon/operation/AppendOnlyFileStoreWrite.java index 0a4d5d56a13e..4a6196453df6 100644 --- a/paimon-core/src/main/java/org/apache/paimon/operation/AppendOnlyFileStoreWrite.java +++ b/paimon-core/src/main/java/org/apache/paimon/operation/AppendOnlyFileStoreWrite.java @@ -132,7 +132,8 @@ protected RecordWriter createWriter( statsCollectors, options.writeBufferSpillDiskSize(), fileIndexOptions, - options.asyncFileWrite()); + options.asyncFileWrite(), + options.statsDenseStore()); } protected abstract CompactManager getCompactManager( @@ -193,7 +194,8 @@ private RowDataRollingFileWriter createRollingFileWriter( statsCollectors, fileIndexOptions, FileSource.COMPACT, - options.asyncFileWrite()); + options.asyncFileWrite(), + options.statsDenseStore()); } private RecordReaderIterator createFilesIterator( @@ -210,6 +212,9 @@ protected void forceBufferSpill() throws Exception { if (ioManager == null) { return; } + if (forceBufferSpill) { + return; + } forceBufferSpill = true; LOG.info( "Force buffer spill for append-only file store write, writer number is: {}", diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/ChangelogDeletion.java b/paimon-core/src/main/java/org/apache/paimon/operation/ChangelogDeletion.java index c20405ff26c9..069e57bb3daf 100644 --- a/paimon-core/src/main/java/org/apache/paimon/operation/ChangelogDeletion.java +++ b/paimon-core/src/main/java/org/apache/paimon/operation/ChangelogDeletion.java @@ -23,8 +23,8 @@ import org.apache.paimon.fs.FileIO; import org.apache.paimon.index.IndexFileHandler; import org.apache.paimon.index.IndexFileMeta; +import org.apache.paimon.manifest.ExpireFileEntry; import org.apache.paimon.manifest.IndexManifestEntry; -import org.apache.paimon.manifest.ManifestEntry; import org.apache.paimon.manifest.ManifestFile; import org.apache.paimon.manifest.ManifestFileMeta; import org.apache.paimon.manifest.ManifestList; @@ -60,7 +60,7 @@ public ChangelogDeletion( } @Override - public void cleanUnusedDataFiles(Changelog changelog, Predicate skipper) { + public void cleanUnusedDataFiles(Changelog changelog, Predicate skipper) { if (changelog.changelogManifestList() != null) { deleteAddedDataFiles(changelog.changelogManifestList()); } diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/CleanOrphanFilesResult.java b/paimon-core/src/main/java/org/apache/paimon/operation/CleanOrphanFilesResult.java new file mode 100644 index 000000000000..d29eede720ac --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/operation/CleanOrphanFilesResult.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.operation; + +import org.apache.paimon.fs.Path; + +import javax.annotation.Nullable; + +import java.util.List; + +/** The result of OrphanFilesClean. */ +public class CleanOrphanFilesResult { + + private final long deletedFileCount; + private final long deletedFileTotalLenInBytes; + + @Nullable private final List deletedFilesPath; + + public CleanOrphanFilesResult(long deletedFileCount, long deletedFileTotalLenInBytes) { + this(deletedFileCount, deletedFileTotalLenInBytes, null); + } + + public CleanOrphanFilesResult( + long deletedFileCount, + long deletedFileTotalLenInBytes, + @Nullable List deletedFilesPath) { + this.deletedFilesPath = deletedFilesPath; + this.deletedFileCount = deletedFileCount; + this.deletedFileTotalLenInBytes = deletedFileTotalLenInBytes; + } + + public long getDeletedFileCount() { + return deletedFileCount; + } + + public long getDeletedFileTotalLenInBytes() { + return deletedFileTotalLenInBytes; + } + + @Nullable + public List getDeletedFilesPath() { + return deletedFilesPath; + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/FileDeletionBase.java b/paimon-core/src/main/java/org/apache/paimon/operation/FileDeletionBase.java index 303a074b0cb8..cfecd767b6fb 100644 --- a/paimon-core/src/main/java/org/apache/paimon/operation/FileDeletionBase.java +++ b/paimon-core/src/main/java/org/apache/paimon/operation/FileDeletionBase.java @@ -24,10 +24,11 @@ import org.apache.paimon.fs.Path; import org.apache.paimon.index.IndexFileHandler; import org.apache.paimon.index.IndexFileMeta; +import org.apache.paimon.manifest.ExpireFileEntry; import org.apache.paimon.manifest.FileEntry; +import org.apache.paimon.manifest.FileEntry.Identifier; import org.apache.paimon.manifest.FileKind; import org.apache.paimon.manifest.IndexManifestEntry; -import org.apache.paimon.manifest.ManifestEntry; import org.apache.paimon.manifest.ManifestFile; import org.apache.paimon.manifest.ManifestFileMeta; import org.apache.paimon.manifest.ManifestList; @@ -46,7 +47,6 @@ import java.util.Collections; import java.util.HashMap; import java.util.HashSet; -import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Set; @@ -54,7 +54,6 @@ import java.util.concurrent.Executor; import java.util.function.Consumer; import java.util.function.Predicate; -import java.util.stream.Collectors; /** * Base class for file deletion including methods for clean data files, manifest files and empty @@ -110,7 +109,7 @@ public FileDeletionBase( * @param skipper if the test result of a data file is true, it will be skipped when deleting; * else it will be deleted */ - public abstract void cleanUnusedDataFiles(T snapshot, Predicate skipper); + public abstract void cleanUnusedDataFiles(T snapshot, Predicate skipper); /** * Clean metadata files that will not be used anymore of a snapshot, including data manifests, @@ -164,21 +163,23 @@ public void cleanEmptyDirectories() { deletionBuckets.clear(); } - protected void recordDeletionBuckets(ManifestEntry entry) { + protected void recordDeletionBuckets(ExpireFileEntry entry) { deletionBuckets .computeIfAbsent(entry.partition(), p -> new HashSet<>()) .add(entry.bucket()); } - public void cleanUnusedDataFiles(String manifestList, Predicate skipper) { + public void cleanUnusedDataFiles(String manifestList, Predicate skipper) { // try read manifests - List manifestFileNames = readManifestFileNames(tryReadManifestList(manifestList)); - List manifestEntries; + List manifests = tryReadManifestList(manifestList); + List manifestEntries; // data file path -> (original manifest entry, extra file paths) - Map>> dataFileToDelete = new HashMap<>(); - for (String manifest : manifestFileNames) { + Map>> dataFileToDelete = new HashMap<>(); + for (ManifestFileMeta manifest : manifests) { try { - manifestEntries = manifestFile.read(manifest); + manifestEntries = + manifestFile.readExpireFileEntries( + manifest.fileName(), manifest.fileSize()); } catch (Exception e) { // cancel deletion if any exception occurs LOG.warn("Failed to read some manifest files. Cancel deletion.", e); @@ -192,12 +193,12 @@ public void cleanUnusedDataFiles(String manifestList, Predicate s } protected void doCleanUnusedDataFile( - Map>> dataFileToDelete, - Predicate skipper) { + Map>> dataFileToDelete, + Predicate skipper) { List actualDataFileToDelete = new ArrayList<>(); dataFileToDelete.forEach( (path, pair) -> { - ManifestEntry entry = pair.getLeft(); + ExpireFileEntry entry = pair.getLeft(); // check whether we should skip the data file if (!skipper.test(entry)) { // delete data files @@ -211,20 +212,20 @@ protected void doCleanUnusedDataFile( } protected void getDataFileToDelete( - Map>> dataFileToDelete, - List dataFileEntries) { + Map>> dataFileToDelete, + List dataFileEntries) { // we cannot delete a data file directly when we meet a DELETE entry, because that // file might be upgraded - for (ManifestEntry entry : dataFileEntries) { + for (ExpireFileEntry entry : dataFileEntries) { Path bucketPath = pathFactory.bucketPath(entry.partition(), entry.bucket()); - Path dataFilePath = new Path(bucketPath, entry.file().fileName()); + Path dataFilePath = new Path(bucketPath, entry.fileName()); switch (entry.kind()) { case ADD: dataFileToDelete.remove(dataFilePath); break; case DELETE: - List extraFiles = new ArrayList<>(entry.file().extraFiles().size()); - for (String file : entry.file().extraFiles()) { + List extraFiles = new ArrayList<>(entry.extraFiles().size()); + for (String file : entry.extraFiles()) { extraFiles.add(new Path(bucketPath, file)); } dataFileToDelete.put(dataFilePath, Pair.of(entry, extraFiles)); @@ -242,27 +243,28 @@ protected void getDataFileToDelete( * @param manifestListName name of manifest list */ public void deleteAddedDataFiles(String manifestListName) { - List manifestFileNames = - readManifestFileNames(tryReadManifestList(manifestListName)); - for (String file : manifestFileNames) { + List manifests = tryReadManifestList(manifestListName); + for (ManifestFileMeta manifest : manifests) { try { - List manifestEntries = manifestFile.read(file); + List manifestEntries = + manifestFile.readExpireFileEntries( + manifest.fileName(), manifest.fileSize()); deleteAddedDataFiles(manifestEntries); } catch (Exception e) { // We want to delete the data file, so just ignore the unavailable files - LOG.info("Failed to read manifest " + file + ". Ignore it.", e); + LOG.info("Failed to read manifest " + manifest.fileName() + ". Ignore it.", e); } } } - private void deleteAddedDataFiles(List manifestEntries) { + private void deleteAddedDataFiles(List manifestEntries) { List dataFileToDelete = new ArrayList<>(); - for (ManifestEntry entry : manifestEntries) { + for (ExpireFileEntry entry : manifestEntries) { if (entry.kind() == FileKind.ADD) { dataFileToDelete.add( new Path( pathFactory.bucketPath(entry.partition(), entry.bucket()), - entry.file().fileName())); + entry.fileName())); recordDeletionBuckets(entry); } } @@ -327,7 +329,7 @@ protected void cleanUnusedManifests( cleanUnusedStatisticsManifests(snapshot, skippingSet); } - public Predicate createDataFileSkipperForTags( + public Predicate createDataFileSkipperForTags( List taggedSnapshots, long expiringSnapshotId) throws Exception { int index = SnapshotManager.findPreviousSnapshot(taggedSnapshots, expiringSnapshotId); // refresh tag data files @@ -358,18 +360,6 @@ protected List tryReadManifestList(String manifestListName) { } } - protected List tryReadDataManifests(Snapshot snapshot) { - List manifestFileMetas = tryReadManifestList(snapshot.baseManifestList()); - manifestFileMetas.addAll(tryReadManifestList(snapshot.deltaManifestList())); - return readManifestFileNames(manifestFileMetas); - } - - protected List readManifestFileNames(List manifestFileMetas) { - return manifestFileMetas.stream() - .map(ManifestFileMeta::fileName) - .collect(Collectors.toCollection(LinkedList::new)); - } - /** * NOTE: This method is used for building data file skipping set. If failed to read some * manifests, it will throw exception which callers must handle. @@ -377,23 +367,26 @@ protected List readManifestFileNames(List manifestFile protected void addMergedDataFiles( Map>> dataFiles, Snapshot snapshot) throws IOException { - for (ManifestEntry entry : readMergedDataFiles(snapshot)) { + for (ExpireFileEntry entry : readMergedDataFiles(snapshot)) { dataFiles .computeIfAbsent(entry.partition(), p -> new HashMap<>()) .computeIfAbsent(entry.bucket(), b -> new HashSet<>()) - .add(entry.file().fileName()); + .add(entry.fileName()); } } - protected Collection readMergedDataFiles(Snapshot snapshot) throws IOException { + protected Collection readMergedDataFiles(Snapshot snapshot) + throws IOException { // read data manifests - List files = tryReadDataManifests(snapshot); + + List manifests = tryReadManifestList(snapshot.baseManifestList()); + manifests.addAll(tryReadManifestList(snapshot.deltaManifestList())); // read and merge manifest entries - Map map = new HashMap<>(); - for (String manifest : files) { - List entries; - entries = manifestFile.readWithIOException(manifest); + Map map = new HashMap<>(); + for (ManifestFileMeta manifest : manifests) { + List entries = + manifestFile.readExpireFileEntries(manifest.fileName(), manifest.fileSize()); FileEntry.mergeEntries(entries, map); } @@ -401,12 +394,12 @@ protected Collection readMergedDataFiles(Snapshot snapshot) throw } protected boolean containsDataFile( - Map>> dataFiles, ManifestEntry testee) { - Map> buckets = dataFiles.get(testee.partition()); + Map>> dataFiles, ExpireFileEntry entry) { + Map> buckets = dataFiles.get(entry.partition()); if (buckets != null) { - Set fileNames = buckets.get(testee.bucket()); + Set fileNames = buckets.get(entry.bucket()); if (fileNames != null) { - return fileNames.contains(testee.file().fileName()); + return fileNames.contains(entry.fileName()); } } return false; diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/FileStoreCommit.java b/paimon-core/src/main/java/org/apache/paimon/operation/FileStoreCommit.java index e15225793076..43456cbe7184 100644 --- a/paimon-core/src/main/java/org/apache/paimon/operation/FileStoreCommit.java +++ b/paimon-core/src/main/java/org/apache/paimon/operation/FileStoreCommit.java @@ -74,6 +74,9 @@ void overwrite( void truncateTable(long commitIdentifier); + /** Compact the manifest entries only. */ + void compactManifest(); + /** Abort an unsuccessful commit. The data files will be deleted. */ void abort(List commitMessages); diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/FileStoreCommitImpl.java b/paimon-core/src/main/java/org/apache/paimon/operation/FileStoreCommitImpl.java index 0b19cae17793..001132e1671c 100644 --- a/paimon-core/src/main/java/org/apache/paimon/operation/FileStoreCommitImpl.java +++ b/paimon-core/src/main/java/org/apache/paimon/operation/FileStoreCommitImpl.java @@ -18,6 +18,7 @@ package org.apache.paimon.operation; +import org.apache.paimon.CoreOptions; import org.apache.paimon.Snapshot; import org.apache.paimon.annotation.VisibleForTesting; import org.apache.paimon.data.BinaryRow; @@ -49,6 +50,7 @@ import org.apache.paimon.table.sink.CommitCallback; import org.apache.paimon.table.sink.CommitMessage; import org.apache.paimon.table.sink.CommitMessageImpl; +import org.apache.paimon.table.source.ScanMode; import org.apache.paimon.types.RowType; import org.apache.paimon.utils.FileStorePathFactory; import org.apache.paimon.utils.IOUtils; @@ -81,6 +83,7 @@ import static org.apache.paimon.manifest.ManifestEntry.recordCount; import static org.apache.paimon.manifest.ManifestEntry.recordCountAdd; import static org.apache.paimon.manifest.ManifestEntry.recordCountDelete; +import static org.apache.paimon.partition.PartitionPredicate.createBinaryPartitions; import static org.apache.paimon.partition.PartitionPredicate.createPartitionPredicate; import static org.apache.paimon.utils.BranchManager.DEFAULT_MAIN_BRANCH; import static org.apache.paimon.utils.InternalRowPartitionComputer.partToSimpleString; @@ -111,6 +114,7 @@ public class FileStoreCommitImpl implements FileStoreCommit { private final FileIO fileIO; private final SchemaManager schemaManager; + private final String tableName; private final String commitUser; private final RowType partitionType; private final String partitionDefaultName; @@ -131,6 +135,7 @@ public class FileStoreCommitImpl implements FileStoreCommit { private final List commitCallbacks; private final StatsFileHandler statsFileHandler; private final BucketMode bucketMode; + private long commitTimeout; private final int commitMaxRetries; @Nullable private Lock lock; @@ -141,8 +146,10 @@ public class FileStoreCommitImpl implements FileStoreCommit { public FileStoreCommitImpl( FileIO fileIO, SchemaManager schemaManager, + String tableName, String commitUser, RowType partitionType, + CoreOptions options, String partitionDefaultName, FileStorePathFactory pathFactory, SnapshotManager snapshotManager, @@ -161,9 +168,11 @@ public FileStoreCommitImpl( BucketMode bucketMode, @Nullable Integer manifestReadParallelism, List commitCallbacks, - int commitMaxRetries) { + int commitMaxRetries, + long commitTimeout) { this.fileIO = fileIO; this.schemaManager = schemaManager; + this.tableName = tableName; this.commitUser = commitUser; this.partitionType = partitionType; this.partitionDefaultName = partitionDefaultName; @@ -173,6 +182,10 @@ public FileStoreCommitImpl( this.manifestList = manifestListFactory.create(); this.indexManifestFile = indexManifestFileFactory.create(); this.scan = scan; + // Stats in DELETE Manifest Entries is useless + if (options.manifestDeleteFileDropStats()) { + this.scan.dropStats(); + } this.numBucket = numBucket; this.manifestTargetSize = manifestTargetSize; this.manifestFullCompactionSize = manifestFullCompactionSize; @@ -183,6 +196,7 @@ public FileStoreCommitImpl( this.manifestReadParallelism = manifestReadParallelism; this.commitCallbacks = commitCallbacks; this.commitMaxRetries = commitMaxRetries; + this.commitTimeout = commitTimeout; this.lock = null; this.ignoreEmptyCommit = true; @@ -251,7 +265,7 @@ public void commit( Map properties, boolean checkAppendFiles) { if (LOG.isDebugEnabled()) { - LOG.debug("Ready to commit\n" + committable.toString()); + LOG.debug("Ready to commit\n{}", committable.toString()); } long started = System.nanoTime(); @@ -520,17 +534,26 @@ public void dropPartitions(List> partitions, long commitIden partitions.stream().map(Objects::toString).collect(Collectors.joining(","))); } - // partitions may be partial partition fields, so here must to use predicate way. - Predicate predicate = - partitions.stream() - .map( - partition -> - createPartitionPredicate( - partition, partitionType, partitionDefaultName)) - .reduce(PredicateBuilder::or) - .orElseThrow(() -> new RuntimeException("Failed to get partition filter.")); - PartitionPredicate partitionFilter = - PartitionPredicate.fromPredicate(partitionType, predicate); + boolean fullMode = + partitions.stream().allMatch(part -> part.size() == partitionType.getFieldCount()); + PartitionPredicate partitionFilter; + if (fullMode) { + List binaryPartitions = + createBinaryPartitions(partitions, partitionType, partitionDefaultName); + partitionFilter = PartitionPredicate.fromMultiple(partitionType, binaryPartitions); + } else { + // partitions may be partial partition fields, so here must to use predicate way. + Predicate predicate = + partitions.stream() + .map( + partition -> + createPartitionPredicate( + partition, partitionType, partitionDefaultName)) + .reduce(PredicateBuilder::or) + .orElseThrow( + () -> new RuntimeException("Failed to get partition filter.")); + partitionFilter = PartitionPredicate.fromPredicate(partitionType, predicate); + } tryOverwrite( partitionFilter, @@ -711,32 +734,44 @@ private int tryCommit( ConflictCheck conflictCheck, String branchName, @Nullable String statsFileName) { - int cnt = 0; + int retryCount = 0; + RetryResult retryResult = null; + long startMillis = System.currentTimeMillis(); while (true) { Snapshot latestSnapshot = snapshotManager.latestSnapshot(); - cnt++; - if (cnt >= commitMaxRetries) { + CommitResult result = + tryCommitOnce( + retryResult, + tableFiles, + changelogFiles, + indexFiles, + identifier, + watermark, + logOffsets, + commitKind, + latestSnapshot, + conflictCheck, + branchName, + statsFileName); + + if (result.isSuccess()) { + break; + } + + retryResult = (RetryResult) result; + + if (System.currentTimeMillis() - startMillis > commitTimeout + || retryCount >= commitMaxRetries) { + retryResult.cleanAll(); throw new RuntimeException( String.format( - "Commit failed after %s attempts, there maybe exist commit conflicts between multiple jobs.", - commitMaxRetries)); - } - if (tryCommitOnce( - tableFiles, - changelogFiles, - indexFiles, - identifier, - watermark, - logOffsets, - commitKind, - latestSnapshot, - conflictCheck, - branchName, - statsFileName)) { - break; + "Commit failed after %s millis with %s retries, there maybe exist commit conflicts between multiple jobs.", + commitTimeout, retryCount)); } + + retryCount++; } - return cnt; + return retryCount + 1; } private int tryOverwrite( @@ -746,70 +781,58 @@ private int tryOverwrite( long identifier, @Nullable Long watermark, Map logOffsets) { - int cnt = 0; - while (true) { - Snapshot latestSnapshot = snapshotManager.latestSnapshot(); - - cnt++; - if (cnt >= commitMaxRetries) { - throw new RuntimeException( - String.format( - "Commit failed after %s attempts, there maybe exist commit conflicts between multiple jobs.", - commitMaxRetries)); + // collect all files with overwrite + Snapshot latestSnapshot = snapshotManager.latestSnapshot(); + List changesWithOverwrite = new ArrayList<>(); + List indexChangesWithOverwrite = new ArrayList<>(); + if (latestSnapshot != null) { + List currentEntries = + scan.withSnapshot(latestSnapshot) + .withPartitionFilter(partitionFilter) + .withKind(ScanMode.ALL) + .plan() + .files(); + for (ManifestEntry entry : currentEntries) { + changesWithOverwrite.add( + new ManifestEntry( + FileKind.DELETE, + entry.partition(), + entry.bucket(), + entry.totalBuckets(), + entry.file())); } - List changesWithOverwrite = new ArrayList<>(); - List indexChangesWithOverwrite = new ArrayList<>(); - if (latestSnapshot != null) { - List currentEntries = - scan.withSnapshot(latestSnapshot) - .withPartitionFilter(partitionFilter) - .plan() - .files(); - for (ManifestEntry entry : currentEntries) { - changesWithOverwrite.add( - new ManifestEntry( - FileKind.DELETE, - entry.partition(), - entry.bucket(), - entry.totalBuckets(), - entry.file())); - } - // collect index files - if (latestSnapshot.indexManifest() != null) { - List entries = - indexManifestFile.read(latestSnapshot.indexManifest()); - for (IndexManifestEntry entry : entries) { - if (partitionFilter == null || partitionFilter.test(entry.partition())) { - indexChangesWithOverwrite.add(entry.toDeleteEntry()); - } + // collect index files + if (latestSnapshot.indexManifest() != null) { + List entries = + indexManifestFile.read(latestSnapshot.indexManifest()); + for (IndexManifestEntry entry : entries) { + if (partitionFilter == null || partitionFilter.test(entry.partition())) { + indexChangesWithOverwrite.add(entry.toDeleteEntry()); } } } - changesWithOverwrite.addAll(changes); - indexChangesWithOverwrite.addAll(indexFiles); - - if (tryCommitOnce( - changesWithOverwrite, - Collections.emptyList(), - indexChangesWithOverwrite, - identifier, - watermark, - logOffsets, - Snapshot.CommitKind.OVERWRITE, - latestSnapshot, - mustConflictCheck(), - branchName, - null)) { - break; - } } - return cnt; + changesWithOverwrite.addAll(changes); + indexChangesWithOverwrite.addAll(indexFiles); + + return tryCommit( + changesWithOverwrite, + Collections.emptyList(), + indexChangesWithOverwrite, + identifier, + watermark, + logOffsets, + Snapshot.CommitKind.OVERWRITE, + mustConflictCheck(), + branchName, + null); } @VisibleForTesting - boolean tryCommitOnce( - List tableFiles, + CommitResult tryCommitOnce( + @Nullable RetryResult retryResult, + List deltaFiles, List changelogFiles, List indexFiles, long identifier, @@ -830,7 +853,7 @@ boolean tryCommitOnce( if (LOG.isDebugEnabled()) { LOG.debug("Ready to commit table files to snapshot {}", newSnapshotId); - for (ManifestEntry entry : tableFiles) { + for (ManifestEntry entry : deltaFiles) { LOG.debug(" * {}", entry); } LOG.debug("Ready to commit changelog to snapshot {}", newSnapshotId); @@ -839,30 +862,56 @@ boolean tryCommitOnce( } } + List baseDataFiles = new ArrayList<>(); if (latestSnapshot != null && conflictCheck.shouldCheck(latestSnapshot.id())) { // latestSnapshotId is different from the snapshot id we've checked for conflicts, // so we have to check again - noConflictsOrFail(latestSnapshot.commitUser(), latestSnapshot, tableFiles); + try { + List changedPartitions = + deltaFiles.stream() + .map(ManifestEntry::partition) + .distinct() + .collect(Collectors.toList()); + if (retryResult != null && retryResult.latestSnapshot != null) { + baseDataFiles = new ArrayList<>(retryResult.baseDataFiles); + List incremental = + readIncrementalChanges( + retryResult.latestSnapshot, latestSnapshot, changedPartitions); + if (!incremental.isEmpty()) { + baseDataFiles.addAll(incremental); + baseDataFiles = new ArrayList<>(FileEntry.mergeEntries(baseDataFiles)); + } + } else { + baseDataFiles = + readAllEntriesFromChangedPartitions(latestSnapshot, changedPartitions); + } + noConflictsOrFail( + latestSnapshot.commitUser(), + baseDataFiles, + SimpleFileEntry.from(deltaFiles)); + } catch (Exception e) { + if (retryResult != null) { + retryResult.cleanAll(); + } + throw e; + } } Snapshot newSnapshot; - String previousChangesListName = null; - String newChangesListName = null; - String changelogListName = null; - String newIndexManifest = null; - List oldMetas = new ArrayList<>(); - List newMetas = new ArrayList<>(); - List changelogMetas = new ArrayList<>(); + String baseManifestList = null; + String deltaManifestList = null; + String changelogManifestList = null; + String oldIndexManifest = null; + String indexManifest = null; + List mergeBeforeManifests = new ArrayList<>(); + List mergeAfterManifests = new ArrayList<>(); try { long previousTotalRecordCount = 0L; Long currentWatermark = watermark; - String previousIndexManifest = null; if (latestSnapshot != null) { previousTotalRecordCount = scan.totalRecordCount(latestSnapshot); - List previousManifests = - manifestList.readDataManifests(latestSnapshot); // read all previous manifest files - oldMetas.addAll(previousManifests); + mergeBeforeManifests = manifestList.readDataManifests(latestSnapshot); // read the last snapshot to complete the bucket's offsets when logOffsets does not // contain all buckets Map latestLogOffsets = latestSnapshot.logOffsets(); @@ -876,41 +925,49 @@ boolean tryCommitOnce( ? latestWatermark : Math.max(currentWatermark, latestWatermark); } - previousIndexManifest = latestSnapshot.indexManifest(); + oldIndexManifest = latestSnapshot.indexManifest(); } - // merge manifest files with changes - newMetas.addAll( + + // try to merge old manifest files to create base manifest list + mergeAfterManifests = ManifestFileMerger.merge( - oldMetas, + mergeBeforeManifests, manifestFile, manifestTargetSize.getBytes(), manifestMergeMinCount, manifestFullCompactionSize.getBytes(), partitionType, - manifestReadParallelism)); - previousChangesListName = manifestList.write(newMetas); + manifestReadParallelism); + baseManifestList = manifestList.write(mergeAfterManifests); // the added records subtract the deleted records from - long deltaRecordCount = recordCountAdd(tableFiles) - recordCountDelete(tableFiles); + long deltaRecordCount = recordCountAdd(deltaFiles) - recordCountDelete(deltaFiles); long totalRecordCount = previousTotalRecordCount + deltaRecordCount; - // write new changes into manifest files - List newChangesManifests = manifestFile.write(tableFiles); - newMetas.addAll(newChangesManifests); - newChangesListName = manifestList.write(newChangesManifests); + boolean rewriteIndexManifest = true; + if (retryResult != null) { + deltaManifestList = retryResult.deltaManifestList; + changelogManifestList = retryResult.changelogManifestList; + if (Objects.equals(oldIndexManifest, retryResult.oldIndexManifest)) { + rewriteIndexManifest = false; + indexManifest = retryResult.newIndexManifest; + LOG.info("Reusing index manifest {} for retry.", indexManifest); + } else { + cleanIndexManifest(retryResult.oldIndexManifest, retryResult.newIndexManifest); + } + } else { + // write new delta files into manifest files + deltaManifestList = manifestList.write(manifestFile.write(deltaFiles)); - // write changelog into manifest files - if (!changelogFiles.isEmpty()) { - changelogMetas.addAll(manifestFile.write(changelogFiles)); - changelogListName = manifestList.write(changelogMetas); + // write changelog into manifest files + if (!changelogFiles.isEmpty()) { + changelogManifestList = manifestList.write(manifestFile.write(changelogFiles)); + } } - // write new index manifest - String indexManifest = - indexManifestFile.writeIndexFiles( - previousIndexManifest, indexFiles, bucketMode); - if (!Objects.equals(indexManifest, previousIndexManifest)) { - newIndexManifest = indexManifest; + if (rewriteIndexManifest) { + indexManifest = + indexManifestFile.writeIndexFiles(oldIndexManifest, indexFiles, bucketMode); } long latestSchemaId = schemaManager.latest().get().id(); @@ -935,9 +992,9 @@ boolean tryCommitOnce( new Snapshot( newSnapshotId, latestSchemaId, - previousChangesListName, - newChangesListName, - changelogListName, + baseManifestList, + deltaManifestList, + changelogManifestList, indexManifest, commitUser, identifier, @@ -951,14 +1008,12 @@ boolean tryCommitOnce( statsFileName); } catch (Throwable e) { // fails when preparing for commit, we should clean up - cleanUpTmpManifests( - previousChangesListName, - newChangesListName, - changelogListName, - newIndexManifest, - oldMetas, - newMetas, - changelogMetas); + if (retryResult != null) { + retryResult.cleanAll(); + } + cleanUpReuseTmpManifests( + deltaManifestList, changelogManifestList, oldIndexManifest, indexManifest); + cleanUpNoReuseTmpManifests(baseManifestList, mergeBeforeManifests, mergeAfterManifests); throw new RuntimeException( String.format( "Exception occurs when preparing snapshot #%d (path %s) by user %s " @@ -971,47 +1026,7 @@ boolean tryCommitOnce( e); } - boolean success; - try { - Callable callable = - () -> { - boolean committed = - fileIO.tryToWriteAtomic(newSnapshotPath, newSnapshot.toJson()); - if (committed) { - snapshotManager.commitLatestHint(newSnapshotId); - } - return committed; - }; - if (lock != null) { - success = - lock.runWithLock( - () -> - // fs.rename may not returns false if target file - // already exists, or even not atomic - // as we're relying on external locking, we can first - // check if file exist then rename to work around this - // case - !fileIO.exists(newSnapshotPath) && callable.call()); - } else { - success = callable.call(); - } - } catch (Throwable e) { - // exception when performing the atomic rename, - // we cannot clean up because we can't determine the success - throw new RuntimeException( - String.format( - "Exception occurs when committing snapshot #%d (path %s) by user %s " - + "with identifier %s and kind %s. " - + "Cannot clean up because we can't determine the success.", - newSnapshotId, - newSnapshotPath, - commitUser, - identifier, - commitKind.name()), - e); - } - - if (success) { + if (commitSnapshotImpl(newSnapshot, newSnapshotPath)) { if (LOG.isDebugEnabled()) { LOG.debug( String.format( @@ -1023,8 +1038,8 @@ boolean tryCommitOnce( identifier, commitKind.name())); } - commitCallbacks.forEach(callback -> callback.call(tableFiles, newSnapshot)); - return true; + commitCallbacks.forEach(callback -> callback.call(deltaFiles, newSnapshot)); + return new SuccessResult(); } // atomic rename fails, clean up and try again @@ -1040,15 +1055,179 @@ boolean tryCommitOnce( identifier, commitKind.name(), commitTime)); - cleanUpTmpManifests( - previousChangesListName, - newChangesListName, - changelogListName, - newIndexManifest, - oldMetas, - newMetas, - changelogMetas); - return false; + cleanUpNoReuseTmpManifests(baseManifestList, mergeBeforeManifests, mergeAfterManifests); + return new RetryResult( + deltaManifestList, + changelogManifestList, + oldIndexManifest, + indexManifest, + latestSnapshot, + baseDataFiles); + } + + public void compactManifest() { + int retryCount = 0; + ManifestCompactResult retryResult = null; + long startMillis = System.currentTimeMillis(); + while (true) { + retryResult = compactManifest(retryResult); + if (retryResult.isSuccess()) { + break; + } + + if (System.currentTimeMillis() - startMillis > commitTimeout + || retryCount >= commitMaxRetries) { + retryResult.cleanAll(); + throw new RuntimeException( + String.format( + "Commit failed after %s millis with %s retries, there maybe exist commit conflicts between multiple jobs.", + commitTimeout, retryCount)); + } + + retryCount++; + } + } + + private ManifestCompactResult compactManifest(@Nullable ManifestCompactResult lastResult) { + Snapshot latestSnapshot = snapshotManager.latestSnapshot(); + + if (latestSnapshot == null) { + return new SuccessManifestCompactResult(); + } + + List mergeBeforeManifests = + manifestList.readDataManifests(latestSnapshot); + List mergeAfterManifests; + + if (lastResult != null) { + List oldMergeBeforeManifests = lastResult.mergeBeforeManifests; + List oldMergeAfterManifests = lastResult.mergeAfterManifests; + + Set retryMergeBefore = + oldMergeBeforeManifests.stream() + .map(ManifestFileMeta::fileName) + .collect(Collectors.toSet()); + + List manifestsFromOther = + mergeBeforeManifests.stream() + .filter(m -> !retryMergeBefore.remove(m.fileName())) + .collect(Collectors.toList()); + + if (retryMergeBefore.isEmpty()) { + // no manifest compact from latest failed commit to latest commit + mergeAfterManifests = new ArrayList<>(oldMergeAfterManifests); + mergeAfterManifests.addAll(manifestsFromOther); + } else { + // manifest compact happens, quit + lastResult.cleanAll(); + return new SuccessManifestCompactResult(); + } + } else { + // the fist trial + mergeAfterManifests = + ManifestFileMerger.merge( + mergeBeforeManifests, + manifestFile, + manifestTargetSize.getBytes(), + 1, + 1, + partitionType, + manifestReadParallelism); + + if (new HashSet<>(mergeBeforeManifests).equals(new HashSet<>(mergeAfterManifests))) { + // no need to commit this snapshot, because no compact were happened + return new SuccessManifestCompactResult(); + } + } + + String baseManifestList = manifestList.write(mergeAfterManifests); + String deltaManifestList = manifestList.write(Collections.emptyList()); + + // prepare snapshot file + Snapshot newSnapshot = + new Snapshot( + latestSnapshot.id() + 1, + latestSnapshot.schemaId(), + baseManifestList, + deltaManifestList, + null, + latestSnapshot.indexManifest(), + commitUser, + Long.MAX_VALUE, + Snapshot.CommitKind.COMPACT, + System.currentTimeMillis(), + latestSnapshot.logOffsets(), + latestSnapshot.totalRecordCount(), + 0L, + 0L, + latestSnapshot.watermark(), + latestSnapshot.statistics()); + + Path newSnapshotPath = + branchName.equals(DEFAULT_MAIN_BRANCH) + ? snapshotManager.snapshotPath(newSnapshot.id()) + : snapshotManager.copyWithBranch(branchName).snapshotPath(newSnapshot.id()); + + if (!commitSnapshotImpl(newSnapshot, newSnapshotPath)) { + return new ManifestCompactResult( + baseManifestList, deltaManifestList, mergeBeforeManifests, mergeAfterManifests); + } else { + return new SuccessManifestCompactResult(); + } + } + + private boolean commitSnapshotImpl(Snapshot newSnapshot, Path newSnapshotPath) { + try { + Callable callable = + () -> { + boolean committed = + fileIO.tryToWriteAtomic(newSnapshotPath, newSnapshot.toJson()); + if (committed) { + snapshotManager.commitLatestHint(newSnapshot.id()); + } + return committed; + }; + if (lock != null) { + return lock.runWithLock( + () -> + // fs.rename may not returns false if target file + // already exists, or even not atomic + // as we're relying on external locking, we can first + // check if file exist then rename to work around this + // case + !fileIO.exists(newSnapshotPath) && callable.call()); + } else { + return callable.call(); + } + } catch (Throwable e) { + // exception when performing the atomic rename, + // we cannot clean up because we can't determine the success + throw new RuntimeException( + String.format( + "Exception occurs when committing snapshot #%d (path %s) by user %s " + + "with identifier %s and kind %s. " + + "Cannot clean up because we can't determine the success.", + newSnapshot.id(), + newSnapshotPath, + commitUser, + newSnapshot.commitIdentifier(), + newSnapshot.commitKind().name()), + e); + } + } + + private List readIncrementalChanges( + Snapshot from, Snapshot to, List changedPartitions) { + List entries = new ArrayList<>(); + for (long i = from.id() + 1; i <= to.id(); i++) { + List delta = + scan.withSnapshot(i) + .withKind(ScanMode.DELTA) + .withPartitionFilter(changedPartitions) + .readSimpleEntries(); + entries.addAll(delta); + } + return entries; } @SafeVarargs @@ -1060,8 +1239,14 @@ private final List readAllEntriesFromChangedPartitions( .map(ManifestEntry::partition) .distinct() .collect(Collectors.toList()); + return readAllEntriesFromChangedPartitions(snapshot, changedPartitions); + } + + private List readAllEntriesFromChangedPartitions( + Snapshot snapshot, List changedPartitions) { try { return scan.withSnapshot(snapshot) + .withKind(ScanMode.ALL) .withPartitionFilter(changedPartitions) .readSimpleEntries(); } catch (Throwable e) { @@ -1069,14 +1254,6 @@ private final List readAllEntriesFromChangedPartitions( } } - private void noConflictsOrFail( - String baseCommitUser, Snapshot latestSnapshot, List changes) { - noConflictsOrFail( - baseCommitUser, - readAllEntriesFromChangedPartitions(latestSnapshot, changes), - SimpleFileEntry.from(changes)); - } - private void noConflictsOrFail( String baseCommitUser, List baseEntries, @@ -1158,8 +1335,9 @@ private void assertNoDelete( for (SimpleFileEntry entry : mergedEntries) { Preconditions.checkState( entry.kind() != FileKind.DELETE, - "Trying to delete file %s which is not previously added.", - entry.fileName()); + "Trying to delete file %s for table %s which is not previously added.", + entry.fileName(), + tableName); } } catch (Throwable e) { if (partitionExpire != null && partitionExpire.isValueExpiration()) { @@ -1276,37 +1454,49 @@ private Pair createConflictException( } } - private void cleanUpTmpManifests( - String previousChangesListName, - String newChangesListName, - String changelogListName, - String newIndexManifest, - List oldMetas, - List newMetas, - List changelogMetas) { - // clean up newly created manifest list - if (previousChangesListName != null) { - manifestList.delete(previousChangesListName); - } - if (newChangesListName != null) { - manifestList.delete(newChangesListName); + private void cleanUpNoReuseTmpManifests( + String baseManifestList, + List mergeBeforeManifests, + List mergeAfterManifests) { + if (baseManifestList != null) { + manifestList.delete(baseManifestList); } - if (changelogListName != null) { - manifestList.delete(changelogListName); + Set oldMetaSet = + mergeBeforeManifests.stream() + .map(ManifestFileMeta::fileName) + .collect(Collectors.toSet()); + for (ManifestFileMeta suspect : mergeAfterManifests) { + if (!oldMetaSet.contains(suspect.fileName())) { + manifestFile.delete(suspect.fileName()); + } } - if (newIndexManifest != null) { - indexManifestFile.delete(newIndexManifest); + } + + private void cleanUpReuseTmpManifests( + String deltaManifestList, + String changelogManifestList, + String oldIndexManifest, + String newIndexManifest) { + if (deltaManifestList != null) { + for (ManifestFileMeta manifest : manifestList.read(deltaManifestList)) { + manifestFile.delete(manifest.fileName()); + } + manifestList.delete(deltaManifestList); } - // clean up newly merged manifest files - Set oldMetaSet = new HashSet<>(oldMetas); // for faster searching - for (ManifestFileMeta suspect : newMetas) { - if (!oldMetaSet.contains(suspect)) { - manifestList.delete(suspect.fileName()); + + if (changelogManifestList != null) { + for (ManifestFileMeta manifest : manifestList.read(changelogManifestList)) { + manifestFile.delete(manifest.fileName()); } + manifestList.delete(changelogManifestList); } - // clean up changelog manifests - for (ManifestFileMeta meta : changelogMetas) { - manifestList.delete(meta.fileName()); + + cleanIndexManifest(oldIndexManifest, newIndexManifest); + } + + private void cleanIndexManifest(String oldIndexManifest, String newIndexManifest) { + if (newIndexManifest != null && !Objects.equals(oldIndexManifest, newIndexManifest)) { + indexManifestFile.delete(newIndexManifest); } } @@ -1363,4 +1553,97 @@ static ConflictCheck noConflictCheck() { static ConflictCheck mustConflictCheck() { return latestSnapshot -> true; } + + private interface CommitResult { + boolean isSuccess(); + } + + private static class SuccessResult implements CommitResult { + + @Override + public boolean isSuccess() { + return true; + } + } + + private class RetryResult implements CommitResult { + + private final String deltaManifestList; + private final String changelogManifestList; + + private final String oldIndexManifest; + private final String newIndexManifest; + + private final Snapshot latestSnapshot; + private final List baseDataFiles; + + private RetryResult( + String deltaManifestList, + String changelogManifestList, + String oldIndexManifest, + String newIndexManifest, + Snapshot latestSnapshot, + List baseDataFiles) { + this.deltaManifestList = deltaManifestList; + this.changelogManifestList = changelogManifestList; + this.oldIndexManifest = oldIndexManifest; + this.newIndexManifest = newIndexManifest; + this.latestSnapshot = latestSnapshot; + this.baseDataFiles = baseDataFiles; + } + + private void cleanAll() { + cleanUpReuseTmpManifests( + deltaManifestList, changelogManifestList, oldIndexManifest, newIndexManifest); + } + + @Override + public boolean isSuccess() { + return false; + } + } + + private class ManifestCompactResult implements CommitResult { + + private final String baseManifestList; + private final String deltaManifestList; + private final List mergeBeforeManifests; + private final List mergeAfterManifests; + + public ManifestCompactResult( + String baseManifestList, + String deltaManifestList, + List mergeBeforeManifests, + List mergeAfterManifests) { + this.baseManifestList = baseManifestList; + this.deltaManifestList = deltaManifestList; + this.mergeBeforeManifests = mergeBeforeManifests; + this.mergeAfterManifests = mergeAfterManifests; + } + + public void cleanAll() { + manifestList.delete(deltaManifestList); + cleanUpNoReuseTmpManifests(baseManifestList, mergeBeforeManifests, mergeAfterManifests); + } + + @Override + public boolean isSuccess() { + return false; + } + } + + private class SuccessManifestCompactResult extends ManifestCompactResult { + + public SuccessManifestCompactResult() { + super(null, null, null, null); + } + + @Override + public void cleanAll() {} + + @Override + public boolean isSuccess() { + return true; + } + } } diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/FileStoreScan.java b/paimon-core/src/main/java/org/apache/paimon/operation/FileStoreScan.java index bc0d7ff27301..179d16de6cd2 100644 --- a/paimon-core/src/main/java/org/apache/paimon/operation/FileStoreScan.java +++ b/paimon-core/src/main/java/org/apache/paimon/operation/FileStoreScan.java @@ -67,12 +67,12 @@ public interface FileStoreScan { FileStoreScan withSnapshot(Snapshot snapshot); - FileStoreScan withManifestList(List manifests); - FileStoreScan withKind(ScanMode scanMode); FileStoreScan withLevelFilter(Filter levelFilter); + FileStoreScan enableValueFilter(); + FileStoreScan withManifestEntryFilter(Filter filter); FileStoreScan withManifestCacheFilter(ManifestCacheFilter manifestFilter); @@ -81,6 +81,8 @@ public interface FileStoreScan { FileStoreScan withMetrics(ScanMetrics metrics); + FileStoreScan dropStats(); + @Nullable Integer parallelism(); diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/KeyValueFileStoreScan.java b/paimon-core/src/main/java/org/apache/paimon/operation/KeyValueFileStoreScan.java index 3311161b54a5..8d8c51996cfe 100644 --- a/paimon-core/src/main/java/org/apache/paimon/operation/KeyValueFileStoreScan.java +++ b/paimon-core/src/main/java/org/apache/paimon/operation/KeyValueFileStoreScan.java @@ -21,31 +21,38 @@ import org.apache.paimon.CoreOptions.ChangelogProducer; import org.apache.paimon.CoreOptions.MergeEngine; import org.apache.paimon.KeyValueFileStore; +import org.apache.paimon.fileindex.FileIndexPredicate; +import org.apache.paimon.io.DataFileMeta; +import org.apache.paimon.manifest.FilteredManifestEntry; import org.apache.paimon.manifest.ManifestEntry; import org.apache.paimon.manifest.ManifestFile; import org.apache.paimon.predicate.Predicate; import org.apache.paimon.schema.KeyValueFieldsExtractor; import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.schema.TableSchema; -import org.apache.paimon.stats.SimpleStats; -import org.apache.paimon.stats.SimpleStatsConverter; -import org.apache.paimon.stats.SimpleStatsConverters; +import org.apache.paimon.stats.SimpleStatsEvolution; +import org.apache.paimon.stats.SimpleStatsEvolutions; import org.apache.paimon.table.source.ScanMode; +import org.apache.paimon.types.RowType; import org.apache.paimon.utils.SnapshotManager; +import javax.annotation.Nullable; + +import java.io.IOException; import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; import static org.apache.paimon.CoreOptions.MergeEngine.AGGREGATE; -import static org.apache.paimon.CoreOptions.MergeEngine.FIRST_ROW; import static org.apache.paimon.CoreOptions.MergeEngine.PARTIAL_UPDATE; /** {@link FileStoreScan} for {@link KeyValueFileStore}. */ public class KeyValueFileStoreScan extends AbstractFileStoreScan { - private final SimpleStatsConverters fieldKeyStatsConverters; - private final SimpleStatsConverters fieldValueStatsConverters; + private final SimpleStatsEvolutions fieldKeyStatsConverters; + private final SimpleStatsEvolutions fieldValueStatsConverters; private final BucketSelectConverter bucketSelectConverter; private Predicate keyFilter; @@ -54,6 +61,11 @@ public class KeyValueFileStoreScan extends AbstractFileStoreScan { private final MergeEngine mergeEngine; private final ChangelogProducer changelogProducer; + private final boolean fileIndexReadEnabled; + private final Map schemaId2DataFilter = new HashMap<>(); + + private boolean valueFilterForceEnabled = false; + public KeyValueFileStoreScan( ManifestsReader manifestsReader, BucketSelectConverter bucketSelectConverter, @@ -65,7 +77,8 @@ public KeyValueFileStoreScan( Integer scanManifestParallelism, boolean deletionVectorsEnabled, MergeEngine mergeEngine, - ChangelogProducer changelogProducer) { + ChangelogProducer changelogProducer, + boolean fileIndexReadEnabled) { super( manifestsReader, snapshotManager, @@ -75,16 +88,17 @@ public KeyValueFileStoreScan( scanManifestParallelism); this.bucketSelectConverter = bucketSelectConverter; this.fieldKeyStatsConverters = - new SimpleStatsConverters( + new SimpleStatsEvolutions( sid -> keyValueFieldsExtractor.keyFields(scanTableSchema(sid)), schema.id()); this.fieldValueStatsConverters = - new SimpleStatsConverters( + new SimpleStatsEvolutions( sid -> keyValueFieldsExtractor.valueFields(scanTableSchema(sid)), schema.id()); this.deletionVectorsEnabled = deletionVectorsEnabled; this.mergeEngine = mergeEngine; this.changelogProducer = changelogProducer; + this.fileIndexReadEnabled = fileIndexReadEnabled; } public KeyValueFileStoreScan withKeyFilter(Predicate predicate) { @@ -98,43 +112,68 @@ public KeyValueFileStoreScan withValueFilter(Predicate predicate) { return this; } + @Override + public FileStoreScan enableValueFilter() { + this.valueFilterForceEnabled = true; + return this; + } + /** Note: Keep this thread-safe. */ @Override protected boolean filterByStats(ManifestEntry entry) { - Predicate filter = null; - SimpleStatsConverter serializer = null; - SimpleStats stats = null; - if (isValueFilterEnabled(entry)) { - filter = valueFilter; - serializer = fieldValueStatsConverters.getOrCreate(entry.file().schemaId()); - stats = entry.file().valueStats(); + DataFileMeta file = entry.file(); + if (isValueFilterEnabled() && !filterByValueFilter(entry)) { + return false; } - if (filter == null && keyFilter != null) { - filter = keyFilter; - serializer = fieldKeyStatsConverters.getOrCreate(entry.file().schemaId()); - stats = entry.file().keyStats(); + if (keyFilter != null) { + SimpleStatsEvolution.Result stats = + fieldKeyStatsConverters + .getOrCreate(file.schemaId()) + .evolution(file.keyStats(), file.rowCount(), null); + return keyFilter.test( + file.rowCount(), stats.minValues(), stats.maxValues(), stats.nullCounts()); } - if (filter == null) { + return true; + } + + @Override + protected ManifestEntry dropStats(ManifestEntry entry) { + if (!isValueFilterEnabled() && wholeBucketFilterEnabled()) { + return new FilteredManifestEntry(entry.copyWithoutStats(), filterByValueFilter(entry)); + } + return entry.copyWithoutStats(); + } + + private boolean filterByFileIndex(@Nullable byte[] embeddedIndexBytes, ManifestEntry entry) { + if (embeddedIndexBytes == null) { return true; } - return filter.test( - entry.file().rowCount(), - serializer.evolution(stats.minValues()), - serializer.evolution(stats.maxValues()), - serializer.evolution(stats.nullCounts(), entry.file().rowCount())); + RowType dataRowType = scanTableSchema(entry.file().schemaId()).logicalRowType(); + try (FileIndexPredicate predicate = + new FileIndexPredicate(embeddedIndexBytes, dataRowType)) { + Predicate dataPredicate = + schemaId2DataFilter.computeIfAbsent( + entry.file().schemaId(), + id -> + fieldValueStatsConverters.convertFilter( + entry.file().schemaId(), valueFilter)); + return predicate.evaluate(dataPredicate).remain(); + } catch (IOException e) { + throw new RuntimeException("Exception happens while checking fileIndex predicate.", e); + } } - private boolean isValueFilterEnabled(ManifestEntry entry) { + private boolean isValueFilterEnabled() { if (valueFilter == null) { return false; } switch (scanMode) { case ALL: - return (deletionVectorsEnabled || mergeEngine == FIRST_ROW) && entry.level() > 0; + return valueFilterForceEnabled; case DELTA: return false; case CHANGELOG: @@ -145,13 +184,13 @@ private boolean isValueFilterEnabled(ManifestEntry entry) { } } - /** Note: Keep this thread-safe. */ @Override - protected List filterWholeBucketByStats(List entries) { - if (valueFilter == null || scanMode != ScanMode.ALL) { - return entries; - } + protected boolean wholeBucketFilterEnabled() { + return valueFilter != null && scanMode == ScanMode.ALL; + } + @Override + protected List filterWholeBucketByStats(List entries) { return noOverlapping(entries) ? filterWholeBucketPerFile(entries) : filterWholeBucketAllFiles(entries); @@ -184,14 +223,22 @@ private List filterWholeBucketAllFiles(List entrie } private boolean filterByValueFilter(ManifestEntry entry) { - SimpleStatsConverter serializer = - fieldValueStatsConverters.getOrCreate(entry.file().schemaId()); - SimpleStats stats = entry.file().valueStats(); + if (entry instanceof FilteredManifestEntry) { + return ((FilteredManifestEntry) entry).selected(); + } + + DataFileMeta file = entry.file(); + SimpleStatsEvolution.Result result = + fieldValueStatsConverters + .getOrCreate(file.schemaId()) + .evolution(file.valueStats(), file.rowCount(), file.valueStatsCols()); return valueFilter.test( - entry.file().rowCount(), - serializer.evolution(stats.minValues()), - serializer.evolution(stats.maxValues()), - serializer.evolution(stats.nullCounts(), entry.file().rowCount())); + file.rowCount(), + result.minValues(), + result.maxValues(), + result.nullCounts()) + && (!fileIndexReadEnabled + || filterByFileIndex(entry.file().embeddedIndex(), entry)); } private static boolean noOverlapping(List entries) { diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/LocalOrphanFilesClean.java b/paimon-core/src/main/java/org/apache/paimon/operation/LocalOrphanFilesClean.java index a17434faaa8f..6a4276662468 100644 --- a/paimon-core/src/main/java/org/apache/paimon/operation/LocalOrphanFilesClean.java +++ b/paimon-core/src/main/java/org/apache/paimon/operation/LocalOrphanFilesClean.java @@ -21,12 +21,12 @@ import org.apache.paimon.CoreOptions; import org.apache.paimon.catalog.Catalog; import org.apache.paimon.catalog.Identifier; -import org.apache.paimon.fs.FileStatus; import org.apache.paimon.fs.Path; import org.apache.paimon.manifest.ManifestEntry; import org.apache.paimon.manifest.ManifestFile; import org.apache.paimon.table.FileStoreTable; import org.apache.paimon.table.Table; +import org.apache.paimon.utils.Pair; import org.apache.paimon.utils.SerializableConsumer; import javax.annotation.Nullable; @@ -40,18 +40,22 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Consumer; import java.util.function.Function; import java.util.stream.Collectors; import static org.apache.paimon.utils.Preconditions.checkArgument; import static org.apache.paimon.utils.ThreadPoolUtils.createCachedThreadPool; -import static org.apache.paimon.utils.ThreadPoolUtils.randomlyExecute; +import static org.apache.paimon.utils.ThreadPoolUtils.randomlyExecuteSequentialReturn; +import static org.apache.paimon.utils.ThreadPoolUtils.randomlyOnlyExecute; /** * Local {@link OrphanFilesClean}, it will use thread pool to execute deletion. @@ -65,6 +69,10 @@ public class LocalOrphanFilesClean extends OrphanFilesClean { private final List deleteFiles; + private final AtomicLong deletedFilesLenInBytes = new AtomicLong(0); + + private Set candidateDeletes; + public LocalOrphanFilesClean(FileStoreTable table) { this(table, System.currentTimeMillis() - TimeUnit.DAYS.toMillis(1)); } @@ -82,17 +90,20 @@ public LocalOrphanFilesClean( table.coreOptions().deleteFileThreadNum(), "ORPHAN_FILES_CLEAN"); } - public List clean() throws IOException, ExecutionException, InterruptedException { + public CleanOrphanFilesResult clean() + throws IOException, ExecutionException, InterruptedException { List branches = validBranches(); // specially handle to clear snapshot dir - cleanSnapshotDir(branches, deleteFiles::add); + cleanSnapshotDir(branches, deleteFiles::add, deletedFilesLenInBytes::addAndGet); // delete candidate files - Map candidates = getCandidateDeletingFiles(); + Map> candidates = getCandidateDeletingFiles(); if (candidates.isEmpty()) { - return deleteFiles; + return new CleanOrphanFilesResult( + deleteFiles.size(), deletedFilesLenInBytes.get(), deleteFiles); } + candidateDeletes = new HashSet<>(candidates.keySet()); // find used files Set usedFiles = @@ -101,22 +112,71 @@ public List clean() throws IOException, ExecutionException, InterruptedExc .collect(Collectors.toSet()); // delete unused files - Set deleted = new HashSet<>(candidates.keySet()); - deleted.removeAll(usedFiles); - deleted.stream().map(candidates::get).forEach(fileCleaner); - deleteFiles.addAll(deleted.stream().map(candidates::get).collect(Collectors.toList())); + candidateDeletes.removeAll(usedFiles); + candidateDeletes.stream() + .map(candidates::get) + .forEach( + deleteFileInfo -> { + deletedFilesLenInBytes.addAndGet(deleteFileInfo.getRight()); + fileCleaner.accept(deleteFileInfo.getLeft()); + }); + deleteFiles.addAll( + candidateDeletes.stream() + .map(candidates::get) + .map(Pair::getLeft) + .collect(Collectors.toList())); + candidateDeletes.clear(); + + return new CleanOrphanFilesResult( + deleteFiles.size(), deletedFilesLenInBytes.get(), deleteFiles); + } - return deleteFiles; + private void collectWithoutDataFile( + String branch, Consumer usedFileConsumer, Consumer manifestConsumer) + throws IOException { + randomlyOnlyExecute( + executor, + snapshot -> { + try { + collectWithoutDataFile( + branch, snapshot, usedFileConsumer, manifestConsumer); + } catch (IOException e) { + throw new RuntimeException(e); + } + }, + safelyGetAllSnapshots(branch)); } - private List getUsedFiles(String branch) { - List usedFiles = new ArrayList<>(); + private Set getUsedFiles(String branch) { + Set usedFiles = ConcurrentHashMap.newKeySet(); ManifestFile manifestFile = table.switchToBranch(branch).store().manifestFileFactory().create(); try { - List manifests = new ArrayList<>(); + Set manifests = ConcurrentHashMap.newKeySet(); collectWithoutDataFile(branch, usedFiles::add, manifests::add); - usedFiles.addAll(retryReadingDataFiles(manifestFile, manifests)); + randomlyOnlyExecute( + executor, + manifestName -> { + try { + retryReadingFiles( + () -> manifestFile.readWithIOException(manifestName), + Collections.emptyList()) + .stream() + .map(ManifestEntry::file) + .forEach( + f -> { + if (candidateDeletes.contains(f.fileName())) { + usedFiles.add(f.fileName()); + } + f.extraFiles().stream() + .filter(candidateDeletes::contains) + .forEach(usedFiles::add); + }); + } catch (IOException e) { + throw new RuntimeException(e); + } + }, + manifests); } catch (IOException e) { throw new RuntimeException(e); } @@ -127,52 +187,20 @@ private List getUsedFiles(String branch) { * Get all the candidate deleting files in the specified directories and filter them by * olderThanMillis. */ - private Map getCandidateDeletingFiles() { + private Map> getCandidateDeletingFiles() { List fileDirs = listPaimonFileDirs(); - Function> processor = + Function>> processor = path -> tryBestListingDirs(path).stream() .filter(this::oldEnough) - .map(FileStatus::getPath) + .map(status -> Pair.of(status.getPath(), status.getLen())) .collect(Collectors.toList()); - Iterator allPaths = randomlyExecute(executor, processor, fileDirs); - Map result = new HashMap<>(); - while (allPaths.hasNext()) { - Path next = allPaths.next(); - result.put(next.getName(), next); - } - return result; - } - - private List retryReadingDataFiles( - ManifestFile manifestFile, List manifestNames) throws IOException { - List dataFiles = new ArrayList<>(); - for (String manifestName : manifestNames) { - retryReadingFiles( - () -> manifestFile.readWithIOException(manifestName), - Collections.emptyList()) - .stream() - .map(ManifestEntry::file) - .forEach( - f -> { - dataFiles.add(f.fileName()); - dataFiles.addAll(f.extraFiles()); - }); - } - return dataFiles; - } - - public static List showDeletedFiles(List deleteFiles, int showLimit) { - int showSize = Math.min(deleteFiles.size(), showLimit); - List result = new ArrayList<>(); - if (deleteFiles.size() > showSize) { - result.add( - String.format( - "Total %s files, only %s lines are displayed.", - deleteFiles.size(), showSize)); - } - for (int i = 0; i < showSize; i++) { - result.add(deleteFiles.get(i).toUri().getPath()); + Iterator> allFilesInfo = + randomlyExecuteSequentialReturn(executor, processor, fileDirs); + Map> result = new HashMap<>(); + while (allFilesInfo.hasNext()) { + Pair fileInfo = allFilesInfo.next(); + result.put(fileInfo.getLeft().getName(), fileInfo); } return result; } @@ -185,7 +213,6 @@ public static List createOrphanFilesCleans( SerializableConsumer fileCleaner, @Nullable Integer parallelism) throws Catalog.DatabaseNotExistException, Catalog.TableNotExistException { - List orphanFilesCleans = new ArrayList<>(); List tableNames = Collections.singletonList(tableName); if (tableName == null || "*".equals(tableName)) { tableNames = catalog.listTables(databaseName); @@ -202,6 +229,7 @@ public static List createOrphanFilesCleans( } }; + List orphanFilesCleans = new ArrayList<>(tableNames.size()); for (String t : tableNames) { Identifier identifier = new Identifier(databaseName, t); Table table = catalog.getTable(identifier).copy(dynamicOptions); @@ -218,7 +246,7 @@ public static List createOrphanFilesCleans( return orphanFilesCleans; } - public static long executeDatabaseOrphanFiles( + public static CleanOrphanFilesResult executeDatabaseOrphanFiles( Catalog catalog, String databaseName, @Nullable String tableName, @@ -237,15 +265,17 @@ public static long executeDatabaseOrphanFiles( ExecutorService executorService = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors()); - List>> tasks = new ArrayList<>(); + List> tasks = new ArrayList<>(tableCleans.size()); for (LocalOrphanFilesClean clean : tableCleans) { tasks.add(executorService.submit(clean::clean)); } - List cleanOrphanFiles = new ArrayList<>(); - for (Future> task : tasks) { + long deletedFileCount = 0; + long deletedFileTotalLenInBytes = 0; + for (Future task : tasks) { try { - cleanOrphanFiles.addAll(task.get()); + deletedFileCount += task.get().getDeletedFileCount(); + deletedFileTotalLenInBytes += task.get().getDeletedFileTotalLenInBytes(); } catch (InterruptedException e) { Thread.currentThread().interrupt(); throw new RuntimeException(e); @@ -255,6 +285,6 @@ public static long executeDatabaseOrphanFiles( } executorService.shutdownNow(); - return cleanOrphanFiles.size(); + return new CleanOrphanFilesResult(deletedFileCount, deletedFileTotalLenInBytes); } } diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/MemoryFileStoreWrite.java b/paimon-core/src/main/java/org/apache/paimon/operation/MemoryFileStoreWrite.java index b7feeead4bbb..a2733121eece 100644 --- a/paimon-core/src/main/java/org/apache/paimon/operation/MemoryFileStoreWrite.java +++ b/paimon-core/src/main/java/org/apache/paimon/operation/MemoryFileStoreWrite.java @@ -73,11 +73,15 @@ public MemoryFileStoreWrite( indexFactory, dvMaintainerFactory, tableName, + options, options.bucket(), partitionType, - options.writeMaxWritersToSpill()); + options.writeMaxWritersToSpill(), + options.legacyPartitionName()); this.options = options; - this.cacheManager = new CacheManager(options.lookupCacheMaxMemory()); + this.cacheManager = + new CacheManager( + options.lookupCacheMaxMemory(), options.lookupCacheHighPrioPoolRatio()); } @Override diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/MergeFileSplitRead.java b/paimon-core/src/main/java/org/apache/paimon/operation/MergeFileSplitRead.java index d1f332a0384a..23a3a576e4a6 100644 --- a/paimon-core/src/main/java/org/apache/paimon/operation/MergeFileSplitRead.java +++ b/paimon-core/src/main/java/org/apache/paimon/operation/MergeFileSplitRead.java @@ -44,6 +44,7 @@ import org.apache.paimon.schema.TableSchema; import org.apache.paimon.table.source.DataSplit; import org.apache.paimon.table.source.DeletionFile; +import org.apache.paimon.types.DataField; import org.apache.paimon.types.RowType; import org.apache.paimon.utils.ProjectedRow; import org.apache.paimon.utils.Projection; @@ -77,6 +78,7 @@ public class MergeFileSplitRead implements SplitRead { private final MergeFunctionFactory mfFactory; private final MergeSorter mergeSorter; private final List sequenceFields; + private final boolean sequenceOrder; @Nullable private RowType readKeyType; @@ -106,6 +108,7 @@ public MergeFileSplitRead( new MergeSorter( CoreOptions.fromMap(tableSchema.options()), keyType, valueType, null); this.sequenceFields = options.sequenceField(); + this.sequenceOrder = options.sequenceFieldSortOrderIsAscending(); } public Comparator keyComparator() { @@ -129,11 +132,9 @@ public MergeFileSplitRead withReadKeyType(RowType readKeyType) { @Override public MergeFileSplitRead withReadType(RowType readType) { // todo: replace projectedFields with readType + RowType tableRowType = tableSchema.logicalRowType(); int[][] projectedFields = - Arrays.stream( - tableSchema - .logicalRowType() - .getFieldIndices(readType.getFieldNames())) + Arrays.stream(tableRowType.getFieldIndices(readType.getFieldNames())) .mapToObj(i -> new int[] {i}) .toArray(int[][]::new); int[][] newProjectedFields = projectedFields; @@ -159,13 +160,18 @@ public MergeFileSplitRead withReadType(RowType readType) { this.pushdownProjection = projection.pushdownProjection; this.outerProjection = projection.outerProjection; if (pushdownProjection != null) { - RowType pushdownRowType = - tableSchema - .logicalRowType() - .project( - Arrays.stream(pushdownProjection) - .mapToInt(arr -> arr[0]) - .toArray()); + List tableFields = tableRowType.getFields(); + List readFields = readType.getFields(); + List finalReadFields = new ArrayList<>(); + for (int i : Arrays.stream(pushdownProjection).mapToInt(arr -> arr[0]).toArray()) { + DataField requiredField = tableFields.get(i); + finalReadFields.add( + readFields.stream() + .filter(x -> x.name().equals(requiredField.name())) + .findFirst() + .orElse(requiredField)); + } + RowType pushdownRowType = new RowType(finalReadFields); readerFactoryBuilder.withReadValueType(pushdownRowType); mergeSorter.setProjectedValueType(pushdownRowType); } @@ -338,6 +344,6 @@ private RecordReader projectOuter(RecordReader reader) { @Nullable public UserDefinedSeqComparator createUdsComparator() { return UserDefinedSeqComparator.create( - readerFactoryBuilder.readValueType(), sequenceFields); + readerFactoryBuilder.readValueType(), sequenceFields, sequenceOrder); } } diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/OrphanFilesClean.java b/paimon-core/src/main/java/org/apache/paimon/operation/OrphanFilesClean.java index 0f2bad27fc9a..274cdd52fe14 100644 --- a/paimon-core/src/main/java/org/apache/paimon/operation/OrphanFilesClean.java +++ b/paimon-core/src/main/java/org/apache/paimon/operation/OrphanFilesClean.java @@ -105,7 +105,8 @@ protected List validBranches() { List abnormalBranches = new ArrayList<>(); for (String branch : branches) { - if (!new SchemaManager(table.fileIO(), table.location(), branch).latest().isPresent()) { + SchemaManager schemaManager = table.schemaManager().copyWithBranch(branch); + if (!schemaManager.latest().isPresent()) { abnormalBranches.add(branch); } } @@ -119,23 +120,47 @@ protected List validBranches() { return branches; } - protected void cleanSnapshotDir(List branches, Consumer deletedFileConsumer) { + protected void cleanSnapshotDir( + List branches, + Consumer deletedFilesConsumer, + Consumer deletedFilesLenInBytesConsumer) { for (String branch : branches) { FileStoreTable branchTable = table.switchToBranch(branch); SnapshotManager snapshotManager = branchTable.snapshotManager(); // specially handle the snapshot directory - List nonSnapshotFiles = snapshotManager.tryGetNonSnapshotFiles(this::oldEnough); - nonSnapshotFiles.forEach(fileCleaner); - nonSnapshotFiles.forEach(deletedFileConsumer); + List> nonSnapshotFiles = + snapshotManager.tryGetNonSnapshotFiles(this::oldEnough); + nonSnapshotFiles.forEach( + nonSnapshotFile -> + cleanFile( + nonSnapshotFile, + deletedFilesConsumer, + deletedFilesLenInBytesConsumer)); // specially handle the changelog directory - List nonChangelogFiles = snapshotManager.tryGetNonChangelogFiles(this::oldEnough); - nonChangelogFiles.forEach(fileCleaner); - nonChangelogFiles.forEach(deletedFileConsumer); + List> nonChangelogFiles = + snapshotManager.tryGetNonChangelogFiles(this::oldEnough); + nonChangelogFiles.forEach( + nonChangelogFile -> + cleanFile( + nonChangelogFile, + deletedFilesConsumer, + deletedFilesLenInBytesConsumer)); } } + private void cleanFile( + Pair deleteFileInfo, + Consumer deletedFilesConsumer, + Consumer deletedFilesLenInBytesConsumer) { + Path filePath = deleteFileInfo.getLeft(); + Long fileSize = deleteFileInfo.getRight(); + deletedFilesConsumer.accept(filePath); + deletedFilesLenInBytesConsumer.accept(fileSize); + fileCleaner.accept(filePath); + } + protected Set safelyGetAllSnapshots(String branch) throws IOException { FileStoreTable branchTable = table.switchToBranch(branch); SnapshotManager snapshotManager = branchTable.snapshotManager(); @@ -146,14 +171,6 @@ protected Set safelyGetAllSnapshots(String branch) throws IOException return readSnapshots; } - protected void collectWithoutDataFile( - String branch, Consumer usedFileConsumer, Consumer manifestConsumer) - throws IOException { - for (Snapshot snapshot : safelyGetAllSnapshots(branch)) { - collectWithoutDataFile(branch, snapshot, usedFileConsumer, manifestConsumer); - } - } - protected void collectWithoutDataFile( String branch, Snapshot snapshot, diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/PartitionExpire.java b/paimon-core/src/main/java/org/apache/paimon/operation/PartitionExpire.java index 62a9b796476a..d432a37dfd9c 100644 --- a/paimon-core/src/main/java/org/apache/paimon/operation/PartitionExpire.java +++ b/paimon-core/src/main/java/org/apache/paimon/operation/PartitionExpire.java @@ -54,7 +54,7 @@ public class PartitionExpire { private LocalDateTime lastCheck; private final PartitionExpireStrategy strategy; private final boolean endInputCheckPartitionExpire; - private int maxExpires; + private int maxExpireNum; public PartitionExpire( Duration expirationTime, @@ -63,7 +63,8 @@ public PartitionExpire( FileStoreScan scan, FileStoreCommit commit, @Nullable MetastoreClient metastoreClient, - boolean endInputCheckPartitionExpire) { + boolean endInputCheckPartitionExpire, + int maxExpireNum) { this.expirationTime = expirationTime; this.checkInterval = checkInterval; this.strategy = strategy; @@ -72,7 +73,7 @@ public PartitionExpire( this.metastoreClient = metastoreClient; this.lastCheck = LocalDateTime.now(); this.endInputCheckPartitionExpire = endInputCheckPartitionExpire; - this.maxExpires = Integer.MAX_VALUE; + this.maxExpireNum = maxExpireNum; } public PartitionExpire( @@ -81,8 +82,17 @@ public PartitionExpire( PartitionExpireStrategy strategy, FileStoreScan scan, FileStoreCommit commit, - @Nullable MetastoreClient metastoreClient) { - this(expirationTime, checkInterval, strategy, scan, commit, metastoreClient, false); + @Nullable MetastoreClient metastoreClient, + int maxExpireNum) { + this( + expirationTime, + checkInterval, + strategy, + scan, + commit, + metastoreClient, + false, + maxExpireNum); } public PartitionExpire withLock(Lock lock) { @@ -90,8 +100,8 @@ public PartitionExpire withLock(Lock lock) { return this; } - public PartitionExpire withMaxExpires(int maxExpires) { - this.maxExpires = maxExpires; + public PartitionExpire withMaxExpireNum(int maxExpireNum) { + this.maxExpireNum = maxExpireNum; return this; } @@ -145,6 +155,7 @@ private List> doExpire( List> expired = new ArrayList<>(); if (!expiredPartValues.isEmpty()) { + // convert partition value to partition string, and limit the partition num expired = convertToPartitionString(expiredPartValues); LOG.info("Expire Partitions: {}", expired); if (metastoreClient != null) { @@ -175,7 +186,7 @@ private List> convertToPartitionString( .sorted() .map(s -> s.split(DELIMITER)) .map(strategy::toPartitionString) - .limit(Math.min(expiredPartValues.size(), maxExpires)) + .limit(Math.min(expiredPartValues.size(), maxExpireNum)) .collect(Collectors.toList()); } } diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/RawFileSplitRead.java b/paimon-core/src/main/java/org/apache/paimon/operation/RawFileSplitRead.java index fcd2f8798e6c..4fda82f4e88f 100644 --- a/paimon-core/src/main/java/org/apache/paimon/operation/RawFileSplitRead.java +++ b/paimon-core/src/main/java/org/apache/paimon/operation/RawFileSplitRead.java @@ -23,18 +23,22 @@ import org.apache.paimon.deletionvectors.ApplyDeletionVectorReader; import org.apache.paimon.deletionvectors.DeletionVector; import org.apache.paimon.disk.IOManager; +import org.apache.paimon.fileindex.FileIndexResult; +import org.apache.paimon.fileindex.bitmap.ApplyBitmapIndexRecordReader; +import org.apache.paimon.fileindex.bitmap.BitmapIndexResult; import org.apache.paimon.format.FileFormatDiscover; import org.apache.paimon.format.FormatKey; import org.apache.paimon.format.FormatReaderContext; import org.apache.paimon.fs.FileIO; import org.apache.paimon.io.DataFileMeta; import org.apache.paimon.io.DataFilePathFactory; -import org.apache.paimon.io.FileIndexSkipper; -import org.apache.paimon.io.FileRecordReader; +import org.apache.paimon.io.DataFileRecordReader; +import org.apache.paimon.io.FileIndexEvaluator; import org.apache.paimon.mergetree.compact.ConcatRecordReader; import org.apache.paimon.partition.PartitionUtils; import org.apache.paimon.predicate.Predicate; -import org.apache.paimon.reader.EmptyRecordReader; +import org.apache.paimon.reader.EmptyFileRecordReader; +import org.apache.paimon.reader.FileRecordReader; import org.apache.paimon.reader.ReaderSupplier; import org.apache.paimon.reader.RecordReader; import org.apache.paimon.schema.SchemaManager; @@ -42,9 +46,9 @@ import org.apache.paimon.table.source.DataSplit; import org.apache.paimon.types.DataField; import org.apache.paimon.types.RowType; -import org.apache.paimon.utils.BulkFormatMapping; -import org.apache.paimon.utils.BulkFormatMapping.BulkFormatMappingBuilder; import org.apache.paimon.utils.FileStorePathFactory; +import org.apache.paimon.utils.FormatReaderMapping; +import org.apache.paimon.utils.FormatReaderMapping.Builder; import org.apache.paimon.utils.IOExceptionSupplier; import org.slf4j.Logger; @@ -71,7 +75,7 @@ public class RawFileSplitRead implements SplitRead { private final TableSchema schema; private final FileFormatDiscover formatDiscover; private final FileStorePathFactory pathFactory; - private final Map bulkFormatMappings; + private final Map formatReaderMappings; private final boolean fileIndexReadEnabled; private RowType readRowType; @@ -90,7 +94,7 @@ public RawFileSplitRead( this.schema = schema; this.formatDiscover = formatDiscover; this.pathFactory = pathFactory; - this.bulkFormatMappings = new HashMap<>(); + this.formatReaderMappings = new HashMap<>(); this.fileIndexReadEnabled = fileIndexReadEnabled; this.readRowType = rowType; } @@ -146,26 +150,25 @@ public RecordReader createReader( List> suppliers = new ArrayList<>(); List readTableFields = readRowType.getFields(); - BulkFormatMappingBuilder bulkFormatMappingBuilder = - new BulkFormatMappingBuilder( - formatDiscover, readTableFields, TableSchema::fields, filters); + Builder formatReaderMappingBuilder = + new Builder(formatDiscover, readTableFields, TableSchema::fields, filters); for (int i = 0; i < files.size(); i++) { DataFileMeta file = files.get(i); String formatIdentifier = DataFilePathFactory.formatIdentifier(file.fileName()); long schemaId = file.schemaId(); - Supplier formatSupplier = + Supplier formatSupplier = () -> - bulkFormatMappingBuilder.build( + formatReaderMappingBuilder.build( formatIdentifier, schema, schemaId == schema.id() ? schema : schemaManager.schema(schemaId)); - BulkFormatMapping bulkFormatMapping = - bulkFormatMappings.computeIfAbsent( + FormatReaderMapping formatReaderMapping = + formatReaderMappings.computeIfAbsent( new FormatKey(file.schemaId(), formatIdentifier), key -> formatSupplier.get()); @@ -177,43 +180,53 @@ public RecordReader createReader( partition, file, dataFilePathFactory, - bulkFormatMapping, + formatReaderMapping, dvFactory)); } return ConcatRecordReader.create(suppliers); } - private RecordReader createFileReader( + private FileRecordReader createFileReader( BinaryRow partition, DataFileMeta file, DataFilePathFactory dataFilePathFactory, - BulkFormatMapping bulkFormatMapping, + FormatReaderMapping formatReaderMapping, IOExceptionSupplier dvFactory) throws IOException { + FileIndexResult fileIndexResult = null; if (fileIndexReadEnabled) { - boolean skip = - FileIndexSkipper.skip( + fileIndexResult = + FileIndexEvaluator.evaluate( fileIO, - bulkFormatMapping.getDataSchema(), - bulkFormatMapping.getDataFilters(), + formatReaderMapping.getDataSchema(), + formatReaderMapping.getDataFilters(), dataFilePathFactory, file); - if (skip) { - return new EmptyRecordReader<>(); + if (!fileIndexResult.remain()) { + return new EmptyFileRecordReader<>(); } } - FileRecordReader fileRecordReader = - new FileRecordReader( - bulkFormatMapping.getReaderFactory(), - new FormatReaderContext( - fileIO, - dataFilePathFactory.toPath(file.fileName()), - file.fileSize()), - bulkFormatMapping.getIndexMapping(), - bulkFormatMapping.getCastMapping(), - PartitionUtils.create(bulkFormatMapping.getPartitionPair(), partition)); + FormatReaderContext formatReaderContext = + new FormatReaderContext( + fileIO, + dataFilePathFactory.toPath(file.fileName()), + file.fileSize(), + fileIndexResult); + FileRecordReader fileRecordReader = + new DataFileRecordReader( + formatReaderMapping.getReaderFactory(), + formatReaderContext, + formatReaderMapping.getIndexMapping(), + formatReaderMapping.getCastMapping(), + PartitionUtils.create(formatReaderMapping.getPartitionPair(), partition)); + + if (fileIndexResult instanceof BitmapIndexResult) { + fileRecordReader = + new ApplyBitmapIndexRecordReader( + fileRecordReader, (BitmapIndexResult) fileIndexResult); + } DeletionVector deletionVector = dvFactory == null ? null : dvFactory.get(); if (deletionVector != null && !deletionVector.isEmpty()) { diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/SnapshotDeletion.java b/paimon-core/src/main/java/org/apache/paimon/operation/SnapshotDeletion.java index d86907ecea54..7d55b64c8eac 100644 --- a/paimon-core/src/main/java/org/apache/paimon/operation/SnapshotDeletion.java +++ b/paimon-core/src/main/java/org/apache/paimon/operation/SnapshotDeletion.java @@ -23,8 +23,8 @@ import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.Path; import org.apache.paimon.index.IndexFileHandler; +import org.apache.paimon.manifest.ExpireFileEntry; import org.apache.paimon.manifest.FileSource; -import org.apache.paimon.manifest.ManifestEntry; import org.apache.paimon.manifest.ManifestFile; import org.apache.paimon.manifest.ManifestList; import org.apache.paimon.stats.StatsFileHandler; @@ -65,15 +65,15 @@ public SnapshotDeletion( } @Override - public void cleanUnusedDataFiles(Snapshot snapshot, Predicate skipper) { + public void cleanUnusedDataFiles(Snapshot snapshot, Predicate skipper) { if (changelogDecoupled && !produceChangelog) { // Skip clean the 'APPEND' data files.If we do not have the file source information // eg: the old version table file, we just skip clean this here, let it done by // ExpireChangelogImpl - Predicate enriched = + Predicate enriched = manifestEntry -> skipper.test(manifestEntry) - || (manifestEntry.file().fileSource().orElse(FileSource.APPEND) + || (manifestEntry.fileSource().orElse(FileSource.APPEND) == FileSource.APPEND); cleanUnusedDataFiles(snapshot.deltaManifestList(), enriched); } else { @@ -92,8 +92,8 @@ public void cleanUnusedManifests(Snapshot snapshot, Set skippingSet) { } @VisibleForTesting - void cleanUnusedDataFile(List dataFileLog) { - Map>> dataFileToDelete = new HashMap<>(); + void cleanUnusedDataFile(List dataFileLog) { + Map>> dataFileToDelete = new HashMap<>(); getDataFileToDelete(dataFileToDelete, dataFileLog); doCleanUnusedDataFile(dataFileToDelete, f -> false); } diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/TagDeletion.java b/paimon-core/src/main/java/org/apache/paimon/operation/TagDeletion.java index a6cd338d5859..2722ed0c7ec8 100644 --- a/paimon-core/src/main/java/org/apache/paimon/operation/TagDeletion.java +++ b/paimon-core/src/main/java/org/apache/paimon/operation/TagDeletion.java @@ -23,7 +23,7 @@ import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.Path; import org.apache.paimon.index.IndexFileHandler; -import org.apache.paimon.manifest.ManifestEntry; +import org.apache.paimon.manifest.ExpireFileEntry; import org.apache.paimon.manifest.ManifestFile; import org.apache.paimon.manifest.ManifestList; import org.apache.paimon.stats.StatsFileHandler; @@ -68,8 +68,8 @@ public TagDeletion( } @Override - public void cleanUnusedDataFiles(Snapshot taggedSnapshot, Predicate skipper) { - Collection manifestEntries; + public void cleanUnusedDataFiles(Snapshot taggedSnapshot, Predicate skipper) { + Collection manifestEntries; try { manifestEntries = readMergedDataFiles(taggedSnapshot); } catch (IOException e) { @@ -78,11 +78,11 @@ public void cleanUnusedDataFiles(Snapshot taggedSnapshot, Predicate dataFileToDelete = new HashSet<>(); - for (ManifestEntry entry : manifestEntries) { + for (ExpireFileEntry entry : manifestEntries) { if (!skipper.test(entry)) { Path bucketPath = pathFactory.bucketPath(entry.partition(), entry.bucket()); - dataFileToDelete.add(new Path(bucketPath, entry.file().fileName())); - for (String file : entry.file().extraFiles()) { + dataFileToDelete.add(new Path(bucketPath, entry.fileName())); + for (String file : entry.extraFiles()) { dataFileToDelete.add(new Path(bucketPath, file)); } @@ -98,11 +98,12 @@ public void cleanUnusedManifests(Snapshot taggedSnapshot, Set skippingSe cleanUnusedManifests(taggedSnapshot, skippingSet, true, false); } - public Predicate dataFileSkipper(Snapshot fromSnapshot) throws Exception { + public Predicate dataFileSkipper(Snapshot fromSnapshot) throws Exception { return dataFileSkipper(Collections.singletonList(fromSnapshot)); } - public Predicate dataFileSkipper(List fromSnapshots) throws Exception { + public Predicate dataFileSkipper(List fromSnapshots) + throws Exception { Map>> skipped = new HashMap<>(); for (Snapshot snapshot : fromSnapshots) { addMergedDataFiles(skipped, snapshot); diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/metrics/CommitMetrics.java b/paimon-core/src/main/java/org/apache/paimon/operation/metrics/CommitMetrics.java index 49476db7fad4..0f8ccbc65ce2 100644 --- a/paimon-core/src/main/java/org/apache/paimon/operation/metrics/CommitMetrics.java +++ b/paimon-core/src/main/java/org/apache/paimon/operation/metrics/CommitMetrics.java @@ -76,6 +76,9 @@ public MetricGroup getMetricGroup() { @VisibleForTesting static final String LAST_PARTITIONS_WRITTEN = "lastPartitionsWritten"; @VisibleForTesting static final String LAST_BUCKETS_WRITTEN = "lastBucketsWritten"; + static final String LAST_COMPACTION_INPUT_FILE_SIZE = "lastCompactionInputFileSize"; + static final String LAST_COMPACTION_OUTPUT_FILE_SIZE = "lastCompactionOutputFileSize"; + private void registerGenericCommitMetrics() { metricGroup.gauge( LAST_COMMIT_DURATION, () -> latestCommit == null ? 0L : latestCommit.getDuration()); @@ -121,6 +124,12 @@ private void registerGenericCommitMetrics() { metricGroup.gauge( LAST_CHANGELOG_RECORDS_COMMIT_COMPACTED, () -> latestCommit == null ? 0L : latestCommit.getChangelogRecordsCompacted()); + metricGroup.gauge( + LAST_COMPACTION_INPUT_FILE_SIZE, + () -> latestCommit == null ? 0L : latestCommit.getCompactionInputFileSize()); + metricGroup.gauge( + LAST_COMPACTION_OUTPUT_FILE_SIZE, + () -> latestCommit == null ? 0L : latestCommit.getCompactionOutputFileSize()); } public void reportCommit(CommitStats commitStats) { diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/metrics/CommitStats.java b/paimon-core/src/main/java/org/apache/paimon/operation/metrics/CommitStats.java index a19d93508a90..657bd6aae153 100644 --- a/paimon-core/src/main/java/org/apache/paimon/operation/metrics/CommitStats.java +++ b/paimon-core/src/main/java/org/apache/paimon/operation/metrics/CommitStats.java @@ -20,6 +20,7 @@ import org.apache.paimon.annotation.VisibleForTesting; import org.apache.paimon.data.BinaryRow; +import org.apache.paimon.io.DataFileMeta; import org.apache.paimon.manifest.FileKind; import org.apache.paimon.manifest.ManifestEntry; @@ -41,6 +42,8 @@ public class CommitStats { private final long tableFilesAppended; private final long tableFilesDeleted; private final long changelogFilesAppended; + private final long compactionInputFileSize; + private final long compactionOutputFileSize; private final long changelogFilesCompacted; private final long changelogRecordsCompacted; @@ -61,15 +64,28 @@ public CommitStats( int generatedSnapshots, int attempts) { List addedTableFiles = new ArrayList<>(appendTableFiles); - addedTableFiles.addAll( + List compactAfterFiles = compactTableFiles.stream() .filter(f -> FileKind.ADD.equals(f.kind())) - .collect(Collectors.toList())); + .collect(Collectors.toList()); + addedTableFiles.addAll(compactAfterFiles); List deletedTableFiles = compactTableFiles.stream() .filter(f -> FileKind.DELETE.equals(f.kind())) .collect(Collectors.toList()); + this.compactionInputFileSize = + deletedTableFiles.stream() + .map(ManifestEntry::file) + .map(DataFileMeta::fileSize) + .reduce(Long::sum) + .orElse(0L); + this.compactionOutputFileSize = + compactAfterFiles.stream() + .map(ManifestEntry::file) + .map(DataFileMeta::fileSize) + .reduce(Long::sum) + .orElse(0L); this.tableFilesAdded = addedTableFiles.size(); this.tableFilesAppended = appendTableFiles.size(); this.tableFilesDeleted = deletedTableFiles.size(); @@ -203,4 +219,12 @@ protected long getDuration() { protected int getAttempts() { return attempts; } + + public long getCompactionInputFileSize() { + return compactionInputFileSize; + } + + public long getCompactionOutputFileSize() { + return compactionOutputFileSize; + } } diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/metrics/CompactionMetrics.java b/paimon-core/src/main/java/org/apache/paimon/operation/metrics/CompactionMetrics.java index dd899e9bf2c4..a3074daebbde 100644 --- a/paimon-core/src/main/java/org/apache/paimon/operation/metrics/CompactionMetrics.java +++ b/paimon-core/src/main/java/org/apache/paimon/operation/metrics/CompactionMetrics.java @@ -20,6 +20,7 @@ import org.apache.paimon.annotation.VisibleForTesting; import org.apache.paimon.data.BinaryRow; +import org.apache.paimon.metrics.Counter; import org.apache.paimon.metrics.MetricGroup; import org.apache.paimon.metrics.MetricRegistry; @@ -41,6 +42,12 @@ public class CompactionMetrics { public static final String AVG_LEVEL0_FILE_COUNT = "avgLevel0FileCount"; public static final String COMPACTION_THREAD_BUSY = "compactionThreadBusy"; public static final String AVG_COMPACTION_TIME = "avgCompactionTime"; + public static final String COMPACTION_COMPLETED_COUNT = "compactionCompletedCount"; + public static final String COMPACTION_QUEUED_COUNT = "compactionQueuedCount"; + public static final String MAX_COMPACTION_INPUT_SIZE = "maxCompactionInputSize"; + public static final String MAX_COMPACTION_OUTPUT_SIZE = "maxCompactionOutputSize"; + public static final String AVG_COMPACTION_INPUT_SIZE = "avgCompactionInputSize"; + public static final String AVG_COMPACTION_OUTPUT_SIZE = "avgCompactionOutputSize"; private static final long BUSY_MEASURE_MILLIS = 60_000; private static final int COMPACTION_TIME_WINDOW = 100; @@ -48,6 +55,8 @@ public class CompactionMetrics { private final Map reporters; private final Map compactTimers; private final Queue compactionTimes; + private Counter compactionsCompletedCounter; + private Counter compactionsQueuedCounter; public CompactionMetrics(MetricRegistry registry, String tableName) { this.metricGroup = registry.tableMetricGroup(GROUP_NAME, tableName); @@ -67,15 +76,37 @@ private void registerGenericCompactionMetrics() { metricGroup.gauge(MAX_LEVEL0_FILE_COUNT, () -> getLevel0FileCountStream().max().orElse(-1)); metricGroup.gauge( AVG_LEVEL0_FILE_COUNT, () -> getLevel0FileCountStream().average().orElse(-1)); + metricGroup.gauge( + MAX_COMPACTION_INPUT_SIZE, () -> getCompactionInputSizeStream().max().orElse(-1)); + metricGroup.gauge( + MAX_COMPACTION_OUTPUT_SIZE, () -> getCompactionOutputSizeStream().max().orElse(-1)); + metricGroup.gauge( + AVG_COMPACTION_INPUT_SIZE, + () -> getCompactionInputSizeStream().average().orElse(-1)); + metricGroup.gauge( + AVG_COMPACTION_OUTPUT_SIZE, + () -> getCompactionOutputSizeStream().average().orElse(-1)); + metricGroup.gauge( AVG_COMPACTION_TIME, () -> getCompactionTimeStream().average().orElse(0.0)); metricGroup.gauge(COMPACTION_THREAD_BUSY, () -> getCompactBusyStream().sum()); + + compactionsCompletedCounter = metricGroup.counter(COMPACTION_COMPLETED_COUNT); + compactionsQueuedCounter = metricGroup.counter(COMPACTION_QUEUED_COUNT); } private LongStream getLevel0FileCountStream() { return reporters.values().stream().mapToLong(r -> r.level0FileCount); } + private LongStream getCompactionInputSizeStream() { + return reporters.values().stream().mapToLong(r -> r.compactionInputSize); + } + + private LongStream getCompactionOutputSizeStream() { + return reporters.values().stream().mapToLong(r -> r.compactionOutputSize); + } + private DoubleStream getCompactBusyStream() { return compactTimers.values().stream() .mapToDouble(t -> 100.0 * t.calculateLength() / BUSY_MEASURE_MILLIS); @@ -98,6 +129,16 @@ public interface Reporter { void reportCompactionTime(long time); + void increaseCompactionsCompletedCount(); + + void increaseCompactionsQueuedCount(); + + void decreaseCompactionsQueuedCount(); + + void reportCompactionInputSize(long bytes); + + void reportCompactionOutputSize(long bytes); + void unregister(); } @@ -105,6 +146,8 @@ private class ReporterImpl implements Reporter { private final PartitionAndBucket key; private long level0FileCount; + private long compactionInputSize = 0; + private long compactionOutputSize = 0; private ReporterImpl(PartitionAndBucket key) { this.key = key; @@ -128,11 +171,36 @@ public void reportCompactionTime(long time) { } } + @Override + public void reportCompactionInputSize(long bytes) { + this.compactionInputSize = bytes; + } + + @Override + public void reportCompactionOutputSize(long bytes) { + this.compactionOutputSize = bytes; + } + @Override public void reportLevel0FileCount(long count) { this.level0FileCount = count; } + @Override + public void increaseCompactionsCompletedCount() { + compactionsCompletedCounter.inc(); + } + + @Override + public void increaseCompactionsQueuedCount() { + compactionsQueuedCounter.inc(); + } + + @Override + public void decreaseCompactionsQueuedCount() { + compactionsQueuedCounter.dec(); + } + @Override public void unregister() { reporters.remove(key); diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/metrics/ScanMetrics.java b/paimon-core/src/main/java/org/apache/paimon/operation/metrics/ScanMetrics.java index 9fcbb8960fc5..96f0aec1c0b2 100644 --- a/paimon-core/src/main/java/org/apache/paimon/operation/metrics/ScanMetrics.java +++ b/paimon-core/src/main/java/org/apache/paimon/operation/metrics/ScanMetrics.java @@ -49,12 +49,6 @@ public MetricGroup getMetricGroup() { public static final String SCAN_DURATION = "scanDuration"; public static final String LAST_SCANNED_MANIFESTS = "lastScannedManifests"; - public static final String LAST_SKIPPED_BY_PARTITION_AND_STATS = - "lastSkippedByPartitionAndStats"; - - public static final String LAST_SKIPPED_BY_WHOLE_BUCKET_FILES_FILTER = - "lastSkippedByWholeBucketFilesFilter"; - public static final String LAST_SCAN_SKIPPED_TABLE_FILES = "lastScanSkippedTableFiles"; public static final String LAST_SCAN_RESULTED_TABLE_FILES = "lastScanResultedTableFiles"; @@ -66,12 +60,6 @@ private void registerGenericScanMetrics() { metricGroup.gauge( LAST_SCANNED_MANIFESTS, () -> latestScan == null ? 0L : latestScan.getScannedManifests()); - metricGroup.gauge( - LAST_SKIPPED_BY_PARTITION_AND_STATS, - () -> latestScan == null ? 0L : latestScan.getSkippedByPartitionAndStats()); - metricGroup.gauge( - LAST_SKIPPED_BY_WHOLE_BUCKET_FILES_FILTER, - () -> latestScan == null ? 0L : latestScan.getSkippedByWholeBucketFiles()); metricGroup.gauge( LAST_SCAN_SKIPPED_TABLE_FILES, () -> latestScan == null ? 0L : latestScan.getSkippedTableFiles()); diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/metrics/ScanStats.java b/paimon-core/src/main/java/org/apache/paimon/operation/metrics/ScanStats.java index e760282e687a..700619c3680f 100644 --- a/paimon-core/src/main/java/org/apache/paimon/operation/metrics/ScanStats.java +++ b/paimon-core/src/main/java/org/apache/paimon/operation/metrics/ScanStats.java @@ -25,23 +25,15 @@ public class ScanStats { // the unit is milliseconds private final long duration; private final long scannedManifests; - private final long skippedByPartitionAndStats; - private final long skippedByWholeBucketFiles; private final long skippedTableFiles; private final long resultedTableFiles; public ScanStats( - long duration, - long scannedManifests, - long skippedByPartitionAndStats, - long skippedByWholeBucketFiles, - long resultedTableFiles) { + long duration, long scannedManifests, long skippedTableFiles, long resultedTableFiles) { this.duration = duration; this.scannedManifests = scannedManifests; - this.skippedByPartitionAndStats = skippedByPartitionAndStats; - this.skippedByWholeBucketFiles = skippedByWholeBucketFiles; - this.skippedTableFiles = skippedByPartitionAndStats + skippedByWholeBucketFiles; + this.skippedTableFiles = skippedTableFiles; this.resultedTableFiles = resultedTableFiles; } @@ -60,16 +52,6 @@ protected long getResultedTableFiles() { return resultedTableFiles; } - @VisibleForTesting - protected long getSkippedByPartitionAndStats() { - return skippedByPartitionAndStats; - } - - @VisibleForTesting - protected long getSkippedByWholeBucketFiles() { - return skippedByWholeBucketFiles; - } - @VisibleForTesting protected long getDuration() { return duration; diff --git a/paimon-core/src/main/java/org/apache/paimon/partition/PartitionPredicate.java b/paimon-core/src/main/java/org/apache/paimon/partition/PartitionPredicate.java index 12ea884be15f..1f6c2cfe454e 100644 --- a/paimon-core/src/main/java/org/apache/paimon/partition/PartitionPredicate.java +++ b/paimon-core/src/main/java/org/apache/paimon/partition/PartitionPredicate.java @@ -19,8 +19,10 @@ package org.apache.paimon.partition; import org.apache.paimon.data.BinaryRow; +import org.apache.paimon.data.GenericRow; import org.apache.paimon.data.InternalArray; import org.apache.paimon.data.InternalRow; +import org.apache.paimon.data.serializer.InternalRowSerializer; import org.apache.paimon.data.serializer.InternalSerializers; import org.apache.paimon.data.serializer.Serializer; import org.apache.paimon.format.SimpleColStats; @@ -33,6 +35,7 @@ import javax.annotation.Nullable; +import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -40,6 +43,7 @@ import java.util.Set; import static org.apache.paimon.utils.InternalRowPartitionComputer.convertSpecToInternal; +import static org.apache.paimon.utils.InternalRowPartitionComputer.convertSpecToInternalRow; import static org.apache.paimon.utils.Preconditions.checkArgument; import static org.apache.paimon.utils.Preconditions.checkNotNull; @@ -231,4 +235,15 @@ static Predicate createPartitionPredicate( .map(p -> createPartitionPredicate(p, rowType, defaultPartValue)) .toArray(Predicate[]::new)); } + + static List createBinaryPartitions( + List> partitions, RowType partitionType, String defaultPartValue) { + InternalRowSerializer serializer = new InternalRowSerializer(partitionType); + List result = new ArrayList<>(); + for (Map spec : partitions) { + GenericRow row = convertSpecToInternalRow(spec, partitionType, defaultPartValue); + result.add(serializer.toBinaryRow(row).copy()); + } + return result; + } } diff --git a/paimon-core/src/main/java/org/apache/paimon/partition/PartitionValuesTimeExpireStrategy.java b/paimon-core/src/main/java/org/apache/paimon/partition/PartitionValuesTimeExpireStrategy.java index 80ae633fd297..51c53282c476 100644 --- a/paimon-core/src/main/java/org/apache/paimon/partition/PartitionValuesTimeExpireStrategy.java +++ b/paimon-core/src/main/java/org/apache/paimon/partition/PartitionValuesTimeExpireStrategy.java @@ -81,23 +81,34 @@ public boolean test(BinaryRow partition) { LocalDateTime partTime = timeExtractor.extract(partitionKeys, Arrays.asList(array)); return expireDateTime.isAfter(partTime); } catch (DateTimeParseException e) { - String partitionInfo = - IntStream.range(0, partitionKeys.size()) - .mapToObj(i -> partitionKeys.get(i) + ":" + array[i]) - .collect(Collectors.joining(",")); LOG.warn( "Can't extract datetime from partition {}. If you want to configure partition expiration, please:\n" + " 1. Check the expiration configuration.\n" + " 2. Manually delete the partition using the drop-partition command if the partition" + " value is non-date formatted.\n" + " 3. Use '{}' expiration strategy by set '{}', which supports non-date formatted partition.", - partitionInfo, + formatPartitionInfo(array), + CoreOptions.PartitionExpireStrategy.UPDATE_TIME, + CoreOptions.PARTITION_EXPIRATION_STRATEGY.key()); + return false; + } catch (NullPointerException e) { + // there might exist NULL partition value + LOG.warn( + "This partition {} cannot be expired because it contains null value. " + + "You can try to drop it manually or use '{}' expiration strategy by set '{}'.", + formatPartitionInfo(array), CoreOptions.PartitionExpireStrategy.UPDATE_TIME, CoreOptions.PARTITION_EXPIRATION_STRATEGY.key()); return false; } } + private String formatPartitionInfo(Object[] array) { + return IntStream.range(0, partitionKeys.size()) + .mapToObj(i -> partitionKeys.get(i) + ":" + array[i]) + .collect(Collectors.joining(",")); + } + @Override public boolean test( long rowCount, diff --git a/paimon-core/src/main/java/org/apache/paimon/privilege/NoPrivilegeException.java b/paimon-core/src/main/java/org/apache/paimon/privilege/NoPrivilegeException.java index c868aebe78d3..5feb78058377 100644 --- a/paimon-core/src/main/java/org/apache/paimon/privilege/NoPrivilegeException.java +++ b/paimon-core/src/main/java/org/apache/paimon/privilege/NoPrivilegeException.java @@ -23,6 +23,9 @@ /** Thrown when tries to perform an operation but the current user does not have the privilege. */ public class NoPrivilegeException extends RuntimeException { + private final String user; + private final String objectType; + private final String identifier; public NoPrivilegeException( String user, String objectType, String identifier, PrivilegeType... privilege) { @@ -35,5 +38,20 @@ public NoPrivilegeException( .collect(Collectors.joining(" or ")), objectType, identifier)); + this.user = user; + this.objectType = objectType; + this.identifier = identifier; + } + + String getUser() { + return user; + } + + String getObjectType() { + return objectType; + } + + String getIdentifier() { + return identifier; } } diff --git a/paimon-core/src/main/java/org/apache/paimon/privilege/PrivilegeChecker.java b/paimon-core/src/main/java/org/apache/paimon/privilege/PrivilegeChecker.java index ebd579bdb15f..1771d40f4028 100644 --- a/paimon-core/src/main/java/org/apache/paimon/privilege/PrivilegeChecker.java +++ b/paimon-core/src/main/java/org/apache/paimon/privilege/PrivilegeChecker.java @@ -24,6 +24,22 @@ /** Check if current user has privilege to perform related operations. */ public interface PrivilegeChecker extends Serializable { + default void assertCanSelectOrInsert(Identifier identifier) { + try { + assertCanSelect(identifier); + } catch (NoPrivilegeException e) { + try { + assertCanInsert(identifier); + } catch (NoPrivilegeException e1) { + throw new NoPrivilegeException( + e1.getUser(), + e1.getObjectType(), + e1.getIdentifier(), + PrivilegeType.SELECT, + PrivilegeType.INSERT); + } + } + } void assertCanSelect(Identifier identifier); diff --git a/paimon-core/src/main/java/org/apache/paimon/privilege/PrivilegedCatalog.java b/paimon-core/src/main/java/org/apache/paimon/privilege/PrivilegedCatalog.java index bda06a08d136..2e88213a24b9 100644 --- a/paimon-core/src/main/java/org/apache/paimon/privilege/PrivilegedCatalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/privilege/PrivilegedCatalog.java @@ -102,12 +102,16 @@ public void renameTable(Identifier fromTable, Identifier toTable, boolean ignore throws TableNotExistException, TableAlreadyExistException { privilegeManager.getPrivilegeChecker().assertCanAlterTable(fromTable); wrapped.renameTable(fromTable, toTable, ignoreIfNotExists); - Preconditions.checkState( - wrapped.tableExists(toTable), - "Table " - + toTable - + " does not exist. There might be concurrent renaming. " - + "Aborting updates in privilege system."); + + try { + getTable(toTable); + } catch (TableNotExistException e) { + throw new IllegalStateException( + "Table " + + toTable + + " does not exist. There might be concurrent renaming. " + + "Aborting updates in privilege system."); + } privilegeManager.objectRenamed(fromTable.getFullName(), toTable.getFullName()); } @@ -123,7 +127,7 @@ public void alterTable( public Table getTable(Identifier identifier) throws TableNotExistException { Table table = wrapped.getTable(identifier); if (table instanceof FileStoreTable) { - return new PrivilegedFileStoreTable( + return PrivilegedFileStoreTable.wrap( (FileStoreTable) table, privilegeManager.getPrivilegeChecker(), identifier); } else { return table; @@ -157,8 +161,11 @@ public void grantPrivilegeOnDatabase( Preconditions.checkArgument( privilege.canGrantOnDatabase(), "Privilege " + privilege + " can't be granted on a database"); - Preconditions.checkArgument( - databaseExists(databaseName), "Database " + databaseName + " does not exist"); + try { + getDatabase(databaseName); + } catch (DatabaseNotExistException e) { + throw new IllegalArgumentException("Database " + databaseName + " does not exist"); + } privilegeManager.grant(user, databaseName, privilege); } @@ -166,8 +173,12 @@ public void grantPrivilegeOnTable(String user, Identifier identifier, PrivilegeT Preconditions.checkArgument( privilege.canGrantOnTable(), "Privilege " + privilege + " can't be granted on a table"); - Preconditions.checkArgument( - tableExists(identifier), "Table " + identifier + " does not exist"); + + try { + getTable(identifier); + } catch (TableNotExistException e) { + throw new IllegalArgumentException("Table " + identifier + " does not exist"); + } privilegeManager.grant(user, identifier.getFullName(), privilege); } diff --git a/paimon-core/src/main/java/org/apache/paimon/privilege/PrivilegedFileStore.java b/paimon-core/src/main/java/org/apache/paimon/privilege/PrivilegedFileStore.java index ec068b25acb3..3ee0d5fa9b01 100644 --- a/paimon-core/src/main/java/org/apache/paimon/privilege/PrivilegedFileStore.java +++ b/paimon-core/src/main/java/org/apache/paimon/privilege/PrivilegedFileStore.java @@ -20,6 +20,7 @@ import org.apache.paimon.CoreOptions; import org.apache.paimon.FileStore; +import org.apache.paimon.Snapshot; import org.apache.paimon.catalog.Identifier; import org.apache.paimon.fs.Path; import org.apache.paimon.index.IndexFileHandler; @@ -47,6 +48,8 @@ import org.apache.paimon.utils.SnapshotManager; import org.apache.paimon.utils.TagManager; +import org.apache.paimon.shade.caffeine2.com.github.benmanes.caffeine.cache.Cache; + import javax.annotation.Nullable; import java.util.List; @@ -72,7 +75,7 @@ public FileStorePathFactory pathFactory() { @Override public SnapshotManager snapshotManager() { - privilegeChecker.assertCanSelect(identifier); + privilegeChecker.assertCanSelectOrInsert(identifier); return wrapped.snapshotManager(); } @@ -210,4 +213,9 @@ public List createTagCallbacks() { public void setManifestCache(SegmentsCache manifestCache) { wrapped.setManifestCache(manifestCache); } + + @Override + public void setSnapshotCache(Cache cache) { + wrapped.setSnapshotCache(cache); + } } diff --git a/paimon-core/src/main/java/org/apache/paimon/privilege/PrivilegedFileStoreTable.java b/paimon-core/src/main/java/org/apache/paimon/privilege/PrivilegedFileStoreTable.java index 4eb1a1a90c8c..52c806c7c53b 100644 --- a/paimon-core/src/main/java/org/apache/paimon/privilege/PrivilegedFileStoreTable.java +++ b/paimon-core/src/main/java/org/apache/paimon/privilege/PrivilegedFileStoreTable.java @@ -19,6 +19,7 @@ package org.apache.paimon.privilege; import org.apache.paimon.FileStore; +import org.apache.paimon.Snapshot; import org.apache.paimon.catalog.Identifier; import org.apache.paimon.manifest.ManifestCacheFilter; import org.apache.paimon.schema.TableSchema; @@ -26,6 +27,7 @@ import org.apache.paimon.table.DelegatedFileStoreTable; import org.apache.paimon.table.ExpireSnapshots; import org.apache.paimon.table.FileStoreTable; +import org.apache.paimon.table.object.ObjectTable; import org.apache.paimon.table.query.LocalTableQuery; import org.apache.paimon.table.sink.TableCommitImpl; import org.apache.paimon.table.sink.TableWriteImpl; @@ -35,26 +37,46 @@ import org.apache.paimon.table.source.StreamDataTableScan; import org.apache.paimon.table.source.snapshot.SnapshotReader; import org.apache.paimon.utils.BranchManager; +import org.apache.paimon.utils.SnapshotManager; import org.apache.paimon.utils.TagManager; import java.time.Duration; import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.OptionalLong; /** {@link FileStoreTable} with privilege checks. */ public class PrivilegedFileStoreTable extends DelegatedFileStoreTable { - private final PrivilegeChecker privilegeChecker; - private final Identifier identifier; + protected final PrivilegeChecker privilegeChecker; + protected final Identifier identifier; - public PrivilegedFileStoreTable( + protected PrivilegedFileStoreTable( FileStoreTable wrapped, PrivilegeChecker privilegeChecker, Identifier identifier) { super(wrapped); this.privilegeChecker = privilegeChecker; this.identifier = identifier; } + @Override + public SnapshotManager snapshotManager() { + privilegeChecker.assertCanSelectOrInsert(identifier); + return wrapped.snapshotManager(); + } + + @Override + public OptionalLong latestSnapshotId() { + privilegeChecker.assertCanSelectOrInsert(identifier); + return wrapped.latestSnapshotId(); + } + + @Override + public Snapshot snapshot(long snapshotId) { + privilegeChecker.assertCanSelectOrInsert(identifier); + return wrapped.snapshot(snapshotId); + } + @Override public SnapshotReader newSnapshotReader() { privilegeChecker.assertCanSelect(identifier); @@ -85,18 +107,6 @@ public Optional statistics() { return wrapped.statistics(); } - @Override - public FileStoreTable copy(Map dynamicOptions) { - return new PrivilegedFileStoreTable( - wrapped.copy(dynamicOptions), privilegeChecker, identifier); - } - - @Override - public FileStoreTable copy(TableSchema newTableSchema) { - return new PrivilegedFileStoreTable( - wrapped.copy(newTableSchema), privilegeChecker, identifier); - } - @Override public void rollbackTo(long snapshotId) { privilegeChecker.assertCanInsert(identifier); @@ -181,18 +191,6 @@ public ExpireSnapshots newExpireChangelog() { return wrapped.newExpireChangelog(); } - @Override - public FileStoreTable copyWithoutTimeTravel(Map dynamicOptions) { - return new PrivilegedFileStoreTable( - wrapped.copyWithoutTimeTravel(dynamicOptions), privilegeChecker, identifier); - } - - @Override - public FileStoreTable copyWithLatestSchema() { - return new PrivilegedFileStoreTable( - wrapped.copyWithLatestSchema(), privilegeChecker, identifier); - } - @Override public DataTableScan newScan() { privilegeChecker.assertCanSelect(identifier); @@ -241,11 +239,7 @@ public LocalTableQuery newLocalTableQuery() { return wrapped.newLocalTableQuery(); } - @Override - public FileStoreTable switchToBranch(String branchName) { - return new PrivilegedFileStoreTable( - wrapped.switchToBranch(branchName), privilegeChecker, identifier); - } + // ======================= equals ============================ @Override public boolean equals(Object o) { @@ -260,4 +254,45 @@ public boolean equals(Object o) { && Objects.equals(privilegeChecker, that.privilegeChecker) && Objects.equals(identifier, that.identifier); } + + // ======================= copy ============================ + + @Override + public PrivilegedFileStoreTable copy(Map dynamicOptions) { + return new PrivilegedFileStoreTable( + wrapped.copy(dynamicOptions), privilegeChecker, identifier); + } + + @Override + public PrivilegedFileStoreTable copy(TableSchema newTableSchema) { + return new PrivilegedFileStoreTable( + wrapped.copy(newTableSchema), privilegeChecker, identifier); + } + + @Override + public PrivilegedFileStoreTable copyWithoutTimeTravel(Map dynamicOptions) { + return new PrivilegedFileStoreTable( + wrapped.copyWithoutTimeTravel(dynamicOptions), privilegeChecker, identifier); + } + + @Override + public PrivilegedFileStoreTable copyWithLatestSchema() { + return new PrivilegedFileStoreTable( + wrapped.copyWithLatestSchema(), privilegeChecker, identifier); + } + + @Override + public PrivilegedFileStoreTable switchToBranch(String branchName) { + return new PrivilegedFileStoreTable( + wrapped.switchToBranch(branchName), privilegeChecker, identifier); + } + + public static PrivilegedFileStoreTable wrap( + FileStoreTable table, PrivilegeChecker privilegeChecker, Identifier identifier) { + if (table instanceof ObjectTable) { + return new PrivilegedObjectTable((ObjectTable) table, privilegeChecker, identifier); + } else { + return new PrivilegedFileStoreTable(table, privilegeChecker, identifier); + } + } } diff --git a/paimon-core/src/main/java/org/apache/paimon/privilege/PrivilegedObjectTable.java b/paimon-core/src/main/java/org/apache/paimon/privilege/PrivilegedObjectTable.java new file mode 100644 index 000000000000..c5a319c1fedd --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/privilege/PrivilegedObjectTable.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.privilege; + +import org.apache.paimon.catalog.Identifier; +import org.apache.paimon.fs.FileIO; +import org.apache.paimon.schema.TableSchema; +import org.apache.paimon.table.FileStoreTable; +import org.apache.paimon.table.object.ObjectTable; + +import java.util.Map; + +/** A {@link PrivilegedFileStoreTable} for {@link ObjectTable}. */ +public class PrivilegedObjectTable extends PrivilegedFileStoreTable implements ObjectTable { + + private final ObjectTable objectTable; + + protected PrivilegedObjectTable( + ObjectTable wrapped, PrivilegeChecker privilegeChecker, Identifier identifier) { + super(wrapped, privilegeChecker, identifier); + this.objectTable = wrapped; + } + + @Override + public String objectLocation() { + return objectTable.objectLocation(); + } + + @Override + public FileStoreTable underlyingTable() { + return objectTable.underlyingTable(); + } + + @Override + public FileIO objectFileIO() { + return objectTable.objectFileIO(); + } + + @Override + public long refresh() { + privilegeChecker.assertCanInsert(identifier); + return objectTable.refresh(); + } + + // ======================= copy ============================ + + @Override + public PrivilegedObjectTable copy(Map dynamicOptions) { + return new PrivilegedObjectTable( + objectTable.copy(dynamicOptions), privilegeChecker, identifier); + } + + @Override + public PrivilegedObjectTable copy(TableSchema newTableSchema) { + return new PrivilegedObjectTable( + objectTable.copy(newTableSchema), privilegeChecker, identifier); + } + + @Override + public PrivilegedObjectTable copyWithoutTimeTravel(Map dynamicOptions) { + return new PrivilegedObjectTable( + objectTable.copyWithoutTimeTravel(dynamicOptions), privilegeChecker, identifier); + } + + @Override + public PrivilegedObjectTable copyWithLatestSchema() { + return new PrivilegedObjectTable( + objectTable.copyWithLatestSchema(), privilegeChecker, identifier); + } + + @Override + public PrivilegedObjectTable switchToBranch(String branchName) { + return new PrivilegedObjectTable( + objectTable.switchToBranch(branchName), privilegeChecker, identifier); + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/DefaultErrorHandler.java b/paimon-core/src/main/java/org/apache/paimon/rest/DefaultErrorHandler.java new file mode 100644 index 000000000000..ce2cbb56ae24 --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/rest/DefaultErrorHandler.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest; + +import org.apache.paimon.rest.exceptions.AlreadyExistsException; +import org.apache.paimon.rest.exceptions.BadRequestException; +import org.apache.paimon.rest.exceptions.ForbiddenException; +import org.apache.paimon.rest.exceptions.NoSuchResourceException; +import org.apache.paimon.rest.exceptions.NotAuthorizedException; +import org.apache.paimon.rest.exceptions.RESTException; +import org.apache.paimon.rest.exceptions.ServiceFailureException; +import org.apache.paimon.rest.exceptions.ServiceUnavailableException; +import org.apache.paimon.rest.responses.ErrorResponse; + +/** Default error handler. */ +public class DefaultErrorHandler extends ErrorHandler { + + private static final ErrorHandler INSTANCE = new DefaultErrorHandler(); + + public static ErrorHandler getInstance() { + return INSTANCE; + } + + @Override + public void accept(ErrorResponse error) { + int code = error.getCode(); + String message = error.getMessage(); + switch (code) { + case 400: + throw new BadRequestException(String.format("Malformed request: %s", message)); + case 401: + throw new NotAuthorizedException("Not authorized: %s", message); + case 403: + throw new ForbiddenException("Forbidden: %s", message); + case 404: + throw new NoSuchResourceException("%s", message); + case 405: + case 406: + break; + case 409: + throw new AlreadyExistsException("%s", message); + case 500: + throw new ServiceFailureException("Server error: %s", message); + case 501: + throw new UnsupportedOperationException(message); + case 503: + throw new ServiceUnavailableException("Service unavailable: %s", message); + default: + break; + } + + throw new RESTException("Unable to process: %s", message); + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/ErrorHandler.java b/paimon-core/src/main/java/org/apache/paimon/rest/ErrorHandler.java new file mode 100644 index 000000000000..cdfa4bcdfaac --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/rest/ErrorHandler.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest; + +import org.apache.paimon.rest.responses.ErrorResponse; + +import java.util.function.Consumer; + +/** Error handler for REST client. */ +public abstract class ErrorHandler implements Consumer {} diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/HttpClient.java b/paimon-core/src/main/java/org/apache/paimon/rest/HttpClient.java new file mode 100644 index 000000000000..87f3fad9b2fd --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/rest/HttpClient.java @@ -0,0 +1,146 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest; + +import org.apache.paimon.rest.exceptions.RESTException; +import org.apache.paimon.rest.responses.ErrorResponse; + +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.core.JsonProcessingException; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.databind.ObjectMapper; + +import okhttp3.Dispatcher; +import okhttp3.Headers; +import okhttp3.MediaType; +import okhttp3.OkHttpClient; +import okhttp3.Request; +import okhttp3.RequestBody; +import okhttp3.Response; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Map; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.SynchronousQueue; + +import static okhttp3.ConnectionSpec.CLEARTEXT; +import static okhttp3.ConnectionSpec.COMPATIBLE_TLS; +import static okhttp3.ConnectionSpec.MODERN_TLS; +import static org.apache.paimon.utils.ThreadPoolUtils.createCachedThreadPool; + +/** HTTP client for REST catalog. */ +public class HttpClient implements RESTClient { + + private final OkHttpClient okHttpClient; + private final String uri; + private final ObjectMapper mapper; + private final ErrorHandler errorHandler; + + private static final String THREAD_NAME = "REST-CATALOG-HTTP-CLIENT-THREAD-POOL"; + private static final MediaType MEDIA_TYPE = MediaType.parse("application/json"); + + public HttpClient(HttpClientOptions httpClientOptions) { + this.uri = httpClientOptions.uri(); + this.mapper = httpClientOptions.mapper(); + this.okHttpClient = createHttpClient(httpClientOptions); + this.errorHandler = httpClientOptions.errorHandler(); + } + + @Override + public T get( + String path, Class responseType, Map headers) { + Request request = + new Request.Builder().url(uri + path).get().headers(Headers.of(headers)).build(); + return exec(request, responseType); + } + + @Override + public T post( + String path, RESTRequest body, Class responseType, Map headers) { + try { + RequestBody requestBody = buildRequestBody(body); + Request request = + new Request.Builder() + .url(uri + path) + .post(requestBody) + .headers(Headers.of(headers)) + .build(); + return exec(request, responseType); + } catch (JsonProcessingException e) { + throw new RESTException(e, "build request failed."); + } + } + + @Override + public T delete(String path, Map headers) { + Request request = + new Request.Builder().url(uri + path).delete().headers(Headers.of(headers)).build(); + return exec(request, null); + } + + @Override + public void close() throws IOException { + okHttpClient.dispatcher().cancelAll(); + okHttpClient.connectionPool().evictAll(); + } + + private T exec(Request request, Class responseType) { + try (Response response = okHttpClient.newCall(request).execute()) { + String responseBodyStr = response.body() != null ? response.body().string() : null; + if (!response.isSuccessful()) { + ErrorResponse error = + new ErrorResponse( + responseBodyStr != null ? responseBodyStr : "response body is null", + response.code()); + errorHandler.accept(error); + } + if (responseType != null && responseBodyStr != null) { + return mapper.readValue(responseBodyStr, responseType); + } else if (responseType == null) { + return null; + } else { + throw new RESTException("response body is null."); + } + } catch (RESTException e) { + throw e; + } catch (Exception e) { + throw new RESTException(e, "rest exception"); + } + } + + private RequestBody buildRequestBody(RESTRequest body) throws JsonProcessingException { + return RequestBody.create(mapper.writeValueAsBytes(body), MEDIA_TYPE); + } + + private static OkHttpClient createHttpClient(HttpClientOptions httpClientOptions) { + BlockingQueue workQueue = new SynchronousQueue<>(); + ExecutorService executorService = + createCachedThreadPool(httpClientOptions.threadPoolSize(), THREAD_NAME, workQueue); + + OkHttpClient.Builder builder = + new OkHttpClient.Builder() + .dispatcher(new Dispatcher(executorService)) + .retryOnConnectionFailure(true) + .connectionSpecs(Arrays.asList(MODERN_TLS, COMPATIBLE_TLS, CLEARTEXT)); + httpClientOptions.connectTimeout().ifPresent(builder::connectTimeout); + httpClientOptions.readTimeout().ifPresent(builder::readTimeout); + + return builder.build(); + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/HttpClientOptions.java b/paimon-core/src/main/java/org/apache/paimon/rest/HttpClientOptions.java new file mode 100644 index 000000000000..694779cfdb86 --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/rest/HttpClientOptions.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest; + +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.databind.ObjectMapper; + +import java.time.Duration; +import java.util.Optional; + +/** Options for Http Client. */ +public class HttpClientOptions { + + private final String uri; + private final Optional connectTimeout; + private final Optional readTimeout; + private final ObjectMapper mapper; + private final int threadPoolSize; + private final ErrorHandler errorHandler; + + public HttpClientOptions( + String uri, + Optional connectTimeout, + Optional readTimeout, + ObjectMapper mapper, + int threadPoolSize, + ErrorHandler errorHandler) { + this.uri = uri; + this.connectTimeout = connectTimeout; + this.readTimeout = readTimeout; + this.mapper = mapper; + this.threadPoolSize = threadPoolSize; + this.errorHandler = errorHandler; + } + + public String uri() { + return uri; + } + + public Optional connectTimeout() { + return connectTimeout; + } + + public Optional readTimeout() { + return readTimeout; + } + + public ObjectMapper mapper() { + return mapper; + } + + public int threadPoolSize() { + return threadPoolSize; + } + + public ErrorHandler errorHandler() { + return errorHandler; + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java new file mode 100644 index 000000000000..03b257efbf86 --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java @@ -0,0 +1,273 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest; + +import org.apache.paimon.catalog.Catalog; +import org.apache.paimon.catalog.Database; +import org.apache.paimon.catalog.Identifier; +import org.apache.paimon.fs.FileIO; +import org.apache.paimon.fs.Path; +import org.apache.paimon.manifest.PartitionEntry; +import org.apache.paimon.options.CatalogOptions; +import org.apache.paimon.options.Options; +import org.apache.paimon.rest.auth.AuthSession; +import org.apache.paimon.rest.auth.CredentialsProvider; +import org.apache.paimon.rest.auth.CredentialsProviderFactory; +import org.apache.paimon.rest.exceptions.AlreadyExistsException; +import org.apache.paimon.rest.exceptions.NoSuchResourceException; +import org.apache.paimon.rest.requests.CreateDatabaseRequest; +import org.apache.paimon.rest.responses.ConfigResponse; +import org.apache.paimon.rest.responses.CreateDatabaseResponse; +import org.apache.paimon.rest.responses.DatabaseName; +import org.apache.paimon.rest.responses.GetDatabaseResponse; +import org.apache.paimon.rest.responses.ListDatabasesResponse; +import org.apache.paimon.schema.Schema; +import org.apache.paimon.schema.SchemaChange; +import org.apache.paimon.table.Table; + +import org.apache.paimon.shade.guava30.com.google.common.annotations.VisibleForTesting; +import org.apache.paimon.shade.guava30.com.google.common.collect.ImmutableList; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.databind.ObjectMapper; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.ScheduledExecutorService; +import java.util.stream.Collectors; + +import static org.apache.paimon.utils.ThreadPoolUtils.createScheduledThreadPool; + +/** A catalog implementation for REST. */ +public class RESTCatalog implements Catalog { + + private static final ObjectMapper OBJECT_MAPPER = RESTObjectMapper.create(); + + private final RESTClient client; + private final ResourcePaths resourcePaths; + private final Map options; + private final Map baseHeader; + private final AuthSession catalogAuth; + + private volatile ScheduledExecutorService refreshExecutor = null; + + public RESTCatalog(Options options) { + if (options.getOptional(CatalogOptions.WAREHOUSE).isPresent()) { + throw new IllegalArgumentException("Can not config warehouse in RESTCatalog."); + } + String uri = options.get(RESTCatalogOptions.URI); + Optional connectTimeout = + options.getOptional(RESTCatalogOptions.CONNECTION_TIMEOUT); + Optional readTimeout = options.getOptional(RESTCatalogOptions.READ_TIMEOUT); + Integer threadPoolSize = options.get(RESTCatalogOptions.THREAD_POOL_SIZE); + HttpClientOptions httpClientOptions = + new HttpClientOptions( + uri, + connectTimeout, + readTimeout, + OBJECT_MAPPER, + threadPoolSize, + DefaultErrorHandler.getInstance()); + this.client = new HttpClient(httpClientOptions); + this.baseHeader = configHeaders(options.toMap()); + CredentialsProvider credentialsProvider = + CredentialsProviderFactory.createCredentialsProvider( + options, RESTCatalog.class.getClassLoader()); + if (credentialsProvider.keepRefreshed()) { + this.catalogAuth = + AuthSession.fromRefreshCredentialsProvider( + tokenRefreshExecutor(), this.baseHeader, credentialsProvider); + + } else { + this.catalogAuth = new AuthSession(this.baseHeader, credentialsProvider); + } + Map initHeaders = + RESTUtil.merge(configHeaders(options.toMap()), this.catalogAuth.getHeaders()); + this.options = fetchOptionsFromServer(initHeaders, options.toMap()); + this.resourcePaths = + ResourcePaths.forCatalogProperties( + this.options.get(RESTCatalogInternalOptions.PREFIX)); + } + + @Override + public String warehouse() { + throw new UnsupportedOperationException(); + } + + @Override + public Map options() { + return this.options; + } + + @Override + public FileIO fileIO() { + throw new UnsupportedOperationException(); + } + + @Override + public List listDatabases() { + ListDatabasesResponse response = + client.get(resourcePaths.databases(), ListDatabasesResponse.class, headers()); + if (response.getDatabases() != null) { + return response.getDatabases().stream() + .map(DatabaseName::getName) + .collect(Collectors.toList()); + } + return ImmutableList.of(); + } + + @Override + public void createDatabase(String name, boolean ignoreIfExists, Map properties) + throws DatabaseAlreadyExistException { + CreateDatabaseRequest request = new CreateDatabaseRequest(name, ignoreIfExists, properties); + try { + client.post( + resourcePaths.databases(), request, CreateDatabaseResponse.class, headers()); + } catch (AlreadyExistsException e) { + throw new DatabaseAlreadyExistException(name); + } + } + + @Override + public Database getDatabase(String name) throws DatabaseNotExistException { + try { + GetDatabaseResponse response = + client.get(resourcePaths.database(name), GetDatabaseResponse.class, headers()); + return new Database.DatabaseImpl( + name, response.options(), response.comment().orElseGet(() -> null)); + } catch (NoSuchResourceException e) { + throw new DatabaseNotExistException(name); + } + } + + @Override + public void dropDatabase(String name, boolean ignoreIfNotExists, boolean cascade) + throws DatabaseNotExistException, DatabaseNotEmptyException { + try { + if (!cascade && !this.listTables(name).isEmpty()) { + throw new DatabaseNotEmptyException(name); + } + client.delete(resourcePaths.database(name), headers()); + } catch (NoSuchResourceException e) { + if (!ignoreIfNotExists) { + throw new DatabaseNotExistException(name); + } + } + } + + @Override + public Table getTable(Identifier identifier) throws TableNotExistException { + throw new UnsupportedOperationException(); + } + + @Override + public Path getTableLocation(Identifier identifier) { + throw new UnsupportedOperationException(); + } + + @Override + public List listTables(String databaseName) throws DatabaseNotExistException { + return new ArrayList(); + } + + @Override + public void dropTable(Identifier identifier, boolean ignoreIfNotExists) + throws TableNotExistException { + throw new UnsupportedOperationException(); + } + + @Override + public void createTable(Identifier identifier, Schema schema, boolean ignoreIfExists) + throws TableAlreadyExistException, DatabaseNotExistException { + throw new UnsupportedOperationException(); + } + + @Override + public void renameTable(Identifier fromTable, Identifier toTable, boolean ignoreIfNotExists) + throws TableNotExistException, TableAlreadyExistException { + throw new UnsupportedOperationException(); + } + + @Override + public void alterTable( + Identifier identifier, List changes, boolean ignoreIfNotExists) + throws TableNotExistException, ColumnAlreadyExistException, ColumnNotExistException { + throw new UnsupportedOperationException(); + } + + @Override + public void createPartition(Identifier identifier, Map partitionSpec) + throws TableNotExistException { + throw new UnsupportedOperationException(); + } + + @Override + public void dropPartition(Identifier identifier, Map partitions) + throws TableNotExistException, PartitionNotExistException {} + + @Override + public List listPartitions(Identifier identifier) + throws TableNotExistException { + throw new UnsupportedOperationException(); + } + + @Override + public boolean allowUpperCase() { + return false; + } + + @Override + public void close() throws Exception { + if (refreshExecutor != null) { + refreshExecutor.shutdownNow(); + } + if (client != null) { + client.close(); + } + } + + @VisibleForTesting + Map fetchOptionsFromServer( + Map headers, Map clientProperties) { + ConfigResponse response = + client.get(ResourcePaths.V1_CONFIG, ConfigResponse.class, headers); + return response.merge(clientProperties); + } + + private static Map configHeaders(Map properties) { + return RESTUtil.extractPrefixMap(properties, "header."); + } + + private Map headers() { + return catalogAuth.getHeaders(); + } + + private ScheduledExecutorService tokenRefreshExecutor() { + if (refreshExecutor == null) { + synchronized (this) { + if (refreshExecutor == null) { + this.refreshExecutor = createScheduledThreadPool(1, "token-refresh-thread"); + } + } + } + + return refreshExecutor; + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalogFactory.java b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalogFactory.java new file mode 100644 index 000000000000..a5c773cb4bd5 --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalogFactory.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest; + +import org.apache.paimon.catalog.Catalog; +import org.apache.paimon.catalog.CatalogContext; +import org.apache.paimon.catalog.CatalogFactory; + +/** Factory to create {@link RESTCatalog}. */ +public class RESTCatalogFactory implements CatalogFactory { + public static final String IDENTIFIER = "rest"; + + @Override + public String identifier() { + return IDENTIFIER; + } + + @Override + public Catalog create(CatalogContext context) { + return new RESTCatalog(context.options()); + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalogInternalOptions.java b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalogInternalOptions.java new file mode 100644 index 000000000000..722010923c46 --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalogInternalOptions.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest; + +import org.apache.paimon.options.ConfigOption; +import org.apache.paimon.options.ConfigOptions; + +/** Internal options for REST Catalog. */ +public class RESTCatalogInternalOptions { + public static final ConfigOption PREFIX = + ConfigOptions.key("prefix") + .stringType() + .noDefaultValue() + .withDescription("REST Catalog uri's prefix."); + public static final ConfigOption CREDENTIALS_PROVIDER = + ConfigOptions.key("credentials-provider") + .stringType() + .noDefaultValue() + .withDescription("REST Catalog auth credentials provider."); + public static final ConfigOption DATABASE_COMMENT = + ConfigOptions.key("comment") + .stringType() + .defaultValue(null) + .withDescription("REST Catalog database comment."); +} diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalogOptions.java b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalogOptions.java new file mode 100644 index 000000000000..1af64def4f71 --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalogOptions.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest; + +import org.apache.paimon.options.ConfigOption; +import org.apache.paimon.options.ConfigOptions; + +import java.time.Duration; + +/** Options for REST Catalog. */ +public class RESTCatalogOptions { + + public static final ConfigOption URI = + ConfigOptions.key("uri") + .stringType() + .noDefaultValue() + .withDescription("REST Catalog server's uri."); + + public static final ConfigOption CONNECTION_TIMEOUT = + ConfigOptions.key("rest.client.connection-timeout") + .durationType() + .noDefaultValue() + .withDescription("REST Catalog http client connect timeout."); + + public static final ConfigOption READ_TIMEOUT = + ConfigOptions.key("rest.client.read-timeout") + .durationType() + .noDefaultValue() + .withDescription("REST Catalog http client read timeout."); + + public static final ConfigOption THREAD_POOL_SIZE = + ConfigOptions.key("rest.client.num-threads") + .intType() + .defaultValue(1) + .withDescription("REST Catalog http client thread num."); + + public static final ConfigOption TOKEN = + ConfigOptions.key("token") + .stringType() + .noDefaultValue() + .withDescription("REST Catalog auth token."); + + public static final ConfigOption TOKEN_EXPIRATION_TIME = + ConfigOptions.key("token.expiration-time") + .durationType() + .defaultValue(Duration.ofHours(1)) + .withDescription( + "REST Catalog auth token expires time.The token generates system refresh frequency is t1," + + " the token expires time is t2, we need to guarantee that t2 > t1," + + " the token validity time is [t2 - t1, t2]," + + " and the expires time defined here needs to be less than (t2 - t1)"); + + public static final ConfigOption TOKEN_PROVIDER_PATH = + ConfigOptions.key("token.provider.path") + .stringType() + .noDefaultValue() + .withDescription("REST Catalog auth token provider path."); +} diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/RESTClient.java b/paimon-core/src/main/java/org/apache/paimon/rest/RESTClient.java new file mode 100644 index 000000000000..a255d688bc52 --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/rest/RESTClient.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest; + +import java.io.Closeable; +import java.util.Map; + +/** Interface for a basic HTTP Client for interfacing with the REST catalog. */ +public interface RESTClient extends Closeable { + + T get(String path, Class responseType, Map headers); + + T post( + String path, RESTRequest body, Class responseType, Map headers); + + T delete(String path, Map headers); +} diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/RESTMessage.java b/paimon-core/src/main/java/org/apache/paimon/rest/RESTMessage.java new file mode 100644 index 000000000000..31d46df7ef0f --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/rest/RESTMessage.java @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest; + +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonIgnoreProperties; + +/** Interface to mark both REST requests and responses. */ +@JsonIgnoreProperties(ignoreUnknown = true) +public interface RESTMessage {} diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/RESTObjectMapper.java b/paimon-core/src/main/java/org/apache/paimon/rest/RESTObjectMapper.java new file mode 100644 index 000000000000..b1c83e90224a --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/rest/RESTObjectMapper.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest; + +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.databind.DeserializationFeature; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.databind.SerializationFeature; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; + +/** Object mapper for REST request and response. */ +public class RESTObjectMapper { + public static ObjectMapper create() { + ObjectMapper mapper = new ObjectMapper(); + mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + mapper.configure(SerializationFeature.FAIL_ON_EMPTY_BEANS, false); + mapper.registerModule(new JavaTimeModule()); + return mapper; + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/RESTRequest.java b/paimon-core/src/main/java/org/apache/paimon/rest/RESTRequest.java new file mode 100644 index 000000000000..9c6758df14f0 --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/rest/RESTRequest.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest; + +/** Interface to mark a REST request. */ +public interface RESTRequest extends RESTMessage {} diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/RESTResponse.java b/paimon-core/src/main/java/org/apache/paimon/rest/RESTResponse.java new file mode 100644 index 000000000000..a4149d3fda14 --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/rest/RESTResponse.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest; + +/** Interface to mark a REST response. */ +public interface RESTResponse extends RESTMessage {} diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/RESTUtil.java b/paimon-core/src/main/java/org/apache/paimon/rest/RESTUtil.java new file mode 100644 index 000000000000..3d42e99fa6d5 --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/rest/RESTUtil.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest; + +import org.apache.paimon.utils.Preconditions; + +import org.apache.paimon.shade.guava30.com.google.common.collect.ImmutableMap; +import org.apache.paimon.shade.guava30.com.google.common.collect.Maps; + +import java.util.Map; + +/** Util for REST. */ +public class RESTUtil { + public static Map extractPrefixMap( + Map properties, String prefix) { + Preconditions.checkNotNull(properties, "Invalid properties map: null"); + Map result = Maps.newHashMap(); + for (Map.Entry entry : properties.entrySet()) { + if (entry.getKey() != null && entry.getKey().startsWith(prefix)) { + result.put( + entry.getKey().substring(prefix.length()), properties.get(entry.getKey())); + } + } + return result; + } + + public static Map merge( + Map target, Map updates) { + ImmutableMap.Builder builder = ImmutableMap.builder(); + for (Map.Entry entry : target.entrySet()) { + if (!updates.containsKey(entry.getKey())) { + builder.put(entry.getKey(), entry.getValue()); + } + } + updates.forEach(builder::put); + + return builder.build(); + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/ResourcePaths.java b/paimon-core/src/main/java/org/apache/paimon/rest/ResourcePaths.java new file mode 100644 index 000000000000..b58053374daa --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/rest/ResourcePaths.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest; + +import java.util.StringJoiner; + +/** Resource paths for REST catalog. */ +public class ResourcePaths { + + public static final String V1_CONFIG = "/v1/config"; + private static final StringJoiner SLASH = new StringJoiner("/"); + + public static ResourcePaths forCatalogProperties(String prefix) { + return new ResourcePaths(prefix); + } + + private final String prefix; + + public ResourcePaths(String prefix) { + this.prefix = prefix; + } + + public String databases() { + return SLASH.add("v1").add(prefix).add("databases").toString(); + } + + public String database(String databaseName) { + return SLASH.add("v1").add(prefix).add("databases").add(databaseName).toString(); + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/auth/AuthSession.java b/paimon-core/src/main/java/org/apache/paimon/rest/auth/AuthSession.java new file mode 100644 index 000000000000..3ca7590e5f96 --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/rest/auth/AuthSession.java @@ -0,0 +1,143 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest.auth; + +import org.apache.paimon.annotation.VisibleForTesting; +import org.apache.paimon.rest.RESTUtil; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +/** Auth session. */ +public class AuthSession { + + static final int TOKEN_REFRESH_NUM_RETRIES = 5; + static final long MIN_REFRESH_WAIT_MILLIS = 10; + static final long MAX_REFRESH_WINDOW_MILLIS = 300_000; // 5 minutes + + private static final Logger log = LoggerFactory.getLogger(AuthSession.class); + private final CredentialsProvider credentialsProvider; + private volatile Map headers; + + public AuthSession(Map headers, CredentialsProvider credentialsProvider) { + this.headers = headers; + this.credentialsProvider = credentialsProvider; + } + + public static AuthSession fromRefreshCredentialsProvider( + ScheduledExecutorService executor, + Map headers, + CredentialsProvider credentialsProvider) { + AuthSession session = new AuthSession(headers, credentialsProvider); + + long startTimeMillis = System.currentTimeMillis(); + Optional expiresAtMillisOpt = credentialsProvider.expiresAtMillis(); + + // when init session if credentials expire time is in the past, refresh it and update + // expiresAtMillis + if (expiresAtMillisOpt.isPresent() && expiresAtMillisOpt.get() <= startTimeMillis) { + boolean refreshSuccessful = session.refresh(); + if (refreshSuccessful) { + expiresAtMillisOpt = session.credentialsProvider.expiresAtMillis(); + } + } + + if (null != executor && expiresAtMillisOpt.isPresent()) { + scheduleTokenRefresh(executor, session, expiresAtMillisOpt.get()); + } + + return session; + } + + public Map getHeaders() { + if (this.credentialsProvider.keepRefreshed() && this.credentialsProvider.willSoonExpire()) { + refresh(); + } + return headers; + } + + public Boolean refresh() { + if (this.credentialsProvider.supportRefresh() + && this.credentialsProvider.keepRefreshed() + && this.credentialsProvider.expiresInMills().isPresent()) { + boolean isSuccessful = this.credentialsProvider.refresh(); + if (isSuccessful) { + Map currentHeaders = this.headers; + this.headers = + RESTUtil.merge(currentHeaders, this.credentialsProvider.authHeader()); + } + return isSuccessful; + } + + return false; + } + + @VisibleForTesting + static void scheduleTokenRefresh( + ScheduledExecutorService executor, AuthSession session, long expiresAtMillis) { + scheduleTokenRefresh(executor, session, expiresAtMillis, 0); + } + + @VisibleForTesting + static long getTimeToWaitByExpiresInMills(long expiresInMillis) { + // how much ahead of time to start the refresh to allow it to complete + long refreshWindowMillis = Math.min(expiresInMillis, MAX_REFRESH_WINDOW_MILLIS); + // how much time to wait before expiration + long waitIntervalMillis = expiresInMillis - refreshWindowMillis; + // how much time to actually wait + return Math.max(waitIntervalMillis, MIN_REFRESH_WAIT_MILLIS); + } + + private static void scheduleTokenRefresh( + ScheduledExecutorService executor, + AuthSession session, + long expiresAtMillis, + int retryTimes) { + if (retryTimes < TOKEN_REFRESH_NUM_RETRIES) { + long expiresInMillis = expiresAtMillis - System.currentTimeMillis(); + long timeToWait = getTimeToWaitByExpiresInMills(expiresInMillis); + + executor.schedule( + () -> { + long refreshStartTime = System.currentTimeMillis(); + boolean isSuccessful = session.refresh(); + if (isSuccessful) { + scheduleTokenRefresh( + executor, + session, + refreshStartTime + + session.credentialsProvider.expiresInMills().get(), + 0); + } else { + scheduleTokenRefresh( + executor, session, expiresAtMillis, retryTimes + 1); + } + }, + timeToWait, + TimeUnit.MILLISECONDS); + } else { + log.warn("Failed to refresh token after {} retries.", TOKEN_REFRESH_NUM_RETRIES); + } + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/auth/BaseBearTokenCredentialsProvider.java b/paimon-core/src/main/java/org/apache/paimon/rest/auth/BaseBearTokenCredentialsProvider.java new file mode 100644 index 000000000000..d3df87826164 --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/rest/auth/BaseBearTokenCredentialsProvider.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest.auth; + +import org.apache.paimon.shade.guava30.com.google.common.collect.ImmutableMap; + +import java.util.Map; + +/** Base bear token credentials provider. */ +public abstract class BaseBearTokenCredentialsProvider implements CredentialsProvider { + + private static final String AUTHORIZATION_HEADER = "Authorization"; + private static final String BEARER_PREFIX = "Bearer "; + + @Override + public Map authHeader() { + return ImmutableMap.of(AUTHORIZATION_HEADER, BEARER_PREFIX + token()); + } + + abstract String token(); +} diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/auth/BearTokenCredentialsProvider.java b/paimon-core/src/main/java/org/apache/paimon/rest/auth/BearTokenCredentialsProvider.java new file mode 100644 index 000000000000..89228fe10b28 --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/rest/auth/BearTokenCredentialsProvider.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest.auth; + +/** credentials provider for bear token. */ +public class BearTokenCredentialsProvider extends BaseBearTokenCredentialsProvider { + + private final String token; + + public BearTokenCredentialsProvider(String token) { + this.token = token; + } + + @Override + String token() { + return this.token; + } + + @Override + public boolean refresh() { + return true; + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/auth/BearTokenCredentialsProviderFactory.java b/paimon-core/src/main/java/org/apache/paimon/rest/auth/BearTokenCredentialsProviderFactory.java new file mode 100644 index 000000000000..e63ac5606b01 --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/rest/auth/BearTokenCredentialsProviderFactory.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest.auth; + +import org.apache.paimon.options.Options; +import org.apache.paimon.rest.RESTCatalogOptions; +import org.apache.paimon.utils.StringUtils; + +/** factory for create {@link BearTokenCredentialsProvider}. */ +public class BearTokenCredentialsProviderFactory implements CredentialsProviderFactory { + + @Override + public String identifier() { + return CredentialsProviderType.BEAR_TOKEN.name(); + } + + @Override + public CredentialsProvider create(Options options) { + if (options.getOptional(RESTCatalogOptions.TOKEN) + .map(StringUtils::isNullOrWhitespaceOnly) + .orElse(true)) { + throw new IllegalArgumentException( + RESTCatalogOptions.TOKEN.key() + " is required and not empty"); + } + return new BearTokenCredentialsProvider(options.get(RESTCatalogOptions.TOKEN)); + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/auth/BearTokenFileCredentialsProvider.java b/paimon-core/src/main/java/org/apache/paimon/rest/auth/BearTokenFileCredentialsProvider.java new file mode 100644 index 000000000000..d479caa67fd0 --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/rest/auth/BearTokenFileCredentialsProvider.java @@ -0,0 +1,106 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest.auth; + +import org.apache.paimon.utils.FileIOUtils; +import org.apache.paimon.utils.StringUtils; + +import java.io.File; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.Optional; + +/** credentials provider for get bear token from file. */ +public class BearTokenFileCredentialsProvider extends BaseBearTokenCredentialsProvider { + + public static final double EXPIRED_FACTOR = 0.4; + + private final String tokenFilePath; + private String token; + private boolean keepRefreshed = false; + private Long expiresAtMillis = null; + private Long expiresInMills = null; + + public BearTokenFileCredentialsProvider(String tokenFilePath) { + this.tokenFilePath = tokenFilePath; + this.token = getTokenFromFile(); + } + + public BearTokenFileCredentialsProvider(String tokenFilePath, Long expiresInMills) { + this(tokenFilePath); + this.keepRefreshed = true; + this.expiresAtMillis = -1L; + this.expiresInMills = expiresInMills; + } + + @Override + String token() { + return this.token; + } + + @Override + public boolean refresh() { + long start = System.currentTimeMillis(); + String newToken = getTokenFromFile(); + if (StringUtils.isNullOrWhitespaceOnly(newToken)) { + return false; + } + this.expiresAtMillis = start + this.expiresInMills; + this.token = newToken; + return true; + } + + @Override + public boolean supportRefresh() { + return true; + } + + @Override + public boolean keepRefreshed() { + return this.keepRefreshed; + } + + @Override + public boolean willSoonExpire() { + if (keepRefreshed()) { + return expiresAtMillis().get() - System.currentTimeMillis() + < expiresInMills().get() * EXPIRED_FACTOR; + } else { + return false; + } + } + + @Override + public Optional expiresAtMillis() { + return Optional.ofNullable(this.expiresAtMillis); + } + + @Override + public Optional expiresInMills() { + return Optional.ofNullable(this.expiresInMills); + } + + private String getTokenFromFile() { + try { + return FileIOUtils.readFileUtf8(new File(tokenFilePath)); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/auth/BearTokenFileCredentialsProviderFactory.java b/paimon-core/src/main/java/org/apache/paimon/rest/auth/BearTokenFileCredentialsProviderFactory.java new file mode 100644 index 000000000000..a0fa6b405d62 --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/rest/auth/BearTokenFileCredentialsProviderFactory.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest.auth; + +import org.apache.paimon.options.Options; + +import static org.apache.paimon.rest.RESTCatalogOptions.TOKEN_EXPIRATION_TIME; +import static org.apache.paimon.rest.RESTCatalogOptions.TOKEN_PROVIDER_PATH; + +/** factory for create {@link BearTokenCredentialsProvider}. */ +public class BearTokenFileCredentialsProviderFactory implements CredentialsProviderFactory { + + @Override + public String identifier() { + return CredentialsProviderType.BEAR_TOKEN_FILE.name(); + } + + @Override + public CredentialsProvider create(Options options) { + if (!options.getOptional(TOKEN_PROVIDER_PATH).isPresent()) { + throw new IllegalArgumentException(TOKEN_PROVIDER_PATH.key() + " is required"); + } + String tokenFilePath = options.get(TOKEN_PROVIDER_PATH); + if (options.getOptional(TOKEN_EXPIRATION_TIME).isPresent()) { + long tokenExpireInMills = options.get(TOKEN_EXPIRATION_TIME).toMillis(); + return new BearTokenFileCredentialsProvider(tokenFilePath, tokenExpireInMills); + + } else { + return new BearTokenFileCredentialsProvider(tokenFilePath); + } + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/auth/CredentialsProvider.java b/paimon-core/src/main/java/org/apache/paimon/rest/auth/CredentialsProvider.java new file mode 100644 index 000000000000..7fe8008e5947 --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/rest/auth/CredentialsProvider.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest.auth; + +import java.util.Map; +import java.util.Optional; + +/** Credentials provider. */ +public interface CredentialsProvider { + + Map authHeader(); + + boolean refresh(); + + default boolean supportRefresh() { + return false; + } + + default boolean keepRefreshed() { + return false; + } + + default boolean willSoonExpire() { + return false; + } + + default Optional expiresAtMillis() { + return Optional.empty(); + } + + default Optional expiresInMills() { + return Optional.empty(); + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/auth/CredentialsProviderFactory.java b/paimon-core/src/main/java/org/apache/paimon/rest/auth/CredentialsProviderFactory.java new file mode 100644 index 000000000000..50c3564ad8c6 --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/rest/auth/CredentialsProviderFactory.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest.auth; + +import org.apache.paimon.factories.Factory; +import org.apache.paimon.factories.FactoryUtil; +import org.apache.paimon.options.Options; +import org.apache.paimon.rest.RESTCatalogOptions; + +import static org.apache.paimon.rest.RESTCatalogInternalOptions.CREDENTIALS_PROVIDER; + +/** Factory for creating {@link CredentialsProvider}. */ +public interface CredentialsProviderFactory extends Factory { + + default CredentialsProvider create(Options options) { + throw new UnsupportedOperationException( + "Use create(context) for " + this.getClass().getSimpleName()); + } + + static CredentialsProvider createCredentialsProvider(Options options, ClassLoader classLoader) { + String credentialsProviderIdentifier = getCredentialsProviderTypeByConf(options).name(); + CredentialsProviderFactory credentialsProviderFactory = + FactoryUtil.discoverFactory( + classLoader, + CredentialsProviderFactory.class, + credentialsProviderIdentifier); + return credentialsProviderFactory.create(options); + } + + static CredentialsProviderType getCredentialsProviderTypeByConf(Options options) { + if (options.getOptional(CREDENTIALS_PROVIDER).isPresent()) { + return CredentialsProviderType.valueOf(options.get(CREDENTIALS_PROVIDER)); + } else if (options.getOptional(RESTCatalogOptions.TOKEN_PROVIDER_PATH).isPresent()) { + return CredentialsProviderType.BEAR_TOKEN_FILE; + } + return CredentialsProviderType.BEAR_TOKEN; + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/auth/CredentialsProviderType.java b/paimon-core/src/main/java/org/apache/paimon/rest/auth/CredentialsProviderType.java new file mode 100644 index 000000000000..28c344d70eee --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/rest/auth/CredentialsProviderType.java @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest.auth; + +/** Credentials provider type. */ +public enum CredentialsProviderType { + BEAR_TOKEN, + BEAR_TOKEN_FILE +} diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/exceptions/AlreadyExistsException.java b/paimon-core/src/main/java/org/apache/paimon/rest/exceptions/AlreadyExistsException.java new file mode 100644 index 000000000000..8e30c8375bf9 --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/rest/exceptions/AlreadyExistsException.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest.exceptions; + +/** Exception thrown on HTTP 409 means a resource already exists. */ +public class AlreadyExistsException extends RESTException { + + public AlreadyExistsException(String message, Object... args) { + super(message, args); + } +} diff --git a/paimon-spark/paimon-spark-3.2/src/test/scala/org/apache/spark/paimon/Utils.scala b/paimon-core/src/main/java/org/apache/paimon/rest/exceptions/BadRequestException.java similarity index 74% rename from paimon-spark/paimon-spark-3.2/src/test/scala/org/apache/spark/paimon/Utils.scala rename to paimon-core/src/main/java/org/apache/paimon/rest/exceptions/BadRequestException.java index 1a899f500153..301f3bd63f88 100644 --- a/paimon-spark/paimon-spark-3.2/src/test/scala/org/apache/spark/paimon/Utils.scala +++ b/paimon-core/src/main/java/org/apache/paimon/rest/exceptions/BadRequestException.java @@ -16,17 +16,12 @@ * limitations under the License. */ -package org.apache.spark.paimon +package org.apache.paimon.rest.exceptions; -import org.apache.spark.util.{Utils => SparkUtils} - -import java.io.File - -/** - * A wrapper that some Objects or Classes is limited to access beyond [[org.apache.spark]] package. - */ -object Utils { - - def createTempDir: File = SparkUtils.createTempDir() +/** Exception thrown on HTTP 400 - Bad Request. */ +public class BadRequestException extends RESTException { + public BadRequestException(String message, Object... args) { + super(message, args); + } } diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/exceptions/ForbiddenException.java b/paimon-core/src/main/java/org/apache/paimon/rest/exceptions/ForbiddenException.java new file mode 100644 index 000000000000..3982e5b70417 --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/rest/exceptions/ForbiddenException.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest.exceptions; + +/** Exception thrown on HTTP 403 Forbidden. */ +public class ForbiddenException extends RESTException { + public ForbiddenException(String message, Object... args) { + super(message, args); + } +} diff --git a/paimon-spark/paimon-spark-3.3/src/test/scala/org/apache/spark/paimon/Utils.scala b/paimon-core/src/main/java/org/apache/paimon/rest/exceptions/NoSuchResourceException.java similarity index 74% rename from paimon-spark/paimon-spark-3.3/src/test/scala/org/apache/spark/paimon/Utils.scala rename to paimon-core/src/main/java/org/apache/paimon/rest/exceptions/NoSuchResourceException.java index 1a899f500153..cc4c7881f465 100644 --- a/paimon-spark/paimon-spark-3.3/src/test/scala/org/apache/spark/paimon/Utils.scala +++ b/paimon-core/src/main/java/org/apache/paimon/rest/exceptions/NoSuchResourceException.java @@ -16,17 +16,12 @@ * limitations under the License. */ -package org.apache.spark.paimon +package org.apache.paimon.rest.exceptions; -import org.apache.spark.util.{Utils => SparkUtils} - -import java.io.File - -/** - * A wrapper that some Objects or Classes is limited to access beyond [[org.apache.spark]] package. - */ -object Utils { - - def createTempDir: File = SparkUtils.createTempDir() +/** Exception thrown on HTTP 404 means a resource not exists. */ +public class NoSuchResourceException extends RESTException { + public NoSuchResourceException(String message, Object... args) { + super(message, args); + } } diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/exceptions/NotAuthorizedException.java b/paimon-core/src/main/java/org/apache/paimon/rest/exceptions/NotAuthorizedException.java new file mode 100644 index 000000000000..43c13b1a1c97 --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/rest/exceptions/NotAuthorizedException.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest.exceptions; + +/** Exception thrown on HTTP 401 Unauthorized. */ +public class NotAuthorizedException extends RESTException { + public NotAuthorizedException(String message, Object... args) { + super(String.format(message, args)); + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/exceptions/RESTException.java b/paimon-core/src/main/java/org/apache/paimon/rest/exceptions/RESTException.java new file mode 100644 index 000000000000..532936f43032 --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/rest/exceptions/RESTException.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest.exceptions; + +/** Base class for REST client exceptions. */ +public class RESTException extends RuntimeException { + public RESTException(String message, Object... args) { + super(String.format(message, args)); + } + + public RESTException(Throwable cause, String message, Object... args) { + super(String.format(message, args), cause); + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/exceptions/ServiceFailureException.java b/paimon-core/src/main/java/org/apache/paimon/rest/exceptions/ServiceFailureException.java new file mode 100644 index 000000000000..45c48ec0de09 --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/rest/exceptions/ServiceFailureException.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest.exceptions; + +/** Exception thrown on HTTP 500 - Bad Request. */ +public class ServiceFailureException extends RESTException { + public ServiceFailureException(String message, Object... args) { + super(String.format(message, args)); + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/exceptions/ServiceUnavailableException.java b/paimon-core/src/main/java/org/apache/paimon/rest/exceptions/ServiceUnavailableException.java new file mode 100644 index 000000000000..fb6a05e89f9f --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/rest/exceptions/ServiceUnavailableException.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest.exceptions; + +/** Exception thrown on HTTP 503 - service is unavailable. */ +public class ServiceUnavailableException extends RESTException { + public ServiceUnavailableException(String message, Object... args) { + super(String.format(message, args)); + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/requests/CreateDatabaseRequest.java b/paimon-core/src/main/java/org/apache/paimon/rest/requests/CreateDatabaseRequest.java new file mode 100644 index 000000000000..6067bf544b87 --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/rest/requests/CreateDatabaseRequest.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest.requests; + +import org.apache.paimon.rest.RESTRequest; + +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonGetter; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.Map; + +/** Request for creating database. */ +public class CreateDatabaseRequest implements RESTRequest { + + private static final String FIELD_NAME = "name"; + private static final String FIELD_IGNORE_IF_EXISTS = "ignoreIfExists"; + private static final String FIELD_OPTIONS = "options"; + + @JsonProperty(FIELD_NAME) + private String name; + + @JsonProperty(FIELD_IGNORE_IF_EXISTS) + private boolean ignoreIfExists; + + @JsonProperty(FIELD_OPTIONS) + private Map options; + + @JsonCreator + public CreateDatabaseRequest( + @JsonProperty(FIELD_NAME) String name, + @JsonProperty(FIELD_IGNORE_IF_EXISTS) boolean ignoreIfExists, + @JsonProperty(FIELD_OPTIONS) Map options) { + this.name = name; + this.ignoreIfExists = ignoreIfExists; + this.options = options; + } + + @JsonGetter(FIELD_NAME) + public String getName() { + return name; + } + + @JsonGetter(FIELD_IGNORE_IF_EXISTS) + public boolean getIgnoreIfExists() { + return ignoreIfExists; + } + + @JsonGetter(FIELD_OPTIONS) + public Map getOptions() { + return options; + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/responses/ConfigResponse.java b/paimon-core/src/main/java/org/apache/paimon/rest/responses/ConfigResponse.java new file mode 100644 index 000000000000..e8fff88b09c2 --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/rest/responses/ConfigResponse.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest.responses; + +import org.apache.paimon.rest.RESTResponse; +import org.apache.paimon.utils.Preconditions; + +import org.apache.paimon.shade.guava30.com.google.common.collect.ImmutableMap; +import org.apache.paimon.shade.guava30.com.google.common.collect.Maps; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonGetter; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.Map; +import java.util.Objects; + +/** Response for getting config. */ +public class ConfigResponse implements RESTResponse { + + private static final String FIELD_DEFAULTS = "defaults"; + private static final String FIELD_OVERRIDES = "overrides"; + + @JsonProperty(FIELD_DEFAULTS) + private Map defaults; + + @JsonProperty(FIELD_OVERRIDES) + private Map overrides; + + @JsonCreator + public ConfigResponse( + @JsonProperty(FIELD_DEFAULTS) Map defaults, + @JsonProperty(FIELD_OVERRIDES) Map overrides) { + this.defaults = defaults; + this.overrides = overrides; + } + + public Map merge(Map clientProperties) { + Preconditions.checkNotNull( + clientProperties, + "Cannot merge client properties with server-provided properties. Invalid client configuration: null"); + Map merged = + defaults != null ? Maps.newHashMap(defaults) : Maps.newHashMap(); + merged.putAll(clientProperties); + + if (overrides != null) { + merged.putAll(overrides); + } + + return ImmutableMap.copyOf(Maps.filterValues(merged, Objects::nonNull)); + } + + @JsonGetter(FIELD_DEFAULTS) + public Map getDefaults() { + return defaults; + } + + @JsonGetter(FIELD_OVERRIDES) + public Map getOverrides() { + return overrides; + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/responses/CreateDatabaseResponse.java b/paimon-core/src/main/java/org/apache/paimon/rest/responses/CreateDatabaseResponse.java new file mode 100644 index 000000000000..43c99254f399 --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/rest/responses/CreateDatabaseResponse.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest.responses; + +import org.apache.paimon.rest.RESTResponse; + +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonGetter; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.Map; + +/** Response for creating database. */ +public class CreateDatabaseResponse implements RESTResponse { + + private static final String FIELD_NAME = "name"; + private static final String FIELD_OPTIONS = "options"; + + @JsonProperty(FIELD_NAME) + private String name; + + @JsonProperty(FIELD_OPTIONS) + private Map options; + + @JsonCreator + public CreateDatabaseResponse( + @JsonProperty(FIELD_NAME) String name, + @JsonProperty(FIELD_OPTIONS) Map options) { + this.name = name; + this.options = options; + } + + @JsonGetter(FIELD_NAME) + public String getName() { + return name; + } + + @JsonGetter(FIELD_OPTIONS) + public Map getOptions() { + return options; + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/responses/DatabaseName.java b/paimon-core/src/main/java/org/apache/paimon/rest/responses/DatabaseName.java new file mode 100644 index 000000000000..9a93b2fd1e3d --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/rest/responses/DatabaseName.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest.responses; + +import org.apache.paimon.rest.RESTMessage; + +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonGetter; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty; + +/** Class for Database entity. */ +public class DatabaseName implements RESTMessage { + + private static final String FIELD_NAME = "name"; + + @JsonProperty(FIELD_NAME) + private String name; + + @JsonCreator + public DatabaseName(@JsonProperty(FIELD_NAME) String name) { + this.name = name; + } + + @JsonGetter(FIELD_NAME) + public String getName() { + return this.name; + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/responses/ErrorResponse.java b/paimon-core/src/main/java/org/apache/paimon/rest/responses/ErrorResponse.java new file mode 100644 index 000000000000..d24c8f0f9936 --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/rest/responses/ErrorResponse.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest.responses; + +import org.apache.paimon.rest.RESTResponse; + +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonGetter; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty; + +import java.io.PrintWriter; +import java.io.StringWriter; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +/** Response for error. */ +public class ErrorResponse implements RESTResponse { + + private static final String FIELD_MESSAGE = "message"; + private static final String FIELD_CODE = "code"; + private static final String FIELD_STACK = "stack"; + + @JsonProperty(FIELD_MESSAGE) + private final String message; + + @JsonProperty(FIELD_CODE) + private final Integer code; + + @JsonProperty(FIELD_STACK) + private final List stack; + + public ErrorResponse(String message, Integer code) { + this.code = code; + this.message = message; + this.stack = new ArrayList(); + } + + @JsonCreator + public ErrorResponse( + @JsonProperty(FIELD_MESSAGE) String message, + @JsonProperty(FIELD_CODE) int code, + @JsonProperty(FIELD_STACK) List stack) { + this.message = message; + this.code = code; + this.stack = stack; + } + + public ErrorResponse(String message, int code, Throwable throwable) { + this.message = message; + this.code = code; + this.stack = getStackFromThrowable(throwable); + } + + @JsonGetter(FIELD_MESSAGE) + public String getMessage() { + return message; + } + + @JsonGetter(FIELD_CODE) + public Integer getCode() { + return code; + } + + @JsonGetter(FIELD_STACK) + public List getStack() { + return stack; + } + + private List getStackFromThrowable(Throwable throwable) { + if (throwable == null) { + return new ArrayList(); + } + StringWriter sw = new StringWriter(); + try (PrintWriter pw = new PrintWriter(sw)) { + throwable.printStackTrace(pw); + } + + return Arrays.asList(sw.toString().split("\n")); + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/responses/GetDatabaseResponse.java b/paimon-core/src/main/java/org/apache/paimon/rest/responses/GetDatabaseResponse.java new file mode 100644 index 000000000000..f8f7c8794b7b --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/rest/responses/GetDatabaseResponse.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest.responses; + +import org.apache.paimon.catalog.Database; +import org.apache.paimon.rest.RESTResponse; + +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonGetter; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.Map; +import java.util.Optional; + +import static org.apache.paimon.rest.RESTCatalogInternalOptions.DATABASE_COMMENT; + +/** Response for getting database. */ +public class GetDatabaseResponse implements RESTResponse, Database { + + private static final String FIELD_NAME = "name"; + private static final String FIELD_OPTIONS = "options"; + + @JsonProperty(FIELD_NAME) + private final String name; + + @JsonProperty(FIELD_OPTIONS) + private final Map options; + + @JsonCreator + public GetDatabaseResponse( + @JsonProperty(FIELD_NAME) String name, + @JsonProperty(FIELD_OPTIONS) Map options) { + this.name = name; + this.options = options; + } + + @JsonGetter(FIELD_NAME) + public String getName() { + return name; + } + + @JsonGetter(FIELD_OPTIONS) + public Map getOptions() { + return options; + } + + @Override + public String name() { + return this.getName(); + } + + @Override + public Map options() { + return this.getOptions(); + } + + @Override + public Optional comment() { + return Optional.ofNullable( + this.options.getOrDefault(DATABASE_COMMENT.key(), DATABASE_COMMENT.defaultValue())); + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/responses/ListDatabasesResponse.java b/paimon-core/src/main/java/org/apache/paimon/rest/responses/ListDatabasesResponse.java new file mode 100644 index 000000000000..38773f354b77 --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/rest/responses/ListDatabasesResponse.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest.responses; + +import org.apache.paimon.rest.RESTResponse; + +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonGetter; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.List; + +/** Response for listing databases. */ +public class ListDatabasesResponse implements RESTResponse { + private static final String FIELD_DATABASES = "databases"; + + @JsonProperty(FIELD_DATABASES) + private List databases; + + @JsonCreator + public ListDatabasesResponse(@JsonProperty(FIELD_DATABASES) List databases) { + this.databases = databases; + } + + @JsonGetter(FIELD_DATABASES) + public List getDatabases() { + return this.databases; + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/schema/Schema.java b/paimon-core/src/main/java/org/apache/paimon/schema/Schema.java index c6c79f4d4afd..33309a7cecc9 100644 --- a/paimon-core/src/main/java/org/apache/paimon/schema/Schema.java +++ b/paimon-core/src/main/java/org/apache/paimon/schema/Schema.java @@ -96,6 +96,10 @@ public String comment() { return comment; } + public Schema copy(RowType rowType) { + return new Schema(rowType.getFields(), partitionKeys, primaryKeys, options, comment); + } + private static List normalizeFields( List fields, List primaryKeys, List partitionKeys) { List fieldNames = fields.stream().map(DataField::name).collect(Collectors.toList()); @@ -337,13 +341,4 @@ public Schema build() { return new Schema(columns, partitionKeys, primaryKeys, options, comment); } } - - public static Schema fromTableSchema(TableSchema tableSchema) { - return new Schema( - tableSchema.fields(), - tableSchema.partitionKeys(), - tableSchema.primaryKeys(), - tableSchema.options(), - tableSchema.comment()); - } } diff --git a/paimon-core/src/main/java/org/apache/paimon/schema/SchemaChange.java b/paimon-core/src/main/java/org/apache/paimon/schema/SchemaChange.java index 1e790bf659a1..cefa3c6eb9e7 100644 --- a/paimon-core/src/main/java/org/apache/paimon/schema/SchemaChange.java +++ b/paimon-core/src/main/java/org/apache/paimon/schema/SchemaChange.java @@ -52,28 +52,46 @@ static SchemaChange addColumn(String fieldName, DataType dataType) { } static SchemaChange addColumn(String fieldName, DataType dataType, String comment) { - return new AddColumn(fieldName, dataType, comment, null); + return new AddColumn(new String[] {fieldName}, dataType, comment, null); } static SchemaChange addColumn(String fieldName, DataType dataType, String comment, Move move) { - return new AddColumn(fieldName, dataType, comment, move); + return new AddColumn(new String[] {fieldName}, dataType, comment, move); + } + + static SchemaChange addColumn( + String[] fieldNames, DataType dataType, String comment, Move move) { + return new AddColumn(fieldNames, dataType, comment, move); } static SchemaChange renameColumn(String fieldName, String newName) { - return new RenameColumn(fieldName, newName); + return new RenameColumn(new String[] {fieldName}, newName); + } + + static SchemaChange renameColumn(String[] fieldNames, String newName) { + return new RenameColumn(fieldNames, newName); } static SchemaChange dropColumn(String fieldName) { - return new DropColumn(fieldName); + return new DropColumn(new String[] {fieldName}); + } + + static SchemaChange dropColumn(String[] fieldNames) { + return new DropColumn(fieldNames); } static SchemaChange updateColumnType(String fieldName, DataType newDataType) { - return new UpdateColumnType(fieldName, newDataType, false); + return new UpdateColumnType(new String[] {fieldName}, newDataType, false); } static SchemaChange updateColumnType( String fieldName, DataType newDataType, boolean keepNullability) { - return new UpdateColumnType(fieldName, newDataType, keepNullability); + return new UpdateColumnType(new String[] {fieldName}, newDataType, keepNullability); + } + + static SchemaChange updateColumnType( + String[] fieldNames, DataType newDataType, boolean keepNullability) { + return new UpdateColumnType(fieldNames, newDataType, keepNullability); } static SchemaChange updateColumnNullability(String fieldName, boolean newNullability) { @@ -207,20 +225,20 @@ final class AddColumn implements SchemaChange { private static final long serialVersionUID = 1L; - private final String fieldName; + private final String[] fieldNames; private final DataType dataType; private final String description; private final Move move; - private AddColumn(String fieldName, DataType dataType, String description, Move move) { - this.fieldName = fieldName; + private AddColumn(String[] fieldNames, DataType dataType, String description, Move move) { + this.fieldNames = fieldNames; this.dataType = dataType; this.description = description; this.move = move; } - public String fieldName() { - return fieldName; + public String[] fieldNames() { + return fieldNames; } public DataType dataType() { @@ -246,7 +264,7 @@ public boolean equals(Object o) { return false; } AddColumn addColumn = (AddColumn) o; - return Objects.equals(fieldName, addColumn.fieldName) + return Arrays.equals(fieldNames, addColumn.fieldNames) && dataType.equals(addColumn.dataType) && Objects.equals(description, addColumn.description) && move.equals(addColumn.move); @@ -255,7 +273,7 @@ public boolean equals(Object o) { @Override public int hashCode() { int result = Objects.hash(dataType, description); - result = 31 * result + Objects.hashCode(fieldName); + result = 31 * result + Objects.hashCode(fieldNames); result = 31 * result + Objects.hashCode(move); return result; } @@ -266,16 +284,16 @@ final class RenameColumn implements SchemaChange { private static final long serialVersionUID = 1L; - private final String fieldName; + private final String[] fieldNames; private final String newName; - private RenameColumn(String fieldName, String newName) { - this.fieldName = fieldName; + private RenameColumn(String[] fieldNames, String newName) { + this.fieldNames = fieldNames; this.newName = newName; } - public String fieldName() { - return fieldName; + public String[] fieldNames() { + return fieldNames; } public String newName() { @@ -291,14 +309,14 @@ public boolean equals(Object o) { return false; } RenameColumn that = (RenameColumn) o; - return Objects.equals(fieldName, that.fieldName) + return Arrays.equals(fieldNames, that.fieldNames) && Objects.equals(newName, that.newName); } @Override public int hashCode() { int result = Objects.hash(newName); - result = 31 * result + Objects.hashCode(fieldName); + result = 31 * result + Objects.hashCode(fieldNames); return result; } } @@ -308,14 +326,14 @@ final class DropColumn implements SchemaChange { private static final long serialVersionUID = 1L; - private final String fieldName; + private final String[] fieldNames; - private DropColumn(String fieldName) { - this.fieldName = fieldName; + private DropColumn(String[] fieldNames) { + this.fieldNames = fieldNames; } - public String fieldName() { - return fieldName; + public String[] fieldNames() { + return fieldNames; } @Override @@ -327,12 +345,12 @@ public boolean equals(Object o) { return false; } DropColumn that = (DropColumn) o; - return Objects.equals(fieldName, that.fieldName); + return Arrays.equals(fieldNames, that.fieldNames); } @Override public int hashCode() { - return Objects.hashCode(fieldName); + return Objects.hashCode(fieldNames); } } @@ -341,19 +359,20 @@ final class UpdateColumnType implements SchemaChange { private static final long serialVersionUID = 1L; - private final String fieldName; + private final String[] fieldNames; private final DataType newDataType; // If true, do not change the target field nullability private final boolean keepNullability; - private UpdateColumnType(String fieldName, DataType newDataType, boolean keepNullability) { - this.fieldName = fieldName; + private UpdateColumnType( + String[] fieldNames, DataType newDataType, boolean keepNullability) { + this.fieldNames = fieldNames; this.newDataType = newDataType; this.keepNullability = keepNullability; } - public String fieldName() { - return fieldName; + public String[] fieldNames() { + return fieldNames; } public DataType newDataType() { @@ -373,14 +392,14 @@ public boolean equals(Object o) { return false; } UpdateColumnType that = (UpdateColumnType) o; - return Objects.equals(fieldName, that.fieldName) + return Arrays.equals(fieldNames, that.fieldNames) && newDataType.equals(that.newDataType); } @Override public int hashCode() { int result = Objects.hash(newDataType); - result = 31 * result + Objects.hashCode(fieldName); + result = 31 * result + Objects.hashCode(fieldNames); return result; } } diff --git a/paimon-core/src/main/java/org/apache/paimon/schema/SchemaEvolutionUtil.java b/paimon-core/src/main/java/org/apache/paimon/schema/SchemaEvolutionUtil.java index 083d131ec846..0ae2798c29e0 100644 --- a/paimon-core/src/main/java/org/apache/paimon/schema/SchemaEvolutionUtil.java +++ b/paimon-core/src/main/java/org/apache/paimon/schema/SchemaEvolutionUtil.java @@ -19,9 +19,15 @@ package org.apache.paimon.schema; import org.apache.paimon.KeyValue; +import org.apache.paimon.casting.CastElementGetter; import org.apache.paimon.casting.CastExecutor; import org.apache.paimon.casting.CastExecutors; import org.apache.paimon.casting.CastFieldGetter; +import org.apache.paimon.casting.CastedArray; +import org.apache.paimon.casting.CastedMap; +import org.apache.paimon.casting.CastedRow; +import org.apache.paimon.data.InternalArray; +import org.apache.paimon.data.InternalMap; import org.apache.paimon.data.InternalRow; import org.apache.paimon.predicate.LeafPredicate; import org.apache.paimon.predicate.Predicate; @@ -30,7 +36,6 @@ import org.apache.paimon.types.DataField; import org.apache.paimon.types.DataType; import org.apache.paimon.types.MapType; -import org.apache.paimon.types.MultisetType; import org.apache.paimon.types.RowType; import org.apache.paimon.utils.InternalRowUtils; import org.apache.paimon.utils.ProjectedRow; @@ -67,8 +72,6 @@ public class SchemaEvolutionUtil { * data fields, -1 is the index of 6->b in data fields and 1 is the index of 3->a in data * fields. * - *

    /// TODO should support nest index mapping when nest schema evolution is supported. - * * @param tableFields the fields of table * @param dataFields the fields of underlying data * @return the index mapping @@ -373,6 +376,7 @@ private static CastFieldGetter[] createCastFieldGetterMapping( List tableFields, List dataFields, int[] indexMapping) { CastFieldGetter[] converterMapping = new CastFieldGetter[tableFields.size()]; boolean castExist = false; + for (int i = 0; i < tableFields.size(); i++) { int dataIndex = indexMapping == null ? i : indexMapping[i]; if (dataIndex < 0) { @@ -381,36 +385,84 @@ private static CastFieldGetter[] createCastFieldGetterMapping( } else { DataField tableField = tableFields.get(i); DataField dataField = dataFields.get(dataIndex); - if (dataField.type().equalsIgnoreNullable(tableField.type())) { - // Create getter with index i and projected row data will convert to underlying - // data - converterMapping[i] = - new CastFieldGetter( - InternalRowUtils.createNullCheckingFieldGetter( - dataField.type(), i), - CastExecutors.identityCastExecutor()); - } else { - // TODO support column type evolution in nested type - checkState( - !(tableField.type() instanceof MapType - || dataField.type() instanceof ArrayType - || dataField.type() instanceof MultisetType - || dataField.type() instanceof RowType), - "Only support column type evolution in atomic data type."); - // Create getter with index i and projected row data will convert to underlying - // data - converterMapping[i] = - new CastFieldGetter( - InternalRowUtils.createNullCheckingFieldGetter( - dataField.type(), i), - checkNotNull( - CastExecutors.resolve( - dataField.type(), tableField.type()))); + if (!dataField.type().equalsIgnoreNullable(tableField.type())) { castExist = true; } + + // Create getter with index i and projected row data will convert to underlying data + converterMapping[i] = + new CastFieldGetter( + InternalRowUtils.createNullCheckingFieldGetter(dataField.type(), i), + createCastExecutor(dataField.type(), tableField.type())); } } return castExist ? converterMapping : null; } + + private static CastExecutor createCastExecutor(DataType inputType, DataType targetType) { + if (targetType.equalsIgnoreNullable(inputType)) { + return CastExecutors.identityCastExecutor(); + } else if (inputType instanceof RowType && targetType instanceof RowType) { + return createRowCastExecutor((RowType) inputType, (RowType) targetType); + } else if (inputType instanceof ArrayType && targetType instanceof ArrayType) { + return createArrayCastExecutor((ArrayType) inputType, (ArrayType) targetType); + } else if (inputType instanceof MapType && targetType instanceof MapType) { + return createMapCastExecutor((MapType) inputType, (MapType) targetType); + } else { + return checkNotNull( + CastExecutors.resolve(inputType, targetType), + "Cannot cast from type %s to type %s", + inputType, + targetType); + } + } + + private static CastExecutor createRowCastExecutor( + RowType inputType, RowType targetType) { + int[] indexMapping = createIndexMapping(targetType.getFields(), inputType.getFields()); + CastFieldGetter[] castFieldGetters = + createCastFieldGetterMapping( + targetType.getFields(), inputType.getFields(), indexMapping); + + ProjectedRow projectedRow = indexMapping == null ? null : ProjectedRow.from(indexMapping); + CastedRow castedRow = castFieldGetters == null ? null : CastedRow.from(castFieldGetters); + return value -> { + if (projectedRow != null) { + value = projectedRow.replaceRow(value); + } + if (castedRow != null) { + value = castedRow.replaceRow(value); + } + return value; + }; + } + + private static CastExecutor createArrayCastExecutor( + ArrayType inputType, ArrayType targetType) { + CastElementGetter castElementGetter = + new CastElementGetter( + InternalArray.createElementGetter(inputType.getElementType()), + createCastExecutor( + inputType.getElementType(), targetType.getElementType())); + + CastedArray castedArray = CastedArray.from(castElementGetter); + return castedArray::replaceArray; + } + + private static CastExecutor createMapCastExecutor( + MapType inputType, MapType targetType) { + checkState( + inputType.getKeyType().equals(targetType.getKeyType()), + "Cannot cast map type %s to map type %s, because they have different key types.", + inputType.getKeyType(), + targetType.getKeyType()); + CastElementGetter castElementGetter = + new CastElementGetter( + InternalArray.createElementGetter(inputType.getValueType()), + createCastExecutor(inputType.getValueType(), targetType.getValueType())); + + CastedMap castedMap = CastedMap.from(castElementGetter); + return castedMap::replaceMap; + } } diff --git a/paimon-core/src/main/java/org/apache/paimon/schema/SchemaManager.java b/paimon-core/src/main/java/org/apache/paimon/schema/SchemaManager.java index 962cba952657..2139dca4a990 100644 --- a/paimon-core/src/main/java/org/apache/paimon/schema/SchemaManager.java +++ b/paimon-core/src/main/java/org/apache/paimon/schema/SchemaManager.java @@ -36,16 +36,22 @@ import org.apache.paimon.schema.SchemaChange.UpdateColumnPosition; import org.apache.paimon.schema.SchemaChange.UpdateColumnType; import org.apache.paimon.schema.SchemaChange.UpdateComment; +import org.apache.paimon.table.FileStoreTableFactory; +import org.apache.paimon.types.ArrayType; import org.apache.paimon.types.DataField; import org.apache.paimon.types.DataType; import org.apache.paimon.types.DataTypeCasts; -import org.apache.paimon.types.DataTypeVisitor; +import org.apache.paimon.types.MapType; import org.apache.paimon.types.ReassignFieldId; import org.apache.paimon.types.RowType; import org.apache.paimon.utils.BranchManager; -import org.apache.paimon.utils.JsonSerdeUtil; import org.apache.paimon.utils.Preconditions; import org.apache.paimon.utils.SnapshotManager; +import org.apache.paimon.utils.StringUtils; + +import org.apache.paimon.shade.guava30.com.google.common.base.Joiner; +import org.apache.paimon.shade.guava30.com.google.common.collect.Iterables; +import org.apache.paimon.shade.guava30.com.google.common.collect.Maps; import javax.annotation.Nullable; import javax.annotation.concurrent.ThreadSafe; @@ -68,10 +74,12 @@ import java.util.stream.Collectors; import java.util.stream.LongStream; -import static org.apache.paimon.catalog.Catalog.DB_SUFFIX; +import static org.apache.paimon.CoreOptions.BUCKET_KEY; +import static org.apache.paimon.catalog.AbstractCatalog.DB_SUFFIX; import static org.apache.paimon.catalog.Identifier.UNKNOWN_DATABASE; import static org.apache.paimon.utils.BranchManager.DEFAULT_MAIN_BRANCH; import static org.apache.paimon.utils.FileUtils.listVersionedFiles; +import static org.apache.paimon.utils.Preconditions.checkArgument; import static org.apache.paimon.utils.Preconditions.checkState; /** Schema Manager to manage schema versions. */ @@ -117,10 +125,32 @@ public Optional latest() { } } + public long earliestCreationTime() { + try { + long earliest = 0; + if (!schemaExists(0)) { + Optional min = + listVersionedFiles(fileIO, schemaDirectory(), SCHEMA_PREFIX) + .reduce(Math::min); + checkArgument(min.isPresent()); + earliest = min.get(); + } + + Path schemaPath = toSchemaPath(earliest); + return fileIO.getFileStatus(schemaPath).getModificationTime(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + public List listAll() { return listAllIds().stream().map(this::schema).collect(Collectors.toList()); } + public List schemasWithId(List schemaIds) { + return schemaIds.stream().map(this::schema).collect(Collectors.toList()); + } + public List listWithRange( Optional optionalMaxSchemaId, Optional optionalMinSchemaId) { Long lowerBoundSchemaId = 0L; @@ -178,24 +208,18 @@ public TableSchema createTable(Schema schema) throws Exception { return createTable(schema, false); } - public TableSchema createTable(Schema schema, boolean ignoreIfExistsSame) throws Exception { + public TableSchema createTable(Schema schema, boolean externalTable) throws Exception { while (true) { Optional latest = latest(); if (latest.isPresent()) { - TableSchema oldSchema = latest.get(); - boolean isSame = - Objects.equals(oldSchema.fields(), schema.fields()) - && Objects.equals(oldSchema.partitionKeys(), schema.partitionKeys()) - && Objects.equals(oldSchema.primaryKeys(), schema.primaryKeys()) - && Objects.equals(oldSchema.options(), schema.options()); - if (ignoreIfExistsSame && isSame) { - return oldSchema; + TableSchema latestSchema = latest.get(); + if (externalTable) { + checkSchemaForExternalTable(latestSchema, schema); + return latestSchema; + } else { + throw new IllegalStateException( + "Schema in filesystem exists, creation is not allowed."); } - - throw new IllegalStateException( - "Schema in filesystem exists, please use updating," - + " latest schema is: " - + oldSchema); } List fields = schema.fields(); @@ -214,6 +238,9 @@ public TableSchema createTable(Schema schema, boolean ignoreIfExistsSame) throws options, schema.comment()); + // validate table from creating table + FileStoreTableFactory.create(fileIO, tableRoot, newSchema).store(); + boolean success = commit(newSchema); if (success) { return newSchema; @@ -221,6 +248,38 @@ public TableSchema createTable(Schema schema, boolean ignoreIfExistsSame) throws } } + private void checkSchemaForExternalTable(TableSchema existsSchema, Schema newSchema) { + // When creating an external table, if the table already exists in the location, we can + // choose not to specify the fields. + if (newSchema.fields().isEmpty() + // When the fields are explicitly specified, we need check for consistency. + || (Objects.equals(existsSchema.fields(), newSchema.fields()) + && Objects.equals(existsSchema.partitionKeys(), newSchema.partitionKeys()) + && Objects.equals(existsSchema.primaryKeys(), newSchema.primaryKeys()))) { + // check for options + Map existsOptions = existsSchema.options(); + Map newOptions = newSchema.options(); + newOptions.forEach( + (key, value) -> { + if (!key.equals(Catalog.OWNER_PROP) + && (!existsOptions.containsKey(key) + || !existsOptions.get(key).equals(value))) { + throw new RuntimeException( + "New schema's options are not equal to the exists schema's, new schema: " + + newOptions + + ", exists schema: " + + existsOptions); + } + }); + } else { + throw new RuntimeException( + "New schema is not equal to exists schema, new schema: " + + newSchema + + ", exists schema: " + + existsSchema); + } + } + /** Update {@link SchemaChange}s. */ public TableSchema commitChanges(SchemaChange... changes) throws Exception { return commitChanges(Arrays.asList(changes)); @@ -268,83 +327,90 @@ public TableSchema commitChanges(List changes) } else if (change instanceof AddColumn) { AddColumn addColumn = (AddColumn) change; SchemaChange.Move move = addColumn.move(); - if (newFields.stream().anyMatch(f -> f.name().equals(addColumn.fieldName()))) { - throw new Catalog.ColumnAlreadyExistException( - identifierFromPath(tableRoot.toString(), true, branch), - addColumn.fieldName()); - } Preconditions.checkArgument( addColumn.dataType().isNullable(), "Column %s cannot specify NOT NULL in the %s table.", - addColumn.fieldName(), + String.join(".", addColumn.fieldNames()), identifierFromPath(tableRoot.toString(), true, branch).getFullName()); int id = highestFieldId.incrementAndGet(); DataType dataType = ReassignFieldId.reassign(addColumn.dataType(), highestFieldId); - DataField dataField = - new DataField( - id, addColumn.fieldName(), dataType, addColumn.description()); - - // key: name ; value : index - Map map = new HashMap<>(); - for (int i = 0; i < newFields.size(); i++) { - map.put(newFields.get(i).name(), i); - } - - if (null != move) { - if (move.type().equals(SchemaChange.Move.MoveType.FIRST)) { - newFields.add(0, dataField); - } else if (move.type().equals(SchemaChange.Move.MoveType.AFTER)) { - int fieldIndex = map.get(move.referenceFieldName()); - newFields.add(fieldIndex + 1, dataField); + new NestedColumnModifier(addColumn.fieldNames()) { + @Override + protected void updateLastColumn(List newFields, String fieldName) + throws Catalog.ColumnAlreadyExistException { + assertColumnNotExists(newFields, fieldName); + + DataField dataField = + new DataField(id, fieldName, dataType, addColumn.description()); + + // key: name ; value : index + Map map = new HashMap<>(); + for (int i = 0; i < newFields.size(); i++) { + map.put(newFields.get(i).name(), i); + } + + if (null != move) { + if (move.type().equals(SchemaChange.Move.MoveType.FIRST)) { + newFields.add(0, dataField); + } else if (move.type().equals(SchemaChange.Move.MoveType.AFTER)) { + int fieldIndex = map.get(move.referenceFieldName()); + newFields.add(fieldIndex + 1, dataField); + } + } else { + newFields.add(dataField); + } } - } else { - newFields.add(dataField); - } - + }.updateIntermediateColumn(newFields, 0); } else if (change instanceof RenameColumn) { RenameColumn rename = (RenameColumn) change; - validateNotPrimaryAndPartitionKey(oldTableSchema, rename.fieldName()); - if (newFields.stream().anyMatch(f -> f.name().equals(rename.newName()))) { - throw new Catalog.ColumnAlreadyExistException( - identifierFromPath(tableRoot.toString(), true, branch), - rename.fieldName()); - } + assertNotUpdatingPrimaryKeys(oldTableSchema, rename.fieldNames(), "rename"); + new NestedColumnModifier(rename.fieldNames()) { + @Override + protected void updateLastColumn(List newFields, String fieldName) + throws Catalog.ColumnNotExistException, + Catalog.ColumnAlreadyExistException { + assertColumnExists(newFields, fieldName); + assertColumnNotExists(newFields, rename.newName()); + for (int i = 0; i < newFields.size(); i++) { + DataField field = newFields.get(i); + if (!field.name().equals(fieldName)) { + continue; + } - updateNestedColumn( - newFields, - new String[] {rename.fieldName()}, - 0, - (field) -> - new DataField( - field.id(), - rename.newName(), - field.type(), - field.description())); + DataField newField = + new DataField( + field.id(), + rename.newName(), + field.type(), + field.description()); + newFields.set(i, newField); + return; + } + } + }.updateIntermediateColumn(newFields, 0); } else if (change instanceof DropColumn) { DropColumn drop = (DropColumn) change; - validateNotPrimaryAndPartitionKey(oldTableSchema, drop.fieldName()); - if (!newFields.removeIf( - f -> f.name().equals(((DropColumn) change).fieldName()))) { - throw new Catalog.ColumnNotExistException( - identifierFromPath(tableRoot.toString(), true, branch), - drop.fieldName()); - } - if (newFields.isEmpty()) { - throw new IllegalArgumentException("Cannot drop all fields in table"); - } + dropColumnValidation(oldTableSchema, drop); + new NestedColumnModifier(drop.fieldNames()) { + @Override + protected void updateLastColumn(List newFields, String fieldName) + throws Catalog.ColumnNotExistException { + assertColumnExists(newFields, fieldName); + newFields.removeIf(f -> f.name().equals(fieldName)); + if (newFields.isEmpty()) { + throw new IllegalArgumentException( + "Cannot drop all fields in table"); + } + } + }.updateIntermediateColumn(newFields, 0); } else if (change instanceof UpdateColumnType) { UpdateColumnType update = (UpdateColumnType) change; - if (oldTableSchema.partitionKeys().contains(update.fieldName())) { - throw new IllegalArgumentException( - String.format( - "Cannot update partition column [%s] type in the table[%s].", - update.fieldName(), tableRoot.getName())); - } - updateColumn( + assertNotUpdatingPrimaryKeys(oldTableSchema, update.fieldNames(), "update"); + updateNestedColumn( newFields, - update.fieldName(), + update.fieldNames(), (field) -> { DataType targetType = update.newDataType(); if (update.keepNullability()) { @@ -357,13 +423,6 @@ public TableSchema commitChanges(List changes) String.format( "Column type %s[%s] cannot be converted to %s without loosing information.", field.name(), field.type(), targetType)); - AtomicInteger dummyId = new AtomicInteger(0); - if (dummyId.get() != 0) { - throw new RuntimeException( - String.format( - "Update column to nested row type '%s' is not supported.", - targetType)); - } return new DataField( field.id(), field.name(), targetType, field.description()); }); @@ -378,7 +437,6 @@ public TableSchema commitChanges(List changes) updateNestedColumn( newFields, update.fieldNames(), - 0, (field) -> new DataField( field.id(), @@ -390,7 +448,6 @@ public TableSchema commitChanges(List changes) updateNestedColumn( newFields, update.fieldNames(), - 0, (field) -> new DataField( field.id(), @@ -413,8 +470,10 @@ public TableSchema commitChanges(List changes) new Schema( newFields, oldTableSchema.partitionKeys(), - oldTableSchema.primaryKeys(), - newOptions, + applyNotNestedColumnRename( + oldTableSchema.primaryKeys(), + Iterables.filter(changes, RenameColumn.class)), + applySchemaChanges(newOptions, changes), newComment); TableSchema newTableSchema = new TableSchema( @@ -519,62 +578,214 @@ public boolean mergeSchema(RowType rowType, boolean allowExplicitCast) { } } - private void validateNotPrimaryAndPartitionKey(TableSchema schema, String fieldName) { - /// TODO support partition and primary keys schema evolution - if (schema.partitionKeys().contains(fieldName)) { + private static Map applySchemaChanges( + Map options, Iterable changes) { + Map newOptions = Maps.newHashMap(options); + String bucketKeysStr = options.get(BUCKET_KEY.key()); + if (!StringUtils.isNullOrWhitespaceOnly(bucketKeysStr)) { + List bucketColumns = Arrays.asList(bucketKeysStr.split(",")); + List newBucketColumns = + applyNotNestedColumnRename( + bucketColumns, Iterables.filter(changes, RenameColumn.class)); + newOptions.put(BUCKET_KEY.key(), Joiner.on(',').join(newBucketColumns)); + } + + // TODO: Apply changes to other options that contain column names, such as `sequence.field` + return newOptions; + } + + // Apply column rename changes on not nested columns to the list of column names, this will not + // change the order of the column names + private static List applyNotNestedColumnRename( + List columns, Iterable renames) { + if (Iterables.isEmpty(renames)) { + return columns; + } + + Map columnNames = Maps.newHashMap(); + for (RenameColumn renameColumn : renames) { + if (renameColumn.fieldNames().length == 1) { + columnNames.put(renameColumn.fieldNames()[0], renameColumn.newName()); + } + } + + // The order of the column names will be preserved, as a non-parallel stream is used here. + return columns.stream() + .map(column -> columnNames.getOrDefault(column, column)) + .collect(Collectors.toList()); + } + + private static void dropColumnValidation(TableSchema schema, DropColumn change) { + // primary keys and partition keys can't be nested columns + if (change.fieldNames().length > 1) { + return; + } + String columnToDrop = change.fieldNames()[0]; + if (schema.partitionKeys().contains(columnToDrop) + || schema.primaryKeys().contains(columnToDrop)) { throw new UnsupportedOperationException( - String.format("Cannot drop/rename partition key[%s]", fieldName)); + String.format("Cannot drop partition key or primary key: [%s]", columnToDrop)); + } + } + + private static void assertNotUpdatingPrimaryKeys( + TableSchema schema, String[] fieldNames, String operation) { + // partition keys can't be nested columns + if (fieldNames.length > 1) { + return; } - if (schema.primaryKeys().contains(fieldName)) { + String columnToRename = fieldNames[0]; + if (schema.partitionKeys().contains(columnToRename)) { throw new UnsupportedOperationException( - String.format("Cannot drop/rename primary key[%s]", fieldName)); + String.format( + "Cannot " + operation + " partition column: [%s]", columnToRename)); } } - /** This method is hacky, newFields may be immutable. We should use {@link DataTypeVisitor}. */ - private void updateNestedColumn( - List newFields, - String[] updateFieldNames, - int index, - Function updateFunc) - throws Catalog.ColumnNotExistException { - boolean found = false; - for (int i = 0; i < newFields.size(); i++) { - DataField field = newFields.get(i); - if (field.name().equals(updateFieldNames[index])) { - found = true; - if (index == updateFieldNames.length - 1) { - newFields.set(i, updateFunc.apply(field)); - break; - } else { - List nestedFields = - new ArrayList<>( - ((org.apache.paimon.types.RowType) field.type()).getFields()); - updateNestedColumn(nestedFields, updateFieldNames, index + 1, updateFunc); - newFields.set( - i, - new DataField( - field.id(), - field.name(), - new org.apache.paimon.types.RowType( - field.type().isNullable(), nestedFields), - field.description())); + private abstract class NestedColumnModifier { + + private final String[] updateFieldNames; + + private NestedColumnModifier(String[] updateFieldNames) { + this.updateFieldNames = updateFieldNames; + } + + public void updateIntermediateColumn(List newFields, int depth) + throws Catalog.ColumnNotExistException, Catalog.ColumnAlreadyExistException { + if (depth == updateFieldNames.length - 1) { + updateLastColumn(newFields, updateFieldNames[depth]); + return; + } + + for (int i = 0; i < newFields.size(); i++) { + DataField field = newFields.get(i); + if (!field.name().equals(updateFieldNames[depth])) { + continue; } + + String fullFieldName = + String.join(".", Arrays.asList(updateFieldNames).subList(0, depth + 1)); + List nestedFields = new ArrayList<>(); + int newDepth = + depth + extractRowDataFields(field.type(), fullFieldName, nestedFields); + updateIntermediateColumn(nestedFields, newDepth); + newFields.set( + i, + new DataField( + field.id(), + field.name(), + wrapNewRowType(field.type(), nestedFields), + field.description())); + return; + } + + throw new Catalog.ColumnNotExistException( + identifierFromPath(tableRoot.toString(), true, branch), + String.join(".", Arrays.asList(updateFieldNames).subList(0, depth + 1))); + } + + private int extractRowDataFields( + DataType type, String fullFieldName, List nestedFields) { + switch (type.getTypeRoot()) { + case ROW: + nestedFields.addAll(((RowType) type).getFields()); + return 1; + case ARRAY: + return extractRowDataFields( + ((ArrayType) type).getElementType(), + fullFieldName, + nestedFields) + + 1; + case MAP: + return extractRowDataFields( + ((MapType) type).getValueType(), fullFieldName, nestedFields) + + 1; + default: + throw new IllegalArgumentException( + fullFieldName + " is not a structured type."); + } + } + + private DataType wrapNewRowType(DataType type, List nestedFields) { + switch (type.getTypeRoot()) { + case ROW: + return new RowType(type.isNullable(), nestedFields); + case ARRAY: + return new ArrayType( + type.isNullable(), + wrapNewRowType(((ArrayType) type).getElementType(), nestedFields)); + case MAP: + MapType mapType = (MapType) type; + return new MapType( + type.isNullable(), + mapType.getKeyType(), + wrapNewRowType(mapType.getValueType(), nestedFields)); + default: + throw new IllegalStateException( + "Trying to wrap a row type in " + type + ". This is unexpected."); } } - if (!found) { + + protected abstract void updateLastColumn(List newFields, String fieldName) + throws Catalog.ColumnNotExistException, Catalog.ColumnAlreadyExistException; + + protected void assertColumnExists(List newFields, String fieldName) + throws Catalog.ColumnNotExistException { + for (DataField field : newFields) { + if (field.name().equals(fieldName)) { + return; + } + } throw new Catalog.ColumnNotExistException( identifierFromPath(tableRoot.toString(), true, branch), - Arrays.toString(updateFieldNames)); + getLastFieldName(fieldName)); + } + + protected void assertColumnNotExists(List newFields, String fieldName) + throws Catalog.ColumnAlreadyExistException { + for (DataField field : newFields) { + if (field.name().equals(fieldName)) { + throw new Catalog.ColumnAlreadyExistException( + identifierFromPath(tableRoot.toString(), true, branch), + getLastFieldName(fieldName)); + } + } + } + + private String getLastFieldName(String fieldName) { + List fieldNames = new ArrayList<>(); + for (int i = 0; i + 1 < updateFieldNames.length; i++) { + fieldNames.add(updateFieldNames[i]); + } + fieldNames.add(fieldName); + return String.join(".", fieldNames); } } - private void updateColumn( + private void updateNestedColumn( List newFields, - String updateFieldName, + String[] updateFieldNames, Function updateFunc) - throws Catalog.ColumnNotExistException { - updateNestedColumn(newFields, new String[] {updateFieldName}, 0, updateFunc); + throws Catalog.ColumnNotExistException, Catalog.ColumnAlreadyExistException { + new NestedColumnModifier(updateFieldNames) { + @Override + protected void updateLastColumn(List newFields, String fieldName) + throws Catalog.ColumnNotExistException { + for (int i = 0; i < newFields.size(); i++) { + DataField field = newFields.get(i); + if (!field.name().equals(fieldName)) { + continue; + } + + newFields.set(i, updateFunc.apply(field)); + return; + } + + throw new Catalog.ColumnNotExistException( + identifierFromPath(tableRoot.toString(), true, branch), + String.join(".", updateFieldNames)); + } + }.updateIntermediateColumn(newFields, 0); } @VisibleForTesting @@ -592,11 +803,7 @@ boolean commit(TableSchema newSchema) throws Exception { /** Read schema for schema id. */ public TableSchema schema(long id) { - try { - return JsonSerdeUtil.fromJson(fileIO.readFileUtf8(toSchemaPath(id)), TableSchema.class); - } catch (IOException e) { - throw new UncheckedIOException(e); - } + return TableSchema.fromPath(fileIO, toSchemaPath(id)); } /** Check if a schema exists. */ @@ -612,14 +819,6 @@ public boolean schemaExists(long id) { } } - public static TableSchema fromPath(FileIO fileIO, Path path) { - try { - return JsonSerdeUtil.fromJson(fileIO.readFileUtf8(path), TableSchema.class); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } - private String branchPath() { return BranchManager.branchPath(tableRoot, branch); } diff --git a/paimon-core/src/main/java/org/apache/paimon/schema/SchemaMergingUtils.java b/paimon-core/src/main/java/org/apache/paimon/schema/SchemaMergingUtils.java index 30004b53fcfb..d7e21cd6562d 100644 --- a/paimon-core/src/main/java/org/apache/paimon/schema/SchemaMergingUtils.java +++ b/paimon-core/src/main/java/org/apache/paimon/schema/SchemaMergingUtils.java @@ -48,7 +48,7 @@ public static TableSchema mergeSchemas( AtomicInteger highestFieldId = new AtomicInteger(currentTableSchema.highestFieldId()); RowType newRowType = mergeSchemas(currentType, targetType, highestFieldId, allowExplicitCast); - if (newRowType == currentType) { + if (newRowType.equals(currentType)) { // It happens if the `targetType` only changes `nullability` but we always respect the // current's. return currentTableSchema; diff --git a/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java b/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java index c714b90d5e07..20cbdea66d9a 100644 --- a/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java +++ b/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java @@ -68,8 +68,8 @@ import static org.apache.paimon.CoreOptions.SNAPSHOT_NUM_RETAINED_MIN; import static org.apache.paimon.CoreOptions.STREAMING_READ_OVERWRITE; import static org.apache.paimon.mergetree.compact.PartialUpdateMergeFunction.SEQUENCE_GROUP; -import static org.apache.paimon.table.SystemFields.KEY_FIELD_PREFIX; -import static org.apache.paimon.table.SystemFields.SYSTEM_FIELD_NAMES; +import static org.apache.paimon.table.SpecialFields.KEY_FIELD_PREFIX; +import static org.apache.paimon.table.SpecialFields.SYSTEM_FIELD_NAMES; import static org.apache.paimon.types.DataTypeRoot.ARRAY; import static org.apache.paimon.types.DataTypeRoot.MAP; import static org.apache.paimon.types.DataTypeRoot.MULTISET; @@ -180,7 +180,7 @@ public static void validateTableSchema(TableSchema schema) { if (options.changelogProducer() != ChangelogProducer.LOOKUP && options.changelogProducer() != ChangelogProducer.NONE) { throw new IllegalArgumentException( - "Only support 'none' and 'lookup' changelog-producer on FIRST_MERGE merge engine"); + "Only support 'none' and 'lookup' changelog-producer on FIRST_ROW merge engine"); } } @@ -515,7 +515,7 @@ private static void validateForDeletionVectors(CoreOptions options) { private static void validateSequenceField(TableSchema schema, CoreOptions options) { List sequenceField = options.sequenceField(); - if (sequenceField.size() > 0) { + if (!sequenceField.isEmpty()) { Map fieldCount = sequenceField.stream() .collect(Collectors.toMap(field -> field, field -> 1, Integer::sum)); @@ -540,7 +540,7 @@ private static void validateSequenceField(TableSchema schema, CoreOptions option if (options.mergeEngine() == MergeEngine.FIRST_ROW) { throw new IllegalArgumentException( - "Do not support use sequence field on FIRST_MERGE merge engine."); + "Do not support use sequence field on FIRST_ROW merge engine."); } if (schema.crossPartitionUpdate()) { @@ -596,12 +596,12 @@ private static void validateBucket(TableSchema schema, CoreOptions options) { == MAP || dataField.type().getTypeRoot() == ROW)) - .map(dataField -> dataField.name()) + .map(DataField::name) .collect(Collectors.toList()); - if (nestedFields.size() > 0) { + if (!nestedFields.isEmpty()) { throw new RuntimeException( "nested type can not in bucket-key, in your table these key are " - + nestedFields.toString()); + + nestedFields); } } } diff --git a/paimon-core/src/main/java/org/apache/paimon/schema/TableSchema.java b/paimon-core/src/main/java/org/apache/paimon/schema/TableSchema.java index b5bdeccf10f6..a0a149d1ae9b 100644 --- a/paimon-core/src/main/java/org/apache/paimon/schema/TableSchema.java +++ b/paimon-core/src/main/java/org/apache/paimon/schema/TableSchema.java @@ -29,8 +29,10 @@ import javax.annotation.Nullable; +import java.io.FileNotFoundException; import java.io.IOException; import java.io.Serializable; +import java.io.UncheckedIOException; import java.util.Arrays; import java.util.Collections; import java.util.HashSet; @@ -296,19 +298,6 @@ public TableSchema copy(Map newOptions) { timeMillis); } - public static TableSchema fromJson(String json) { - return JsonSerdeUtil.fromJson(json, TableSchema.class); - } - - public static TableSchema fromPath(FileIO fileIO, Path path) { - try { - String json = fileIO.readFileUtf8(path); - return TableSchema.fromJson(json); - } catch (IOException e) { - throw new RuntimeException("Fails to read schema from path " + path, e); - } - } - @Override public String toString() { return JsonSerdeUtil.toJson(this); @@ -341,4 +330,28 @@ public int hashCode() { public static List newFields(RowType rowType) { return rowType.getFields(); } + + // =================== Utils for reading ========================= + + public static TableSchema fromJson(String json) { + return JsonSerdeUtil.fromJson(json, TableSchema.class); + } + + public static TableSchema fromPath(FileIO fileIO, Path path) { + try { + return tryFromPath(fileIO, path); + } catch (FileNotFoundException e) { + throw new RuntimeException(e.getMessage(), e); + } + } + + public static TableSchema tryFromPath(FileIO fileIO, Path path) throws FileNotFoundException { + try { + return fromJson(fileIO.readFileUtf8(path)); + } catch (FileNotFoundException e) { + throw e; + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } } diff --git a/paimon-core/src/main/java/org/apache/paimon/sort/BinaryExternalSortBuffer.java b/paimon-core/src/main/java/org/apache/paimon/sort/BinaryExternalSortBuffer.java index 1ae45354646b..4bfbcd5ec71f 100644 --- a/paimon-core/src/main/java/org/apache/paimon/sort/BinaryExternalSortBuffer.java +++ b/paimon-core/src/main/java/org/apache/paimon/sort/BinaryExternalSortBuffer.java @@ -101,7 +101,8 @@ public static BinaryExternalSortBuffer create( int pageSize, int maxNumFileHandles, CompressOptions compression, - MemorySize maxDiskSize) { + MemorySize maxDiskSize, + boolean sequenceOrder) { return create( ioManager, rowType, @@ -109,7 +110,8 @@ public static BinaryExternalSortBuffer create( new HeapMemorySegmentPool(bufferSize, pageSize), maxNumFileHandles, compression, - maxDiskSize); + maxDiskSize, + sequenceOrder); } public static BinaryExternalSortBuffer create( @@ -119,8 +121,10 @@ public static BinaryExternalSortBuffer create( MemorySegmentPool pool, int maxNumFileHandles, CompressOptions compression, - MemorySize maxDiskSize) { - RecordComparator comparator = newRecordComparator(rowType.getFieldTypes(), keyFields); + MemorySize maxDiskSize, + boolean sequenceOrder) { + RecordComparator comparator = + newRecordComparator(rowType.getFieldTypes(), keyFields, sequenceOrder); BinaryInMemorySortBuffer sortBuffer = BinaryInMemorySortBuffer.createBuffer( newNormalizedKeyComputer(rowType.getFieldTypes(), keyFields), diff --git a/paimon-core/src/main/java/org/apache/paimon/stats/SimpleStatsConverter.java b/paimon-core/src/main/java/org/apache/paimon/stats/SimpleStatsConverter.java index b76dc72b8fff..13bafd879866 100644 --- a/paimon-core/src/main/java/org/apache/paimon/stats/SimpleStatsConverter.java +++ b/paimon-core/src/main/java/org/apache/paimon/stats/SimpleStatsConverter.java @@ -18,48 +18,78 @@ package org.apache.paimon.stats; -import org.apache.paimon.casting.CastFieldGetter; -import org.apache.paimon.casting.CastedRow; import org.apache.paimon.data.BinaryArray; -import org.apache.paimon.data.BinaryRow; -import org.apache.paimon.data.BinaryString; -import org.apache.paimon.data.Decimal; import org.apache.paimon.data.GenericRow; -import org.apache.paimon.data.InternalArray; -import org.apache.paimon.data.InternalMap; -import org.apache.paimon.data.InternalRow; -import org.apache.paimon.data.Timestamp; import org.apache.paimon.data.serializer.InternalRowSerializer; import org.apache.paimon.format.SimpleColStats; -import org.apache.paimon.types.DataType; import org.apache.paimon.types.RowType; -import org.apache.paimon.utils.ProjectedRow; +import org.apache.paimon.utils.Pair; -import javax.annotation.Nullable; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; /** Converter for array of {@link SimpleColStats}. */ public class SimpleStatsConverter { + private final RowType rowType; + private final boolean denseStore; private final InternalRowSerializer serializer; - - @Nullable private final int[] indexMapping; - @Nullable private final CastFieldGetter[] castFieldGetters; + private final Map, InternalRowSerializer> serializers; public SimpleStatsConverter(RowType type) { - this(type, null, null); + this(type, false); + } + + public SimpleStatsConverter(RowType type, boolean denseStore) { + // as stated in RollingFile.Writer#finish, col stats are not collected currently so + // min/max values are all nulls + this.rowType = + type.copy( + type.getFields().stream() + .map(f -> f.newType(f.type().copy(true))) + .collect(Collectors.toList())); + this.denseStore = denseStore; + this.serializer = new InternalRowSerializer(rowType); + this.serializers = new HashMap<>(); } - public SimpleStatsConverter( - RowType type, - @Nullable int[] indexMapping, - @Nullable CastFieldGetter[] castFieldGetters) { - RowType safeType = toAllFieldsNullableRowType(type); - this.serializer = new InternalRowSerializer(safeType); - this.indexMapping = indexMapping; - this.castFieldGetters = castFieldGetters; + public Pair, SimpleStats> toBinary(SimpleColStats[] stats) { + return denseStore ? toBinaryDenseMode(stats) : Pair.of(null, toBinaryAllMode(stats)); + } + + private Pair, SimpleStats> toBinaryDenseMode(SimpleColStats[] stats) { + List fields = new ArrayList<>(); + List minValues = new ArrayList<>(); + List maxValues = new ArrayList<>(); + List nullCounts = new ArrayList<>(); + for (int i = 0; i < stats.length; i++) { + SimpleColStats colStats = stats[i]; + if (colStats.isNone()) { + continue; + } + + fields.add(rowType.getFields().get(i).name()); + minValues.add(colStats.min()); + maxValues.add(colStats.max()); + nullCounts.add(colStats.nullCount()); + } + + InternalRowSerializer serializer = + serializers.computeIfAbsent( + fields, key -> new InternalRowSerializer(rowType.project(key))); + + SimpleStats simpleStats = + new SimpleStats( + serializer.toBinaryRow(GenericRow.of(minValues.toArray())).copy(), + serializer.toBinaryRow(GenericRow.of(maxValues.toArray())).copy(), + BinaryArray.fromLongArray(nullCounts.toArray(new Long[0]))); + return Pair.of(fields.size() == rowType.getFieldCount() ? null : fields, simpleStats); } - public SimpleStats toBinary(SimpleColStats[] stats) { + public SimpleStats toBinaryAllMode(SimpleColStats[] stats) { int rowFieldCount = stats.length; GenericRow minValues = new GenericRow(rowFieldCount); GenericRow maxValues = new GenericRow(rowFieldCount); @@ -74,192 +104,4 @@ public SimpleStats toBinary(SimpleColStats[] stats) { serializer.toBinaryRow(maxValues).copy(), BinaryArray.fromLongArray(nullCounts)); } - - public InternalRow evolution(BinaryRow values) { - InternalRow row = values; - if (indexMapping != null) { - row = ProjectedRow.from(indexMapping).replaceRow(row); - } - - if (castFieldGetters != null) { - row = CastedRow.from(castFieldGetters).replaceRow(values); - } - - return row; - } - - public InternalArray evolution(BinaryArray nullCounts, @Nullable Long rowCount) { - if (indexMapping == null) { - return nullCounts; - } - - if (rowCount == null) { - throw new RuntimeException("Schema Evolution for stats needs row count."); - } - - return new NullCountsEvoArray(indexMapping, nullCounts, rowCount); - } - - private static RowType toAllFieldsNullableRowType(RowType rowType) { - // as stated in RollingFile.Writer#finish, col stats are not collected currently so - // min/max values are all nulls - return RowType.builder() - .fields( - rowType.getFields().stream() - .map(f -> f.type().copy(true)) - .toArray(DataType[]::new), - rowType.getFieldNames().toArray(new String[0])) - .build(); - } - - private static class NullCountsEvoArray implements InternalArray { - - private final int[] indexMapping; - private final InternalArray array; - private final long notFoundValue; - - protected NullCountsEvoArray(int[] indexMapping, InternalArray array, long notFoundValue) { - this.indexMapping = indexMapping; - this.array = array; - this.notFoundValue = notFoundValue; - } - - @Override - public int size() { - return indexMapping.length; - } - - @Override - public boolean isNullAt(int pos) { - if (indexMapping[pos] < 0) { - return false; - } - return array.isNullAt(indexMapping[pos]); - } - - @Override - public long getLong(int pos) { - if (indexMapping[pos] < 0) { - return notFoundValue; - } - return array.getLong(indexMapping[pos]); - } - - // ============================= Unsupported Methods ================================ - - @Override - public boolean getBoolean(int pos) { - throw new UnsupportedOperationException(); - } - - @Override - public byte getByte(int pos) { - throw new UnsupportedOperationException(); - } - - @Override - public short getShort(int pos) { - throw new UnsupportedOperationException(); - } - - @Override - public int getInt(int pos) { - throw new UnsupportedOperationException(); - } - - @Override - public float getFloat(int pos) { - throw new UnsupportedOperationException(); - } - - @Override - public double getDouble(int pos) { - throw new UnsupportedOperationException(); - } - - @Override - public BinaryString getString(int pos) { - throw new UnsupportedOperationException(); - } - - @Override - public Decimal getDecimal(int pos, int precision, int scale) { - throw new UnsupportedOperationException(); - } - - @Override - public Timestamp getTimestamp(int pos, int precision) { - throw new UnsupportedOperationException(); - } - - @Override - public byte[] getBinary(int pos) { - throw new UnsupportedOperationException(); - } - - @Override - public InternalArray getArray(int pos) { - throw new UnsupportedOperationException(); - } - - @Override - public InternalMap getMap(int pos) { - throw new UnsupportedOperationException(); - } - - @Override - public InternalRow getRow(int pos, int numFields) { - throw new UnsupportedOperationException(); - } - - @Override - public boolean equals(Object o) { - throw new UnsupportedOperationException(); - } - - @Override - public int hashCode() { - throw new UnsupportedOperationException(); - } - - @Override - public String toString() { - throw new UnsupportedOperationException(); - } - - @Override - public boolean[] toBooleanArray() { - throw new UnsupportedOperationException(); - } - - @Override - public byte[] toByteArray() { - throw new UnsupportedOperationException(); - } - - @Override - public short[] toShortArray() { - throw new UnsupportedOperationException(); - } - - @Override - public int[] toIntArray() { - throw new UnsupportedOperationException(); - } - - @Override - public long[] toLongArray() { - throw new UnsupportedOperationException(); - } - - @Override - public float[] toFloatArray() { - throw new UnsupportedOperationException(); - } - - @Override - public double[] toDoubleArray() { - throw new UnsupportedOperationException(); - } - } } diff --git a/paimon-core/src/main/java/org/apache/paimon/stats/SimpleStatsEvolution.java b/paimon-core/src/main/java/org/apache/paimon/stats/SimpleStatsEvolution.java new file mode 100644 index 000000000000..079300a89dd2 --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/stats/SimpleStatsEvolution.java @@ -0,0 +1,282 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.stats; + +import org.apache.paimon.casting.CastFieldGetter; +import org.apache.paimon.casting.CastedRow; +import org.apache.paimon.data.BinaryString; +import org.apache.paimon.data.Decimal; +import org.apache.paimon.data.GenericArray; +import org.apache.paimon.data.GenericRow; +import org.apache.paimon.data.InternalArray; +import org.apache.paimon.data.InternalMap; +import org.apache.paimon.data.InternalRow; +import org.apache.paimon.data.Timestamp; +import org.apache.paimon.format.SimpleColStats; +import org.apache.paimon.types.RowType; +import org.apache.paimon.utils.ProjectedArray; +import org.apache.paimon.utils.ProjectedRow; + +import javax.annotation.Nullable; + +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +/** Converter for array of {@link SimpleColStats}. */ +public class SimpleStatsEvolution { + + private final List fieldNames; + @Nullable private final int[] indexMapping; + @Nullable private final CastFieldGetter[] castFieldGetters; + + private final Map, int[]> indexMappings; + + private final GenericRow emptyValues; + private final GenericArray emptyNullCounts; + + public SimpleStatsEvolution( + RowType rowType, + @Nullable int[] indexMapping, + @Nullable CastFieldGetter[] castFieldGetters) { + this.fieldNames = rowType.getFieldNames(); + this.indexMapping = indexMapping; + this.castFieldGetters = castFieldGetters; + this.indexMappings = new ConcurrentHashMap<>(); + this.emptyValues = new GenericRow(fieldNames.size()); + this.emptyNullCounts = new GenericArray(new Object[fieldNames.size()]); + } + + public Result evolution( + SimpleStats stats, @Nullable Long rowCount, @Nullable List denseFields) { + InternalRow minValues = stats.minValues(); + InternalRow maxValues = stats.maxValues(); + InternalArray nullCounts = stats.nullCounts(); + + if (denseFields != null && denseFields.isEmpty()) { + // optimize for empty dense fields + minValues = emptyValues; + maxValues = emptyValues; + nullCounts = emptyNullCounts; + } else if (denseFields != null) { + int[] denseIndexMapping = + indexMappings.computeIfAbsent( + denseFields, + k -> fieldNames.stream().mapToInt(denseFields::indexOf).toArray()); + minValues = ProjectedRow.from(denseIndexMapping).replaceRow(minValues); + maxValues = ProjectedRow.from(denseIndexMapping).replaceRow(maxValues); + nullCounts = ProjectedArray.from(denseIndexMapping).replaceArray(nullCounts); + } + + if (indexMapping != null) { + minValues = ProjectedRow.from(indexMapping).replaceRow(minValues); + maxValues = ProjectedRow.from(indexMapping).replaceRow(maxValues); + + if (rowCount == null) { + throw new RuntimeException("Schema Evolution for stats needs row count."); + } + + nullCounts = new NullCountsEvoArray(indexMapping, nullCounts, rowCount); + } + + if (castFieldGetters != null) { + minValues = CastedRow.from(castFieldGetters).replaceRow(minValues); + maxValues = CastedRow.from(castFieldGetters).replaceRow(maxValues); + } + + return new Result(minValues, maxValues, nullCounts); + } + + /** Result to {@link SimpleStats} evolution. */ + public static class Result { + + private final InternalRow minValues; + private final InternalRow maxValues; + private final InternalArray nullCounts; + + public Result(InternalRow minValues, InternalRow maxValues, InternalArray nullCounts) { + this.minValues = minValues; + this.maxValues = maxValues; + this.nullCounts = nullCounts; + } + + public InternalRow minValues() { + return minValues; + } + + public InternalRow maxValues() { + return maxValues; + } + + public InternalArray nullCounts() { + return nullCounts; + } + } + + private static class NullCountsEvoArray implements InternalArray { + + private final int[] indexMapping; + private final InternalArray array; + private final long notFoundValue; + + private NullCountsEvoArray(int[] indexMapping, InternalArray array, long notFoundValue) { + this.indexMapping = indexMapping; + this.array = array; + this.notFoundValue = notFoundValue; + } + + @Override + public int size() { + return indexMapping.length; + } + + @Override + public boolean isNullAt(int pos) { + if (indexMapping[pos] < 0) { + return false; + } + return array.isNullAt(indexMapping[pos]); + } + + @Override + public long getLong(int pos) { + if (indexMapping[pos] < 0) { + return notFoundValue; + } + return array.getLong(indexMapping[pos]); + } + + // ============================= Unsupported Methods ================================ + + @Override + public boolean getBoolean(int pos) { + throw new UnsupportedOperationException(); + } + + @Override + public byte getByte(int pos) { + throw new UnsupportedOperationException(); + } + + @Override + public short getShort(int pos) { + throw new UnsupportedOperationException(); + } + + @Override + public int getInt(int pos) { + throw new UnsupportedOperationException(); + } + + @Override + public float getFloat(int pos) { + throw new UnsupportedOperationException(); + } + + @Override + public double getDouble(int pos) { + throw new UnsupportedOperationException(); + } + + @Override + public BinaryString getString(int pos) { + throw new UnsupportedOperationException(); + } + + @Override + public Decimal getDecimal(int pos, int precision, int scale) { + throw new UnsupportedOperationException(); + } + + @Override + public Timestamp getTimestamp(int pos, int precision) { + throw new UnsupportedOperationException(); + } + + @Override + public byte[] getBinary(int pos) { + throw new UnsupportedOperationException(); + } + + @Override + public InternalArray getArray(int pos) { + throw new UnsupportedOperationException(); + } + + @Override + public InternalMap getMap(int pos) { + throw new UnsupportedOperationException(); + } + + @Override + public InternalRow getRow(int pos, int numFields) { + throw new UnsupportedOperationException(); + } + + @Override + public boolean equals(Object o) { + throw new UnsupportedOperationException(); + } + + @Override + public int hashCode() { + throw new UnsupportedOperationException(); + } + + @Override + public String toString() { + throw new UnsupportedOperationException(); + } + + @Override + public boolean[] toBooleanArray() { + throw new UnsupportedOperationException(); + } + + @Override + public byte[] toByteArray() { + throw new UnsupportedOperationException(); + } + + @Override + public short[] toShortArray() { + throw new UnsupportedOperationException(); + } + + @Override + public int[] toIntArray() { + throw new UnsupportedOperationException(); + } + + @Override + public long[] toLongArray() { + throw new UnsupportedOperationException(); + } + + @Override + public float[] toFloatArray() { + throw new UnsupportedOperationException(); + } + + @Override + public double[] toDoubleArray() { + throw new UnsupportedOperationException(); + } + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/stats/SimpleStatsConverters.java b/paimon-core/src/main/java/org/apache/paimon/stats/SimpleStatsEvolutions.java similarity index 86% rename from paimon-core/src/main/java/org/apache/paimon/stats/SimpleStatsConverters.java rename to paimon-core/src/main/java/org/apache/paimon/stats/SimpleStatsEvolutions.java index 10694769b62c..a0814b8c04c4 100644 --- a/paimon-core/src/main/java/org/apache/paimon/stats/SimpleStatsConverters.java +++ b/paimon-core/src/main/java/org/apache/paimon/stats/SimpleStatsEvolutions.java @@ -37,28 +37,29 @@ import static org.apache.paimon.schema.SchemaEvolutionUtil.createIndexCastMapping; /** Converters to create col stats array serializer. */ -public class SimpleStatsConverters { +public class SimpleStatsEvolutions { private final Function> schemaFields; private final long tableSchemaId; private final List tableDataFields; private final AtomicReference> tableFields; - private final ConcurrentMap converters; + private final ConcurrentMap evolutions; - public SimpleStatsConverters(Function> schemaFields, long tableSchemaId) { + public SimpleStatsEvolutions(Function> schemaFields, long tableSchemaId) { this.schemaFields = schemaFields; this.tableSchemaId = tableSchemaId; this.tableDataFields = schemaFields.apply(tableSchemaId); this.tableFields = new AtomicReference<>(); - this.converters = new ConcurrentHashMap<>(); + this.evolutions = new ConcurrentHashMap<>(); } - public SimpleStatsConverter getOrCreate(long dataSchemaId) { - return converters.computeIfAbsent( + public SimpleStatsEvolution getOrCreate(long dataSchemaId) { + return evolutions.computeIfAbsent( dataSchemaId, id -> { if (tableSchemaId == id) { - return new SimpleStatsConverter(new RowType(schemaFields.apply(id))); + return new SimpleStatsEvolution( + new RowType(schemaFields.apply(id)), null, null); } // Get atomic schema fields. @@ -66,10 +67,10 @@ public SimpleStatsConverter getOrCreate(long dataSchemaId) { tableFields.updateAndGet(v -> v == null ? tableDataFields : v); List dataFields = schemaFields.apply(id); IndexCastMapping indexCastMapping = - createIndexCastMapping(schemaTableFields, dataFields); + createIndexCastMapping(schemaTableFields, schemaFields.apply(id)); @Nullable int[] indexMapping = indexCastMapping.getIndexMapping(); // Create col stats array serializer with schema evolution - return new SimpleStatsConverter( + return new SimpleStatsEvolution( new RowType(dataFields), indexMapping, indexCastMapping.getCastMapping()); diff --git a/paimon-core/src/main/java/org/apache/paimon/stats/StatsFileHandler.java b/paimon-core/src/main/java/org/apache/paimon/stats/StatsFileHandler.java index f9e057c7cbb3..5cb88f7257a7 100644 --- a/paimon-core/src/main/java/org/apache/paimon/stats/StatsFileHandler.java +++ b/paimon-core/src/main/java/org/apache/paimon/stats/StatsFileHandler.java @@ -71,13 +71,14 @@ public Optional readStats(long snapshotId) { } public Optional readStats(Snapshot snapshot) { - if (snapshot.statistics() == null) { - return Optional.empty(); - } else { - Statistics stats = statsFile.read(snapshot.statistics()); - stats.deserializeFieldsFromString(schemaManager.schema(stats.schemaId())); - return Optional.of(stats); - } + String file = snapshot.statistics(); + return file == null ? Optional.empty() : Optional.of(readStats(file)); + } + + public Statistics readStats(String file) { + Statistics stats = statsFile.read(file); + stats.deserializeFieldsFromString(schemaManager.schema(stats.schemaId())); + return stats; } /** Delete stats of the specified snapshot. */ diff --git a/paimon-core/src/main/java/org/apache/paimon/table/AbstractFileStoreTable.java b/paimon-core/src/main/java/org/apache/paimon/table/AbstractFileStoreTable.java index 6aacef6ed841..57966d24ce47 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/AbstractFileStoreTable.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/AbstractFileStoreTable.java @@ -58,6 +58,7 @@ import org.apache.paimon.table.source.snapshot.SnapshotReaderImpl; import org.apache.paimon.table.source.snapshot.StaticFromTimestampStartingScanner; import org.apache.paimon.table.source.snapshot.StaticFromWatermarkStartingScanner; +import org.apache.paimon.table.source.snapshot.TimeTravelUtil; import org.apache.paimon.tag.TagPreview; import org.apache.paimon.utils.BranchManager; import org.apache.paimon.utils.Preconditions; @@ -65,9 +66,10 @@ import org.apache.paimon.utils.SimpleFileReader; import org.apache.paimon.utils.SnapshotManager; import org.apache.paimon.utils.SnapshotNotExistException; -import org.apache.paimon.utils.StringUtils; import org.apache.paimon.utils.TagManager; +import org.apache.paimon.shade.caffeine2.com.github.benmanes.caffeine.cache.Cache; + import javax.annotation.Nullable; import java.io.IOException; @@ -90,6 +92,7 @@ abstract class AbstractFileStoreTable implements FileStoreTable { private static final long serialVersionUID = 1L; + private static final String WATERMARK_PREFIX = "watermark-"; protected final FileIO fileIO; @@ -97,6 +100,10 @@ abstract class AbstractFileStoreTable implements FileStoreTable { protected final TableSchema tableSchema; protected final CatalogEnvironment catalogEnvironment; + @Nullable protected transient SegmentsCache manifestCache; + @Nullable protected transient Cache snapshotCache; + @Nullable protected transient Cache statsCache; + protected AbstractFileStoreTable( FileIO fileIO, Path path, @@ -120,9 +127,21 @@ public String currentBranch() { @Override public void setManifestCache(SegmentsCache manifestCache) { + this.manifestCache = manifestCache; store().setManifestCache(manifestCache); } + @Override + public void setSnapshotCache(Cache cache) { + this.snapshotCache = cache; + store().setSnapshotCache(cache); + } + + @Override + public void setStatsCache(Cache cache) { + this.statsCache = cache; + } + @Override public OptionalLong latestSnapshotId() { Long snapshot = store().snapshotManager().latestSnapshotId(); @@ -168,28 +187,33 @@ public Identifier identifier() { } @Override - public Optional statistics() { - Snapshot latestSnapshot; - Long snapshotId = coreOptions().scanSnapshotId(); - String tagName = coreOptions().scanTagName(); - - if (snapshotId == null) { - if (!StringUtils.isEmpty(tagName) && tagManager().tagExists(tagName)) { - return store().newStatsFileHandler() - .readStats(tagManager().tag(tagName).trimToSnapshot()); - } else { - snapshotId = snapshotManager().latestSnapshotId(); - } - } - - if (snapshotId != null && snapshotManager().snapshotExists(snapshotId)) { - latestSnapshot = snapshotManager().snapshot(snapshotId); - } else { - latestSnapshot = snapshotManager().latestSnapshot(); + public String uuid() { + if (catalogEnvironment.uuid() != null) { + return catalogEnvironment.uuid(); } + long earliestCreationTime = schemaManager().earliestCreationTime(); + return fullName() + "." + earliestCreationTime; + } - if (latestSnapshot != null) { - return store().newStatsFileHandler().readStats(latestSnapshot); + @Override + public Optional statistics() { + Snapshot snapshot = TimeTravelUtil.resolveSnapshot(this); + if (snapshot != null) { + String file = snapshot.statistics(); + if (file == null) { + return Optional.empty(); + } + if (statsCache != null) { + Statistics stats = statsCache.getIfPresent(file); + if (stats != null) { + return Optional.of(stats); + } + } + Statistics stats = store().newStatsFileHandler().readStats(file); + if (statsCache != null) { + statsCache.put(file, stats); + } + return Optional.of(stats); } return Optional.empty(); } @@ -344,12 +368,26 @@ public FileStoreTable copyWithLatestSchema() { @Override public FileStoreTable copy(TableSchema newTableSchema) { - return newTableSchema.primaryKeys().isEmpty() - ? new AppendOnlyFileStoreTable(fileIO, path, newTableSchema, catalogEnvironment) - : new PrimaryKeyFileStoreTable(fileIO, path, newTableSchema, catalogEnvironment); + AbstractFileStoreTable copied = + newTableSchema.primaryKeys().isEmpty() + ? new AppendOnlyFileStoreTable( + fileIO, path, newTableSchema, catalogEnvironment) + : new PrimaryKeyFileStoreTable( + fileIO, path, newTableSchema, catalogEnvironment); + if (snapshotCache != null) { + copied.setSnapshotCache(snapshotCache); + } + if (manifestCache != null) { + copied.setManifestCache(manifestCache); + } + if (statsCache != null) { + copied.setStatsCache(statsCache); + } + return copied; } - protected SchemaManager schemaManager() { + @Override + public SchemaManager schemaManager() { return new SchemaManager(fileIO(), path, currentBranch()); } @@ -594,6 +632,19 @@ public void renameTag(String tagName, String targetTagName) { tagManager().renameTag(tagName, targetTagName); } + @Override + public void replaceTag( + String tagName, @Nullable Long fromSnapshotId, @Nullable Duration timeRetained) { + if (fromSnapshotId == null) { + Snapshot latestSnapshot = snapshotManager().latestSnapshot(); + SnapshotNotExistException.checkNotNull( + latestSnapshot, "Cannot replace tag because latest snapshot doesn't exist."); + tagManager().replaceTag(latestSnapshot, tagName, timeRetained); + } else { + tagManager().replaceTag(findSnapshot(fromSnapshotId), tagName, timeRetained); + } + } + @Override public void deleteTag(String tagName) { tagManager() diff --git a/paimon-core/src/main/java/org/apache/paimon/table/AppendOnlyFileStoreTable.java b/paimon-core/src/main/java/org/apache/paimon/table/AppendOnlyFileStoreTable.java index b93cf8e44237..103fa64050aa 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/AppendOnlyFileStoreTable.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/AppendOnlyFileStoreTable.java @@ -24,6 +24,7 @@ import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.Path; import org.apache.paimon.iceberg.AppendOnlyIcebergCommitCallback; +import org.apache.paimon.iceberg.IcebergOptions; import org.apache.paimon.manifest.ManifestCacheFilter; import org.apache.paimon.operation.AppendOnlyFileStoreScan; import org.apache.paimon.operation.AppendOnlyFileStoreWrite; @@ -165,7 +166,8 @@ protected List createCommitCallbacks(String commitUser) { List callbacks = super.createCommitCallbacks(commitUser); CoreOptions options = coreOptions(); - if (options.metadataIcebergCompatible()) { + if (options.toConfiguration().get(IcebergOptions.METADATA_ICEBERG_STORAGE) + != IcebergOptions.StorageType.DISABLED) { callbacks.add(new AppendOnlyIcebergCommitCallback(this, commitUser)); } diff --git a/paimon-core/src/main/java/org/apache/paimon/table/CatalogEnvironment.java b/paimon-core/src/main/java/org/apache/paimon/table/CatalogEnvironment.java index ebaff1266155..a722d9e21ada 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/CatalogEnvironment.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/CatalogEnvironment.java @@ -19,7 +19,6 @@ package org.apache.paimon.table; import org.apache.paimon.catalog.Identifier; -import org.apache.paimon.lineage.LineageMetaFactory; import org.apache.paimon.metastore.MetastoreClient; import org.apache.paimon.operation.Lock; @@ -27,32 +26,29 @@ import java.io.Serializable; -/** - * Catalog environment in table which contains log factory, metastore client factory and lineage - * meta. - */ +/** Catalog environment in table which contains log factory, metastore client factory. */ public class CatalogEnvironment implements Serializable { private static final long serialVersionUID = 1L; @Nullable private final Identifier identifier; + @Nullable private final String uuid; private final Lock.Factory lockFactory; @Nullable private final MetastoreClient.Factory metastoreClientFactory; - @Nullable private final LineageMetaFactory lineageMetaFactory; public CatalogEnvironment( @Nullable Identifier identifier, + @Nullable String uuid, Lock.Factory lockFactory, - @Nullable MetastoreClient.Factory metastoreClientFactory, - @Nullable LineageMetaFactory lineageMetaFactory) { + @Nullable MetastoreClient.Factory metastoreClientFactory) { this.identifier = identifier; + this.uuid = uuid; this.lockFactory = lockFactory; this.metastoreClientFactory = metastoreClientFactory; - this.lineageMetaFactory = lineageMetaFactory; } public static CatalogEnvironment empty() { - return new CatalogEnvironment(null, Lock.emptyFactory(), null, null); + return new CatalogEnvironment(null, null, Lock.emptyFactory(), null); } @Nullable @@ -60,6 +56,11 @@ public Identifier identifier() { return identifier; } + @Nullable + public String uuid() { + return uuid; + } + public Lock.Factory lockFactory() { return lockFactory; } @@ -68,9 +69,4 @@ public Lock.Factory lockFactory() { public MetastoreClient.Factory metastoreClientFactory() { return metastoreClientFactory; } - - @Nullable - public LineageMetaFactory lineageMetaFactory() { - return lineageMetaFactory; - } } diff --git a/paimon-core/src/main/java/org/apache/paimon/table/DataTable.java b/paimon-core/src/main/java/org/apache/paimon/table/DataTable.java index e330db0e04a4..7979daccf756 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/DataTable.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/DataTable.java @@ -21,6 +21,7 @@ import org.apache.paimon.CoreOptions; import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.Path; +import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.table.source.DataTableScan; import org.apache.paimon.table.source.snapshot.SnapshotReader; import org.apache.paimon.utils.BranchManager; @@ -39,6 +40,8 @@ public interface DataTable extends InnerTable { SnapshotManager snapshotManager(); + SchemaManager schemaManager(); + TagManager tagManager(); BranchManager branchManager(); diff --git a/paimon-core/src/main/java/org/apache/paimon/table/DelegatedFileStoreTable.java b/paimon-core/src/main/java/org/apache/paimon/table/DelegatedFileStoreTable.java index 5d6331aa414e..0a548941bedc 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/DelegatedFileStoreTable.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/DelegatedFileStoreTable.java @@ -27,6 +27,7 @@ import org.apache.paimon.manifest.ManifestCacheFilter; import org.apache.paimon.manifest.ManifestEntry; import org.apache.paimon.manifest.ManifestFileMeta; +import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.schema.TableSchema; import org.apache.paimon.stats.Statistics; import org.apache.paimon.table.query.LocalTableQuery; @@ -44,6 +45,8 @@ import org.apache.paimon.utils.SnapshotManager; import org.apache.paimon.utils.TagManager; +import org.apache.paimon.shade.caffeine2.com.github.benmanes.caffeine.cache.Cache; + import java.time.Duration; import java.util.Objects; import java.util.Optional; @@ -72,6 +75,11 @@ public String fullName() { return wrapped.fullName(); } + @Override + public String uuid() { + return wrapped.uuid(); + } + @Override public SnapshotReader newSnapshotReader() { return wrapped.newSnapshotReader(); @@ -87,6 +95,11 @@ public SnapshotManager snapshotManager() { return wrapped.snapshotManager(); } + @Override + public SchemaManager schemaManager() { + return wrapped.schemaManager(); + } + @Override public TagManager tagManager() { return wrapped.tagManager(); @@ -112,6 +125,16 @@ public void setManifestCache(SegmentsCache manifestCache) { wrapped.setManifestCache(manifestCache); } + @Override + public void setSnapshotCache(Cache cache) { + wrapped.setSnapshotCache(cache); + } + + @Override + public void setStatsCache(Cache cache) { + wrapped.setStatsCache(cache); + } + @Override public TableSchema schema() { return wrapped.schema(); @@ -187,6 +210,11 @@ public void renameTag(String tagName, String targetTagName) { wrapped.renameTag(tagName, targetTagName); } + @Override + public void replaceTag(String tagName, Long fromSnapshotId, Duration timeRetained) { + wrapped.replaceTag(tagName, fromSnapshotId, timeRetained); + } + @Override public void deleteTag(String tagName) { wrapped.deleteTag(tagName); diff --git a/paimon-core/src/main/java/org/apache/paimon/table/ExpireChangelogImpl.java b/paimon-core/src/main/java/org/apache/paimon/table/ExpireChangelogImpl.java index 246a85cbdf03..1ffa7485aee5 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/ExpireChangelogImpl.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/ExpireChangelogImpl.java @@ -21,7 +21,7 @@ import org.apache.paimon.Changelog; import org.apache.paimon.Snapshot; import org.apache.paimon.consumer.ConsumerManager; -import org.apache.paimon.manifest.ManifestEntry; +import org.apache.paimon.manifest.ExpireFileEntry; import org.apache.paimon.operation.ChangelogDeletion; import org.apache.paimon.options.ExpireConfig; import org.apache.paimon.utils.Preconditions; @@ -37,6 +37,8 @@ import java.util.Set; import java.util.function.Predicate; +import static org.apache.paimon.table.ExpireSnapshotsImpl.findSkippingTags; + /** Cleanup the changelog in changelog directory. */ public class ExpireChangelogImpl implements ExpireSnapshots { @@ -137,8 +139,7 @@ public int expireUntil(long earliestId, long endExclusiveId) { List taggedSnapshots = tagManager.taggedSnapshots(); List skippingSnapshots = - SnapshotManager.findOverlappedSnapshots( - taggedSnapshots, earliestId, endExclusiveId); + findSkippingTags(taggedSnapshots, earliestId, endExclusiveId); skippingSnapshots.add(snapshotManager.changelog(endExclusiveId)); Set manifestSkippSet = changelogDeletion.manifestSkippingSet(skippingSnapshots); for (long id = earliestId; id < endExclusiveId; id++) { @@ -146,7 +147,7 @@ public int expireUntil(long earliestId, long endExclusiveId) { LOG.debug("Ready to delete changelog files from changelog #" + id); } Changelog changelog = snapshotManager.longLivedChangelog(id); - Predicate skipper; + Predicate skipper; try { skipper = changelogDeletion.createDataFileSkipperForTags(taggedSnapshots, id); } catch (Exception e) { diff --git a/paimon-core/src/main/java/org/apache/paimon/table/ExpireSnapshotsImpl.java b/paimon-core/src/main/java/org/apache/paimon/table/ExpireSnapshotsImpl.java index eb163f8fa8c7..dc1c2d6bdbc5 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/ExpireSnapshotsImpl.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/ExpireSnapshotsImpl.java @@ -22,7 +22,7 @@ import org.apache.paimon.Snapshot; import org.apache.paimon.annotation.VisibleForTesting; import org.apache.paimon.consumer.ConsumerManager; -import org.apache.paimon.manifest.ManifestEntry; +import org.apache.paimon.manifest.ExpireFileEntry; import org.apache.paimon.operation.SnapshotDeletion; import org.apache.paimon.options.ExpireConfig; import org.apache.paimon.utils.Preconditions; @@ -35,11 +35,15 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.io.UncheckedIOException; +import java.util.ArrayList; import java.util.HashSet; import java.util.List; import java.util.Set; import java.util.function.Predicate; +import static org.apache.paimon.utils.SnapshotManager.findPreviousOrEqualSnapshot; +import static org.apache.paimon.utils.SnapshotManager.findPreviousSnapshot; + /** An implementation for {@link ExpireSnapshots}. */ public class ExpireSnapshotsImpl implements ExpireSnapshots { @@ -172,7 +176,7 @@ public int expireUntil(long earliestId, long endExclusiveId) { continue; } // expire merge tree files and collect changed buckets - Predicate skipper; + Predicate skipper; try { skipper = snapshotDeletion.createDataFileSkipperForTags(taggedSnapshots, id); } catch (Exception e) { @@ -212,8 +216,7 @@ public int expireUntil(long earliestId, long endExclusiveId) { // delete manifests and indexFiles List skippingSnapshots = - SnapshotManager.findOverlappedSnapshots( - taggedSnapshots, beginInclusiveId, endExclusiveId); + findSkippingTags(taggedSnapshots, beginInclusiveId, endExclusiveId); try { skippingSnapshots.add(snapshotManager.tryGetSnapshot(endExclusiveId)); @@ -277,4 +280,18 @@ private void writeEarliestHint(long earliest) { public SnapshotDeletion snapshotDeletion() { return snapshotDeletion; } + + /** Find the skipping tags in sortedTags for range of [beginInclusive, endExclusive). */ + public static List findSkippingTags( + List sortedTags, long beginInclusive, long endExclusive) { + List overlappedSnapshots = new ArrayList<>(); + int right = findPreviousSnapshot(sortedTags, endExclusive); + if (right >= 0) { + int left = Math.max(findPreviousOrEqualSnapshot(sortedTags, beginInclusive), 0); + for (int i = left; i <= right; i++) { + overlappedSnapshots.add(sortedTags.get(i)); + } + } + return overlappedSnapshots; + } } diff --git a/paimon-core/src/main/java/org/apache/paimon/table/FallbackReadFileStoreTable.java b/paimon-core/src/main/java/org/apache/paimon/table/FallbackReadFileStoreTable.java index f1a60b9713f9..e3e290f06086 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/FallbackReadFileStoreTable.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/FallbackReadFileStoreTable.java @@ -28,7 +28,6 @@ import org.apache.paimon.options.Options; import org.apache.paimon.predicate.Predicate; import org.apache.paimon.reader.RecordReader; -import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.schema.TableSchema; import org.apache.paimon.table.source.DataFilePlan; import org.apache.paimon.table.source.DataSplit; @@ -103,7 +102,7 @@ public FileStoreTable switchToBranch(String branchName) { private FileStoreTable switchWrappedToBranch(String branchName) { Optional optionalSchema = - new SchemaManager(wrapped.fileIO(), wrapped.location(), branchName).latest(); + wrapped.schemaManager().copyWithBranch(branchName).latest(); Preconditions.checkArgument( optionalSchema.isPresent(), "Branch " + branchName + " does not exist"); diff --git a/paimon-core/src/main/java/org/apache/paimon/table/FileStoreTable.java b/paimon-core/src/main/java/org/apache/paimon/table/FileStoreTable.java index 3bd337294b4f..61aa77d5f36a 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/FileStoreTable.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/FileStoreTable.java @@ -19,12 +19,12 @@ package org.apache.paimon.table; import org.apache.paimon.FileStore; +import org.apache.paimon.Snapshot; import org.apache.paimon.data.InternalRow; import org.apache.paimon.fs.Path; -import org.apache.paimon.io.DataFileMeta; import org.apache.paimon.manifest.ManifestCacheFilter; import org.apache.paimon.schema.TableSchema; -import org.apache.paimon.stats.SimpleStats; +import org.apache.paimon.stats.Statistics; import org.apache.paimon.table.query.LocalTableQuery; import org.apache.paimon.table.sink.RowKeyExtractor; import org.apache.paimon.table.sink.TableCommitImpl; @@ -32,6 +32,8 @@ import org.apache.paimon.types.RowType; import org.apache.paimon.utils.SegmentsCache; +import org.apache.paimon.shade.caffeine2.com.github.benmanes.caffeine.cache.Cache; + import java.util.List; import java.util.Map; import java.util.Optional; @@ -44,6 +46,10 @@ public interface FileStoreTable extends DataTable { void setManifestCache(SegmentsCache manifestCache); + void setSnapshotCache(Cache cache); + + void setStatsCache(Cache cache); + @Override default RowType rowType() { return schema().logicalRowType(); @@ -104,10 +110,6 @@ default Optional comment() { LocalTableQuery newLocalTableQuery(); - default SimpleStats getSchemaFieldStats(DataFileMeta dataFileMeta) { - return dataFileMeta.valueStats(); - } - boolean supportStreamingReadOverwrite(); RowKeyExtractor createRowKeyExtractor(); diff --git a/paimon-core/src/main/java/org/apache/paimon/table/FileStoreTableFactory.java b/paimon-core/src/main/java/org/apache/paimon/table/FileStoreTableFactory.java index 19c87cc7467c..423dc1726319 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/FileStoreTableFactory.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/FileStoreTableFactory.java @@ -29,8 +29,10 @@ import java.io.IOException; import java.io.UncheckedIOException; +import java.util.Optional; import static org.apache.paimon.CoreOptions.PATH; +import static org.apache.paimon.utils.Preconditions.checkArgument; /** Factory to create {@link FileStoreTable}. */ public class FileStoreTableFactory { @@ -93,13 +95,17 @@ public static FileStoreTable create( if (!StringUtils.isNullOrWhitespaceOnly(fallbackBranch)) { Options branchOptions = new Options(dynamicOptions.toMap()); branchOptions.set(CoreOptions.BRANCH, fallbackBranch); + Optional schema = + new SchemaManager(fileIO, tablePath, fallbackBranch).latest(); + checkArgument( + schema.isPresent(), + "Cannot set '%s' = '%s' because the branch '%s' isn't existed.", + CoreOptions.SCAN_FALLBACK_BRANCH.key(), + fallbackBranch, + fallbackBranch); FileStoreTable fallbackTable = createWithoutFallbackBranch( - fileIO, - tablePath, - new SchemaManager(fileIO, tablePath, fallbackBranch).latest().get(), - branchOptions, - catalogEnvironment); + fileIO, tablePath, schema.get(), branchOptions, catalogEnvironment); table = new FallbackReadFileStoreTable(table, fallbackTable); } diff --git a/paimon-core/src/main/java/org/apache/paimon/table/FormatTable.java b/paimon-core/src/main/java/org/apache/paimon/table/FormatTable.java index 3224131d4afd..a4c7788c38af 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/FormatTable.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/FormatTable.java @@ -34,6 +34,7 @@ import javax.annotation.Nullable; import java.time.Duration; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -70,6 +71,19 @@ enum Format { CSV } + /** Parses a file format string to a corresponding {@link Format} enum constant. */ + static Format parseFormat(String fileFormat) { + try { + return Format.valueOf(fileFormat.toUpperCase()); + } catch (IllegalArgumentException e) { + throw new UnsupportedOperationException( + "Format table unsupported file format: " + + fileFormat + + ". Supported formats: " + + Arrays.toString(Format.values())); + } + } + /** Create a new builder for {@link FormatTable}. */ static FormatTable.Builder builder() { return new FormatTable.Builder(); @@ -271,6 +285,11 @@ default void renameTag(String tagName, String targetTagName) { throw new UnsupportedOperationException(); } + @Override + default void replaceTag(String tagName, Long fromSnapshotId, Duration timeRetained) { + throw new UnsupportedOperationException(); + } + @Override default void deleteTag(String tagName) { throw new UnsupportedOperationException(); diff --git a/paimon-core/src/main/java/org/apache/paimon/table/FormatTableOptions.java b/paimon-core/src/main/java/org/apache/paimon/table/FormatTableOptions.java index 64f134d07588..b4010209c32a 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/FormatTableOptions.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/FormatTableOptions.java @@ -25,8 +25,9 @@ public class FormatTableOptions { public static final ConfigOption FIELD_DELIMITER = - ConfigOptions.key("csv.field-delimiter") + ConfigOptions.key("field-delimiter") .stringType() .defaultValue(",") - .withDescription("Optional field delimiter character (',' by default)"); + .withDescription( + "Optional field delimiter character for CSV (',' by default)."); } diff --git a/paimon-core/src/main/java/org/apache/paimon/table/PrimaryKeyFileStoreTable.java b/paimon-core/src/main/java/org/apache/paimon/table/PrimaryKeyFileStoreTable.java index 9c15a7bd1f3d..516ae766cef8 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/PrimaryKeyFileStoreTable.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/PrimaryKeyFileStoreTable.java @@ -23,13 +23,13 @@ import org.apache.paimon.KeyValueFileStore; import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.Path; +import org.apache.paimon.iceberg.IcebergOptions; import org.apache.paimon.iceberg.PrimaryKeyIcebergCommitCallback; import org.apache.paimon.manifest.ManifestCacheFilter; import org.apache.paimon.mergetree.compact.LookupMergeFunction; import org.apache.paimon.mergetree.compact.MergeFunctionFactory; import org.apache.paimon.operation.FileStoreScan; import org.apache.paimon.operation.KeyValueFileStoreScan; -import org.apache.paimon.options.Options; import org.apache.paimon.predicate.Predicate; import org.apache.paimon.schema.KeyValueFieldsExtractor; import org.apache.paimon.schema.TableSchema; @@ -72,8 +72,7 @@ class PrimaryKeyFileStoreTable extends AbstractFileStoreTable { public KeyValueFileStore store() { if (lazyStore == null) { RowType rowType = tableSchema.logicalRowType(); - Options conf = Options.fromMap(tableSchema.options()); - CoreOptions options = new CoreOptions(conf); + CoreOptions options = CoreOptions.fromMap(tableSchema.options()); KeyValueFieldsExtractor extractor = PrimaryKeyTableUtils.PrimaryKeyFieldsExtractor.EXTRACTOR; @@ -185,7 +184,8 @@ protected List createCommitCallbacks(String commitUser) { List callbacks = super.createCommitCallbacks(commitUser); CoreOptions options = coreOptions(); - if (options.metadataIcebergCompatible()) { + if (options.toConfiguration().get(IcebergOptions.METADATA_ICEBERG_STORAGE) + != IcebergOptions.StorageType.DISABLED) { callbacks.add(new PrimaryKeyIcebergCommitCallback(this, commitUser)); } diff --git a/paimon-core/src/main/java/org/apache/paimon/table/PrimaryKeyTableUtils.java b/paimon-core/src/main/java/org/apache/paimon/table/PrimaryKeyTableUtils.java index 58cde07c382c..d156d23a918a 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/PrimaryKeyTableUtils.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/PrimaryKeyTableUtils.java @@ -34,8 +34,8 @@ import java.util.List; import java.util.stream.Collectors; -import static org.apache.paimon.table.SystemFields.KEY_FIELD_ID_START; -import static org.apache.paimon.table.SystemFields.KEY_FIELD_PREFIX; +import static org.apache.paimon.table.SpecialFields.KEY_FIELD_ID_START; +import static org.apache.paimon.table.SpecialFields.KEY_FIELD_PREFIX; /** Utils for creating changelog table with primary keys. */ public class PrimaryKeyTableUtils { diff --git a/paimon-core/src/main/java/org/apache/paimon/table/ReadonlyTable.java b/paimon-core/src/main/java/org/apache/paimon/table/ReadonlyTable.java index 4ae593b5577f..fe5ebbfcd148 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/ReadonlyTable.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/ReadonlyTable.java @@ -197,6 +197,14 @@ default void renameTag(String tagName, String targetTagName) { this.getClass().getSimpleName())); } + @Override + default void replaceTag(String tagName, Long fromSnapshotId, Duration timeRetained) { + throw new UnsupportedOperationException( + String.format( + "Readonly Table %s does not support replaceTag.", + this.getClass().getSimpleName())); + } + @Override default void deleteTag(String tagName) { throw new UnsupportedOperationException( diff --git a/paimon-core/src/main/java/org/apache/paimon/table/RollbackHelper.java b/paimon-core/src/main/java/org/apache/paimon/table/RollbackHelper.java index 374d71e9e2aa..29fecec11353 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/RollbackHelper.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/RollbackHelper.java @@ -21,7 +21,7 @@ import org.apache.paimon.Changelog; import org.apache.paimon.Snapshot; import org.apache.paimon.fs.FileIO; -import org.apache.paimon.manifest.ManifestEntry; +import org.apache.paimon.manifest.ExpireFileEntry; import org.apache.paimon.operation.ChangelogDeletion; import org.apache.paimon.operation.SnapshotDeletion; import org.apache.paimon.operation.TagDeletion; @@ -98,13 +98,6 @@ public void cleanLargerThan(Snapshot retainedSnapshot) { } tagDeletion.cleanUnusedManifests(snapshot, manifestsSkippingSet); } - - // modify the latest hint - try { - snapshotManager.commitLatestHint(retainedSnapshot.id()); - } catch (IOException e) { - throw new UncheckedIOException(e); - } } private List cleanSnapshotsDataFiles(Snapshot retainedSnapshot) { @@ -114,6 +107,13 @@ private List cleanSnapshotsDataFiles(Snapshot retainedSnapshot) { long latest = checkNotNull(snapshotManager.latestSnapshotId(), "Cannot find latest snapshot."); + // modify the latest hint + try { + snapshotManager.commitLatestHint(retainedSnapshot.id()); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + // delete snapshot files first, cannot be read now // it is possible that some snapshots have been expired List toBeCleaned = new ArrayList<>(); @@ -149,24 +149,13 @@ private List cleanLongLivedChangelogDataFiles(Snapshot retainedSnapsh return Collections.emptyList(); } - // delete changelog files first, cannot be read now // it is possible that some snapshots have been expired List toBeCleaned = new ArrayList<>(); long to = Math.max(earliest, retainedSnapshot.id() + 1); for (long i = latest; i >= to; i--) { toBeCleaned.add(snapshotManager.changelog(i)); - fileIO.deleteQuietly(snapshotManager.longLivedChangelogPath(i)); - } - - // delete data files of changelog - for (Changelog changelog : toBeCleaned) { - // clean the deleted file - changelogDeletion.cleanUnusedDataFiles(changelog, manifestEntry -> false); } - // delete directories - snapshotDeletion.cleanEmptyDirectories(); - // modify the latest hint try { if (toBeCleaned.size() > 0) { @@ -182,6 +171,17 @@ private List cleanLongLivedChangelogDataFiles(Snapshot retainedSnapsh throw new UncheckedIOException(e); } + // delete data files of changelog + for (Changelog changelog : toBeCleaned) { + // delete changelog files first, cannot be read now + fileIO.deleteQuietly(snapshotManager.longLivedChangelogPath(changelog.id())); + // clean the deleted file + changelogDeletion.cleanUnusedDataFiles(changelog, manifestEntry -> false); + } + + // delete directories + snapshotDeletion.cleanEmptyDirectories(); + return toBeCleaned; } @@ -205,7 +205,7 @@ private List cleanTagsDataFiles(Snapshot retainedSnapshot) { } // delete data files - Predicate dataFileSkipper = null; + Predicate dataFileSkipper = null; boolean success = true; try { dataFileSkipper = tagDeletion.dataFileSkipper(retainedSnapshot); diff --git a/paimon-core/src/main/java/org/apache/paimon/table/Table.java b/paimon-core/src/main/java/org/apache/paimon/table/Table.java index 613dfca3158a..7ed7ba48a8eb 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/Table.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/Table.java @@ -33,6 +33,7 @@ import java.io.Serializable; import java.time.Duration; +import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.Optional; @@ -51,10 +52,19 @@ public interface Table extends Serializable { /** A name to identify this table. */ String name(); + /** Full name of the table, default is database.tableName. */ default String fullName() { return name(); } + /** + * UUID of the table, metastore can provide the true UUID of this table, default is the full + * name. + */ + default String uuid() { + return fullName(); + } + /** Returns the row type of this table. */ RowType rowType(); @@ -120,14 +130,20 @@ default String fullName() { @Experimental void renameTag(String tagName, String targetTagName); + /** Replace a tag with new snapshot id and new time retained. */ + @Experimental + void replaceTag(String tagName, Long fromSnapshotId, Duration timeRetained); + /** Delete a tag by name. */ @Experimental void deleteTag(String tagName); /** Delete tags, tags are separated by commas. */ @Experimental - default void deleteTags(String tagNames) { - for (String tagName : tagNames.split(",")) { + default void deleteTags(String tagStr) { + String[] tagNames = + Arrays.stream(tagStr.split(",")).map(String::trim).toArray(String[]::new); + for (String tagName : tagNames) { deleteTag(tagName); } } diff --git a/paimon-core/src/main/java/org/apache/paimon/table/object/ObjectRefresh.java b/paimon-core/src/main/java/org/apache/paimon/table/object/ObjectRefresh.java new file mode 100644 index 000000000000..b1be840c5153 --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/table/object/ObjectRefresh.java @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.table.object; + +import org.apache.paimon.data.BinaryString; +import org.apache.paimon.data.GenericMap; +import org.apache.paimon.data.GenericRow; +import org.apache.paimon.data.InternalRow; +import org.apache.paimon.data.Timestamp; +import org.apache.paimon.fs.FileIO; +import org.apache.paimon.fs.FileStatus; +import org.apache.paimon.fs.Path; +import org.apache.paimon.table.sink.BatchTableCommit; +import org.apache.paimon.table.sink.BatchTableWrite; +import org.apache.paimon.table.sink.BatchWriteBuilder; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +/** Util class for refreshing object table. */ +public class ObjectRefresh { + + public static long refresh(ObjectTable table) throws Exception { + String location = table.objectLocation(); + + // 1. collect all files for object table + List fileCollector = new ArrayList<>(); + listAllFiles(table.objectFileIO(), new Path(location), fileCollector); + + // 2. write to underlying table + BatchWriteBuilder writeBuilder = + table.underlyingTable().newBatchWriteBuilder().withOverwrite(); + try (BatchTableWrite write = writeBuilder.newWrite(); + BatchTableCommit commit = writeBuilder.newCommit()) { + for (FileStatus file : fileCollector) { + write.write(toRow(file)); + } + commit.commit(write.prepareCommit()); + } + + return fileCollector.size(); + } + + private static void listAllFiles(FileIO fileIO, Path directory, List fileCollector) + throws IOException { + FileStatus[] files = fileIO.listStatus(directory); + if (files == null) { + return; + } + + for (FileStatus file : files) { + if (file.isDir()) { + listAllFiles(fileIO, file.getPath(), fileCollector); + } else { + fileCollector.add(file); + } + } + } + + private static InternalRow toRow(FileStatus file) { + return toRow( + file.getPath().toString(), + file.getPath().getParent().toString(), + file.getPath().getName(), + file.getLen(), + Timestamp.fromEpochMillis(file.getModificationTime()), + Timestamp.fromEpochMillis(file.getAccessTime()), + file.getOwner(), + null, + null, + null, + null, + null, + new GenericMap(Collections.emptyMap())); + } + + public static GenericRow toRow(Object... values) { + GenericRow row = new GenericRow(values.length); + + for (int i = 0; i < values.length; ++i) { + Object value = values[i]; + if (value instanceof String) { + value = BinaryString.fromString((String) value); + } + row.setField(i, value); + } + + return row; + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/table/object/ObjectTable.java b/paimon-core/src/main/java/org/apache/paimon/table/object/ObjectTable.java new file mode 100644 index 000000000000..97acfe7299c5 --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/table/object/ObjectTable.java @@ -0,0 +1,219 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.table.object; + +import org.apache.paimon.fs.FileIO; +import org.apache.paimon.manifest.ManifestCacheFilter; +import org.apache.paimon.schema.TableSchema; +import org.apache.paimon.table.DelegatedFileStoreTable; +import org.apache.paimon.table.FileStoreTable; +import org.apache.paimon.table.sink.BatchWriteBuilder; +import org.apache.paimon.table.sink.StreamWriteBuilder; +import org.apache.paimon.table.sink.TableCommitImpl; +import org.apache.paimon.table.sink.TableWriteImpl; +import org.apache.paimon.types.DataTypes; +import org.apache.paimon.types.RowType; + +import java.util.HashSet; +import java.util.Map; + +import static org.apache.paimon.utils.Preconditions.checkArgument; + +/** + * A object table refers to a directory that contains multiple objects (files), Object table + * provides metadata indexes for unstructured data objects in this directory. Allowing users to + * analyze unstructured data in Object Storage. + * + *

    Object Table stores the metadata of objects in the underlying table. + */ +public interface ObjectTable extends FileStoreTable { + + RowType SCHEMA = + RowType.builder() + .field("path", DataTypes.STRING().notNull()) + .field("parent_path", DataTypes.STRING().notNull()) + .field("name", DataTypes.STRING().notNull()) + .field("length", DataTypes.BIGINT().notNull()) + .field("mtime", DataTypes.TIMESTAMP_LTZ_MILLIS()) + .field("atime", DataTypes.TIMESTAMP_LTZ_MILLIS()) + .field("owner", DataTypes.STRING().nullable()) + .field("generation", DataTypes.INT().nullable()) + .field("content_type", DataTypes.STRING().nullable()) + .field("storage_class", DataTypes.STRING().nullable()) + .field("md5_hash", DataTypes.STRING().nullable()) + .field("metadata_mtime", DataTypes.TIMESTAMP_LTZ_MILLIS().nullable()) + .field("metadata", DataTypes.MAP(DataTypes.STRING(), DataTypes.STRING())) + .build() + .notNull(); + + /** Object location in file system. */ + String objectLocation(); + + /** Underlying table to store metadata. */ + FileStoreTable underlyingTable(); + + /** File io for object file system. */ + FileIO objectFileIO(); + + long refresh(); + + @Override + ObjectTable copy(Map dynamicOptions); + + @Override + ObjectTable copy(TableSchema newTableSchema); + + @Override + ObjectTable copyWithoutTimeTravel(Map dynamicOptions); + + @Override + ObjectTable copyWithLatestSchema(); + + @Override + ObjectTable switchToBranch(String branchName); + + /** Create a new builder for {@link ObjectTable}. */ + static ObjectTable.Builder builder() { + return new ObjectTable.Builder(); + } + + /** Builder for {@link ObjectTable}. */ + class Builder { + + private FileStoreTable underlyingTable; + private FileIO objectFileIO; + private String objectLocation; + + public ObjectTable.Builder underlyingTable(FileStoreTable underlyingTable) { + this.underlyingTable = underlyingTable; + checkArgument( + new HashSet<>(SCHEMA.getFields()) + .containsAll(underlyingTable.rowType().getFields()), + "Schema of Object Table should be %s, but is %s.", + SCHEMA, + underlyingTable.rowType()); + return this; + } + + public ObjectTable.Builder objectFileIO(FileIO objectFileIO) { + this.objectFileIO = objectFileIO; + return this; + } + + public ObjectTable.Builder objectLocation(String objectLocation) { + this.objectLocation = objectLocation; + return this; + } + + public ObjectTable build() { + return new ObjectTableImpl(underlyingTable, objectFileIO, objectLocation); + } + } + + /** An implementation for {@link ObjectTable}. */ + class ObjectTableImpl extends DelegatedFileStoreTable implements ObjectTable { + + private final FileIO objectFileIO; + private final String objectLocation; + + public ObjectTableImpl( + FileStoreTable underlyingTable, FileIO objectFileIO, String objectLocation) { + super(underlyingTable); + this.objectFileIO = objectFileIO; + this.objectLocation = objectLocation; + } + + @Override + public BatchWriteBuilder newBatchWriteBuilder() { + throw new UnsupportedOperationException("Object table does not support Write."); + } + + @Override + public StreamWriteBuilder newStreamWriteBuilder() { + throw new UnsupportedOperationException("Object table does not support Write."); + } + + @Override + public TableWriteImpl newWrite(String commitUser) { + throw new UnsupportedOperationException("Object table does not support Write."); + } + + @Override + public TableWriteImpl newWrite(String commitUser, ManifestCacheFilter manifestFilter) { + throw new UnsupportedOperationException("Object table does not support Write."); + } + + @Override + public TableCommitImpl newCommit(String commitUser) { + throw new UnsupportedOperationException("Object table does not support Commit."); + } + + @Override + public String objectLocation() { + return objectLocation; + } + + @Override + public FileStoreTable underlyingTable() { + return wrapped; + } + + @Override + public FileIO objectFileIO() { + return objectFileIO; + } + + @Override + public long refresh() { + try { + return ObjectRefresh.refresh(this); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + @Override + public ObjectTable copy(Map dynamicOptions) { + return new ObjectTableImpl(wrapped.copy(dynamicOptions), objectFileIO, objectLocation); + } + + @Override + public ObjectTable copy(TableSchema newTableSchema) { + return new ObjectTableImpl(wrapped.copy(newTableSchema), objectFileIO, objectLocation); + } + + @Override + public ObjectTable copyWithoutTimeTravel(Map dynamicOptions) { + return new ObjectTableImpl( + wrapped.copyWithoutTimeTravel(dynamicOptions), objectFileIO, objectLocation); + } + + @Override + public ObjectTable copyWithLatestSchema() { + return new ObjectTableImpl( + wrapped.copyWithLatestSchema(), objectFileIO, objectLocation); + } + + @Override + public ObjectTable switchToBranch(String branchName) { + return new ObjectTableImpl( + wrapped.switchToBranch(branchName), objectFileIO, objectLocation); + } + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/table/query/LocalTableQuery.java b/paimon-core/src/main/java/org/apache/paimon/table/query/LocalTableQuery.java index 6b19f9c051f3..8ff5ce7a6580 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/query/LocalTableQuery.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/query/LocalTableQuery.java @@ -37,8 +37,10 @@ import org.apache.paimon.mergetree.LookupFile; import org.apache.paimon.mergetree.LookupLevels; import org.apache.paimon.options.Options; +import org.apache.paimon.reader.RecordReader; import org.apache.paimon.table.FileStoreTable; import org.apache.paimon.types.RowType; +import org.apache.paimon.utils.Filter; import org.apache.paimon.utils.KeyComparatorSupplier; import org.apache.paimon.utils.Preconditions; @@ -79,6 +81,8 @@ public class LocalTableQuery implements TableQuery { private final RowType rowType; private final RowType partitionType; + @Nullable private Filter cacheRowFilter; + public LocalTableQuery(FileStoreTable table) { this.options = table.coreOptions(); this.tableView = new HashMap<>(); @@ -97,7 +101,9 @@ public LocalTableQuery(FileStoreTable table) { this.lookupStoreFactory = LookupStoreFactory.create( options, - new CacheManager(options.lookupCacheMaxMemory()), + new CacheManager( + options.lookupCacheMaxMemory(), + options.lookupCacheHighPrioPoolRatio()), new RowCompactedSerializer(keyType).createSliceComparator()); if (options.needLookup()) { @@ -154,12 +160,20 @@ private void newLookupLevels(BinaryRow partition, int bucket, List keyComparatorSupplier.get(), readerFactoryBuilder.keyType(), new LookupLevels.KeyValueProcessor(readerFactoryBuilder.readValueType()), - file -> - factory.createRecordReader( - file.schemaId(), - file.fileName(), - file.fileSize(), - file.level()), + file -> { + RecordReader reader = + factory.createRecordReader( + file.schemaId(), + file.fileName(), + file.fileSize(), + file.level()); + if (cacheRowFilter != null) { + reader = + reader.filter( + keyValue -> cacheRowFilter.test(keyValue.value())); + } + return reader; + }, file -> Preconditions.checkNotNull(ioManager, "IOManager is required.") .createChannel( @@ -206,6 +220,11 @@ public LocalTableQuery withIOManager(IOManager ioManager) { return this; } + public LocalTableQuery withCacheRowFilter(Filter cacheRowFilter) { + this.cacheRowFilter = cacheRowFilter; + return this; + } + @Override public InternalRowSerializer createValueSerializer() { return InternalSerializers.create(readerFactoryBuilder.readValueType()); diff --git a/paimon-core/src/main/java/org/apache/paimon/table/sink/CommitMessageLegacyV2Serializer.java b/paimon-core/src/main/java/org/apache/paimon/table/sink/CommitMessageLegacyV2Serializer.java index 989f32f50beb..3e351cd1dae9 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/sink/CommitMessageLegacyV2Serializer.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/sink/CommitMessageLegacyV2Serializer.java @@ -154,6 +154,7 @@ public DataFileMeta fromRow(InternalRow row) { row.getTimestamp(12, 3), null, null, + null, null); } } diff --git a/paimon-core/src/main/java/org/apache/paimon/table/sink/CommitMessageSerializer.java b/paimon-core/src/main/java/org/apache/paimon/table/sink/CommitMessageSerializer.java index cc8abfa485f3..9fc251c36672 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/sink/CommitMessageSerializer.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/sink/CommitMessageSerializer.java @@ -19,9 +19,13 @@ package org.apache.paimon.table.sink; import org.apache.paimon.data.serializer.VersionedSerializer; +import org.apache.paimon.index.IndexFileMeta; +import org.apache.paimon.index.IndexFileMeta09Serializer; import org.apache.paimon.index.IndexFileMetaSerializer; import org.apache.paimon.io.CompactIncrement; +import org.apache.paimon.io.DataFileMeta; import org.apache.paimon.io.DataFileMeta08Serializer; +import org.apache.paimon.io.DataFileMeta09Serializer; import org.apache.paimon.io.DataFileMetaSerializer; import org.apache.paimon.io.DataIncrement; import org.apache.paimon.io.DataInputDeserializer; @@ -29,6 +33,7 @@ import org.apache.paimon.io.DataOutputView; import org.apache.paimon.io.DataOutputViewStreamWrapper; import org.apache.paimon.io.IndexIncrement; +import org.apache.paimon.utils.IOExceptionSupplier; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -42,12 +47,14 @@ /** {@link VersionedSerializer} for {@link CommitMessage}. */ public class CommitMessageSerializer implements VersionedSerializer { - private static final int CURRENT_VERSION = 3; + private static final int CURRENT_VERSION = 5; private final DataFileMetaSerializer dataFileSerializer; private final IndexFileMetaSerializer indexEntrySerializer; + private DataFileMeta09Serializer dataFile09Serializer; private DataFileMeta08Serializer dataFile08Serializer; + private IndexFileMeta09Serializer indexEntry09Serializer; public CommitMessageSerializer() { this.dataFileSerializer = new DataFileMetaSerializer(); @@ -104,53 +111,48 @@ public List deserializeList(int version, DataInputView view) thro } private CommitMessage deserialize(int version, DataInputView view) throws IOException { - if (version == CURRENT_VERSION) { - return new CommitMessageImpl( - deserializeBinaryRow(view), - view.readInt(), - new DataIncrement( - dataFileSerializer.deserializeList(view), - dataFileSerializer.deserializeList(view), - dataFileSerializer.deserializeList(view)), - new CompactIncrement( - dataFileSerializer.deserializeList(view), - dataFileSerializer.deserializeList(view), - dataFileSerializer.deserializeList(view)), - new IndexIncrement( - indexEntrySerializer.deserializeList(view), - indexEntrySerializer.deserializeList(view))); - } else if (version <= 2) { - return deserialize08(version, view); - } else { - throw new UnsupportedOperationException( - "Expecting CommitMessageSerializer version to be smaller or equal than " - + CURRENT_VERSION - + ", but found " - + version - + "."); - } - } - - private CommitMessage deserialize08(int version, DataInputView view) throws IOException { - if (dataFile08Serializer == null) { - dataFile08Serializer = new DataFileMeta08Serializer(); - } + IOExceptionSupplier> fileDeserializer = fileDeserializer(version, view); + IOExceptionSupplier> indexEntryDeserializer = + indexEntryDeserializer(version, view); return new CommitMessageImpl( deserializeBinaryRow(view), view.readInt(), new DataIncrement( - dataFile08Serializer.deserializeList(view), - dataFile08Serializer.deserializeList(view), - dataFile08Serializer.deserializeList(view)), + fileDeserializer.get(), fileDeserializer.get(), fileDeserializer.get()), new CompactIncrement( - dataFile08Serializer.deserializeList(view), - dataFile08Serializer.deserializeList(view), - dataFile08Serializer.deserializeList(view)), + fileDeserializer.get(), fileDeserializer.get(), fileDeserializer.get()), new IndexIncrement( - indexEntrySerializer.deserializeList(view), - version <= 2 - ? Collections.emptyList() - : indexEntrySerializer.deserializeList(view))); + indexEntryDeserializer.get(), + version <= 2 ? Collections.emptyList() : indexEntryDeserializer.get())); + } + + private IOExceptionSupplier> fileDeserializer( + int version, DataInputView view) { + if (version >= 4) { + return () -> dataFileSerializer.deserializeList(view); + } else if (version == 3) { + if (dataFile09Serializer == null) { + dataFile09Serializer = new DataFileMeta09Serializer(); + } + return () -> dataFile09Serializer.deserializeList(view); + } else { + if (dataFile08Serializer == null) { + dataFile08Serializer = new DataFileMeta08Serializer(); + } + return () -> dataFile08Serializer.deserializeList(view); + } + } + + private IOExceptionSupplier> indexEntryDeserializer( + int version, DataInputView view) { + if (version >= 5) { + return () -> indexEntrySerializer.deserializeList(view); + } else { + if (indexEntry09Serializer == null) { + indexEntry09Serializer = new IndexFileMeta09Serializer(); + } + return () -> indexEntry09Serializer.deserializeList(view); + } } } diff --git a/paimon-core/src/main/java/org/apache/paimon/table/sink/TableCommitImpl.java b/paimon-core/src/main/java/org/apache/paimon/table/sink/TableCommitImpl.java index b4f8fa47dbb1..73c55942a56a 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/sink/TableCommitImpl.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/sink/TableCommitImpl.java @@ -68,7 +68,7 @@ import static org.apache.paimon.table.sink.BatchWriteBuilder.COMMIT_IDENTIFIER; import static org.apache.paimon.utils.ManifestReadThreadPool.getExecutorService; import static org.apache.paimon.utils.Preconditions.checkState; -import static org.apache.paimon.utils.ThreadPoolUtils.randomlyExecute; +import static org.apache.paimon.utils.ThreadPoolUtils.randomlyExecuteSequentialReturn; /** An abstraction layer above {@link FileStoreCommit} to provide snapshot commit and expiration. */ public class TableCommitImpl implements InnerTableCommit { @@ -292,7 +292,7 @@ private void checkFilesExistence(List committables) { List nonExistFiles = Lists.newArrayList( - randomlyExecute( + randomlyExecuteSequentialReturn( getExecutorService(null), f -> nonExists.test(f) ? singletonList(f) : emptyList(), files)); diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractDataTableScan.java b/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractDataTableScan.java index 6a8aa9265e5c..24c6943f546f 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractDataTableScan.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractDataTableScan.java @@ -102,6 +102,12 @@ public AbstractDataTableScan withMetricsRegistry(MetricRegistry metricsRegistry) return this; } + @Override + public AbstractDataTableScan dropStats() { + snapshotReader.dropStats(); + return this; + } + public CoreOptions options() { return options; } diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/DataSplit.java b/paimon-core/src/main/java/org/apache/paimon/table/source/DataSplit.java index 067bf055c241..b9460f28b4e7 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/source/DataSplit.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/source/DataSplit.java @@ -21,6 +21,7 @@ import org.apache.paimon.data.BinaryRow; import org.apache.paimon.io.DataFileMeta; import org.apache.paimon.io.DataFileMeta08Serializer; +import org.apache.paimon.io.DataFileMeta09Serializer; import org.apache.paimon.io.DataFileMetaSerializer; import org.apache.paimon.io.DataInputView; import org.apache.paimon.io.DataInputViewStreamWrapper; @@ -43,13 +44,14 @@ import static org.apache.paimon.io.DataFilePathFactory.INDEX_PATH_SUFFIX; import static org.apache.paimon.utils.Preconditions.checkArgument; +import static org.apache.paimon.utils.Preconditions.checkState; /** Input splits. Needed by most batch computation engines. */ public class DataSplit implements Split { private static final long serialVersionUID = 7L; private static final long MAGIC = -2394839472490812314L; - private static final int VERSION = 2; + private static final int VERSION = 4; private long snapshotId = 0; private BinaryRow partition; @@ -125,6 +127,45 @@ public long rowCount() { return rowCount; } + /** Whether it is possible to calculate the merged row count. */ + public boolean mergedRowCountAvailable() { + return rawConvertible + && (dataDeletionFiles == null + || dataDeletionFiles.stream() + .allMatch(f -> f == null || f.cardinality() != null)); + } + + public long mergedRowCount() { + checkState(mergedRowCountAvailable()); + return partialMergedRowCount(); + } + + /** + * Obtain merged row count as much as possible. There are two scenarios where accurate row count + * can be calculated: + * + *

    1. raw file and no deletion file. + * + *

    2. raw file + deletion file with cardinality. + */ + public long partialMergedRowCount() { + long sum = 0L; + if (rawConvertible) { + List rawFiles = convertToRawFiles().orElse(null); + if (rawFiles != null) { + for (int i = 0; i < rawFiles.size(); i++) { + RawFile rawFile = rawFiles.get(i); + if (dataDeletionFiles == null || dataDeletionFiles.get(i) == null) { + sum += rawFile.rowCount(); + } else if (dataDeletionFiles.get(i).cardinality() != null) { + sum += rawFile.rowCount() - dataDeletionFiles.get(i).cardinality(); + } + } + } + } + return sum; + } + @Override public Optional> convertToRawFiles() { if (rawConvertible) { @@ -271,13 +312,16 @@ public static DataSplit deserialize(DataInputView in) throws IOException { FunctionWithIOException dataFileSer = getFileMetaSerde(version); + FunctionWithIOException deletionFileSerde = + getDeletionFileSerde(version); int beforeNumber = in.readInt(); List beforeFiles = new ArrayList<>(beforeNumber); for (int i = 0; i < beforeNumber; i++) { beforeFiles.add(dataFileSer.apply(in)); } - List beforeDeletionFiles = DeletionFile.deserializeList(in); + List beforeDeletionFiles = + DeletionFile.deserializeList(in, deletionFileSerde); int fileNumber = in.readInt(); List dataFiles = new ArrayList<>(fileNumber); @@ -285,7 +329,7 @@ public static DataSplit deserialize(DataInputView in) throws IOException { dataFiles.add(dataFileSer.apply(in)); } - List dataDeletionFiles = DeletionFile.deserializeList(in); + List dataDeletionFiles = DeletionFile.deserializeList(in, deletionFileSerde); boolean isStreaming = in.readBoolean(); boolean rawConvertible = in.readBoolean(); @@ -316,15 +360,24 @@ private static FunctionWithIOException getFileMetaS DataFileMeta08Serializer serializer = new DataFileMeta08Serializer(); return serializer::deserialize; } else if (version == 2) { + DataFileMeta09Serializer serializer = new DataFileMeta09Serializer(); + return serializer::deserialize; + } else if (version >= 3) { DataFileMetaSerializer serializer = new DataFileMetaSerializer(); return serializer::deserialize; } else { - throw new UnsupportedOperationException( - "Expecting DataSplit version to be smaller or equal than " - + VERSION - + ", but found " - + version - + "."); + throw new UnsupportedOperationException("Unsupported version: " + version); + } + } + + private static FunctionWithIOException getDeletionFileSerde( + int version) { + if (version >= 1 && version <= 3) { + return DeletionFile::deserializeV3; + } else if (version >= 4) { + return DeletionFile::deserialize; + } else { + throw new UnsupportedOperationException("Unsupported version: " + version); } } diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/DataTableBatchScan.java b/paimon-core/src/main/java/org/apache/paimon/table/source/DataTableBatchScan.java index 93ab5ba1644d..a4fe6d73bba1 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/source/DataTableBatchScan.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/source/DataTableBatchScan.java @@ -50,7 +50,7 @@ public DataTableBatchScan( this.hasNext = true; this.defaultValueAssigner = defaultValueAssigner; if (pkTable && (options.deletionVectorsEnabled() || options.mergeEngine() == FIRST_ROW)) { - snapshotReader.withLevelFilter(level -> level > 0); + snapshotReader.withLevelFilter(level -> level > 0).enableValueFilter(); } } @@ -95,32 +95,25 @@ private StartingScanner.Result applyPushDownLimit(StartingScanner.Result result) long scannedRowCount = 0; SnapshotReader.Plan plan = ((ScannedResult) result).plan(); List splits = plan.dataSplits(); + if (splits.isEmpty()) { + return result; + } + List limitedSplits = new ArrayList<>(); for (DataSplit dataSplit : splits) { - long splitRowCount = getRowCountForSplit(dataSplit); - limitedSplits.add(dataSplit); - scannedRowCount += splitRowCount; - if (scannedRowCount >= pushDownLimit) { - break; + if (dataSplit.rawConvertible()) { + long partialMergedRowCount = dataSplit.partialMergedRowCount(); + limitedSplits.add(dataSplit); + scannedRowCount += partialMergedRowCount; + if (scannedRowCount >= pushDownLimit) { + SnapshotReader.Plan newPlan = + new PlanImpl(plan.watermark(), plan.snapshotId(), limitedSplits); + return new ScannedResult(newPlan); + } } } - - SnapshotReader.Plan newPlan = - new PlanImpl(plan.watermark(), plan.snapshotId(), limitedSplits); - return new ScannedResult(newPlan); - } else { - return result; } - } - - /** - * 0 represents that we can't compute the row count of this split, 'cause this split needs to be - * merged. - */ - private long getRowCountForSplit(DataSplit split) { - return split.convertToRawFiles() - .map(files -> files.stream().map(RawFile::rowCount).reduce(Long::sum).orElse(0L)) - .orElse(0L); + return result; } @Override diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/DataTableStreamScan.java b/paimon-core/src/main/java/org/apache/paimon/table/source/DataTableStreamScan.java index a68c7b1cb46d..e8c4ddfa1c7c 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/source/DataTableStreamScan.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/source/DataTableStreamScan.java @@ -194,16 +194,16 @@ private Plan nextPlan() { return SnapshotNotExistPlan.INSTANCE; } - // first check changes of overwrite - if (snapshot.commitKind() == Snapshot.CommitKind.OVERWRITE - && supportStreamingReadOverwrite) { - LOG.debug("Find overwrite snapshot id {}.", nextSnapshotId); - SnapshotReader.Plan overwritePlan = - followUpScanner.getOverwriteChangesPlan(snapshot, snapshotReader); - currentWatermark = overwritePlan.watermark(); - nextSnapshotId++; - return overwritePlan; - } else if (followUpScanner.shouldScanSnapshot(snapshot)) { + // first try to get overwrite changes + if (snapshot.commitKind() == Snapshot.CommitKind.OVERWRITE) { + SnapshotReader.Plan overwritePlan = handleOverwriteSnapshot(snapshot); + if (overwritePlan != null) { + nextSnapshotId++; + return overwritePlan; + } + } + + if (followUpScanner.shouldScanSnapshot(snapshot)) { LOG.debug("Find snapshot id {}.", nextSnapshotId); SnapshotReader.Plan plan = followUpScanner.scan(snapshot, snapshotReader); currentWatermark = plan.watermark(); @@ -228,6 +228,18 @@ private boolean shouldDelaySnapshot(long snapshotId) { return false; } + @Nullable + protected SnapshotReader.Plan handleOverwriteSnapshot(Snapshot snapshot) { + if (supportStreamingReadOverwrite) { + LOG.debug("Find overwrite snapshot id {}.", nextSnapshotId); + SnapshotReader.Plan overwritePlan = + followUpScanner.getOverwriteChangesPlan(snapshot, snapshotReader); + currentWatermark = overwritePlan.watermark(); + return overwritePlan; + } + return null; + } + protected FollowUpScanner createFollowUpScanner() { CoreOptions.StreamScanMode type = options.toConfiguration().get(CoreOptions.STREAM_SCAN_MODE); diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/DeletionFile.java b/paimon-core/src/main/java/org/apache/paimon/table/source/DeletionFile.java index 94dfc615729c..5bcf6898ed99 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/source/DeletionFile.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/source/DeletionFile.java @@ -22,6 +22,7 @@ import org.apache.paimon.io.DataFileMeta; import org.apache.paimon.io.DataInputView; import org.apache.paimon.io.DataOutputView; +import org.apache.paimon.utils.FunctionWithIOException; import javax.annotation.Nullable; @@ -52,11 +53,13 @@ public class DeletionFile implements Serializable { private final String path; private final long offset; private final long length; + @Nullable private final Long cardinality; - public DeletionFile(String path, long offset, long length) { + public DeletionFile(String path, long offset, long length, @Nullable Long cardinality) { this.path = path; this.offset = offset; this.length = length; + this.cardinality = cardinality; } /** Path of the file. */ @@ -74,6 +77,12 @@ public long length() { return length; } + /** the number of deleted rows. */ + @Nullable + public Long cardinality() { + return cardinality; + } + public static void serialize(DataOutputView out, @Nullable DeletionFile file) throws IOException { if (file == null) { @@ -83,6 +92,7 @@ public static void serialize(DataOutputView out, @Nullable DeletionFile file) out.writeUTF(file.path); out.writeLong(file.offset); out.writeLong(file.length); + out.writeLong(file.cardinality == null ? -1 : file.cardinality); } } @@ -108,17 +118,32 @@ public static DeletionFile deserialize(DataInputView in) throws IOException { String path = in.readUTF(); long offset = in.readLong(); long length = in.readLong(); - return new DeletionFile(path, offset, length); + long cardinality = in.readLong(); + return new DeletionFile(path, offset, length, cardinality == -1 ? null : cardinality); } @Nullable - public static List deserializeList(DataInputView in) throws IOException { + public static DeletionFile deserializeV3(DataInputView in) throws IOException { + if (in.readByte() == 0) { + return null; + } + + String path = in.readUTF(); + long offset = in.readLong(); + long length = in.readLong(); + return new DeletionFile(path, offset, length, null); + } + + @Nullable + public static List deserializeList( + DataInputView in, FunctionWithIOException deserialize) + throws IOException { List files = null; if (in.readByte() == 1) { int size = in.readInt(); files = new ArrayList<>(size); for (int i = 0; i < size; i++) { - files.add(DeletionFile.deserialize(in)); + files.add(deserialize.apply(in)); } } return files; @@ -126,22 +151,34 @@ public static List deserializeList(DataInputView in) throws IOExce @Override public boolean equals(Object o) { - if (!(o instanceof DeletionFile)) { + if (o == null || getClass() != o.getClass()) { return false; } - - DeletionFile other = (DeletionFile) o; - return Objects.equals(path, other.path) && offset == other.offset && length == other.length; + DeletionFile that = (DeletionFile) o; + return offset == that.offset + && length == that.length + && Objects.equals(path, that.path) + && Objects.equals(cardinality, that.cardinality); } @Override public int hashCode() { - return Objects.hash(path, offset, length); + return Objects.hash(path, offset, length, cardinality); } @Override public String toString() { - return String.format("{path = %s, offset = %d, length = %d}", path, offset, length); + return "DeletionFile{" + + "path='" + + path + + '\'' + + ", offset=" + + offset + + ", length=" + + length + + ", cardinality=" + + cardinality + + '}'; } static Factory emptyFactory() { diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/InnerTableScan.java b/paimon-core/src/main/java/org/apache/paimon/table/source/InnerTableScan.java index 00a4fc0cde18..c2425ff16f97 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/source/InnerTableScan.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/source/InnerTableScan.java @@ -55,4 +55,9 @@ default InnerTableScan withMetricsRegistry(MetricRegistry metricRegistry) { // do nothing, should implement this if need return this; } + + default InnerTableScan dropStats() { + // do nothing, should implement this if need + return this; + } } diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/ReadBuilder.java b/paimon-core/src/main/java/org/apache/paimon/table/source/ReadBuilder.java index 91d5f1004e91..0c1386ce441d 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/source/ReadBuilder.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/source/ReadBuilder.java @@ -150,6 +150,9 @@ default ReadBuilder withProjection(int[][] projection) { */ ReadBuilder withShard(int indexOfThisSubtask, int numberOfParallelSubtasks); + /** Delete stats in scan plan result. */ + ReadBuilder dropStats(); + /** Create a {@link TableScan} to perform batch planning. */ TableScan newScan(); diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/ReadBuilderImpl.java b/paimon-core/src/main/java/org/apache/paimon/table/source/ReadBuilderImpl.java index 577b0a20a99b..95bfe6f24bc7 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/source/ReadBuilderImpl.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/source/ReadBuilderImpl.java @@ -51,6 +51,8 @@ public class ReadBuilderImpl implements ReadBuilder { private @Nullable RowType readType; + private boolean dropStats = false; + public ReadBuilderImpl(InnerTable table) { this.table = table; } @@ -124,6 +126,12 @@ public ReadBuilder withBucketFilter(Filter bucketFilter) { return this; } + @Override + public ReadBuilder dropStats() { + this.dropStats = true; + return this; + } + @Override public TableScan newScan() { InnerTableScan tableScan = configureScan(table.newScan()); @@ -156,6 +164,9 @@ private InnerTableScan configureScan(InnerTableScan scan) { if (bucketFilter != null) { scan.withBucketFilter(bucketFilter); } + if (dropStats) { + scan.dropStats(); + } return scan; } diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/snapshot/IncrementalStartingScanner.java b/paimon-core/src/main/java/org/apache/paimon/table/source/snapshot/IncrementalStartingScanner.java index 358d86cbe948..9bfb54f2cf60 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/source/snapshot/IncrementalStartingScanner.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/source/snapshot/IncrementalStartingScanner.java @@ -31,7 +31,6 @@ import org.apache.paimon.table.source.ScanMode; import org.apache.paimon.table.source.Split; import org.apache.paimon.table.source.SplitGenerator; -import org.apache.paimon.utils.ManifestReadThreadPool; import org.apache.paimon.utils.Pair; import org.apache.paimon.utils.SnapshotManager; @@ -50,6 +49,7 @@ import java.util.stream.Collectors; import java.util.stream.LongStream; +import static org.apache.paimon.utils.ManifestReadThreadPool.randomlyExecuteSequentialReturn; import static org.apache.paimon.utils.Preconditions.checkArgument; /** {@link StartingScanner} for incremental changes by snapshot. */ @@ -84,7 +84,7 @@ public Result scan(SnapshotReader reader) { .collect(Collectors.toList()); Iterator manifests = - ManifestReadThreadPool.randomlyExecute( + randomlyExecuteSequentialReturn( id -> { Snapshot snapshot = snapshotManager.snapshot(id); switch (scanMode) { @@ -111,7 +111,7 @@ public Result scan(SnapshotReader reader) { reader.parallelism()); Iterator entries = - ManifestReadThreadPool.randomlyExecute( + randomlyExecuteSequentialReturn( reader::readManifest, Lists.newArrayList(manifests), reader.parallelism()); while (entries.hasNext()) { diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/snapshot/SnapshotReader.java b/paimon-core/src/main/java/org/apache/paimon/table/source/snapshot/SnapshotReader.java index 2dd02be04f7c..f3e0a92b8fc7 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/source/snapshot/SnapshotReader.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/source/snapshot/SnapshotReader.java @@ -77,6 +77,8 @@ public interface SnapshotReader { SnapshotReader withLevelFilter(Filter levelFilter); + SnapshotReader enableValueFilter(); + SnapshotReader withManifestEntryFilter(Filter filter); SnapshotReader withBucket(int bucket); @@ -85,6 +87,8 @@ public interface SnapshotReader { SnapshotReader withDataFileNameFilter(Filter fileNameFilter); + SnapshotReader dropStats(); + SnapshotReader withShard(int indexOfThisSubtask, int numberOfParallelSubtasks); SnapshotReader withMetricRegistry(MetricRegistry registry); diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/snapshot/SnapshotReaderImpl.java b/paimon-core/src/main/java/org/apache/paimon/table/source/snapshot/SnapshotReaderImpl.java index f4591734b68e..bf19ba10c689 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/source/snapshot/SnapshotReaderImpl.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/source/snapshot/SnapshotReaderImpl.java @@ -24,6 +24,7 @@ import org.apache.paimon.codegen.RecordComparator; import org.apache.paimon.consumer.ConsumerManager; import org.apache.paimon.data.BinaryRow; +import org.apache.paimon.index.DeletionVectorMeta; import org.apache.paimon.index.IndexFileHandler; import org.apache.paimon.index.IndexFileMeta; import org.apache.paimon.io.DataFileMeta; @@ -234,6 +235,12 @@ public SnapshotReader withLevelFilter(Filter levelFilter) { return this; } + @Override + public SnapshotReader enableValueFilter() { + scan.enableValueFilter(); + return this; + } + @Override public SnapshotReader withManifestEntryFilter(Filter filter) { scan.withManifestEntryFilter(filter); @@ -264,6 +271,12 @@ public SnapshotReader withDataFileNameFilter(Filter fileNameFilter) { return this; } + @Override + public SnapshotReader dropStats() { + scan.dropStats(); + return this; + } + @Override public SnapshotReader withShard(int indexOfThisSubtask, int numberOfParallelSubtasks) { if (splitGenerator.alwaysRawConvertible()) { @@ -480,23 +493,24 @@ private List getDeletionFiles( List deletionFiles = new ArrayList<>(dataFiles.size()); Map dataFileToIndexFileMeta = new HashMap<>(); for (IndexFileMeta indexFileMeta : indexFileMetas) { - if (indexFileMeta.deletionVectorsRanges() != null) { - for (String dataFileName : indexFileMeta.deletionVectorsRanges().keySet()) { - dataFileToIndexFileMeta.put(dataFileName, indexFileMeta); + if (indexFileMeta.deletionVectorMetas() != null) { + for (DeletionVectorMeta dvMeta : indexFileMeta.deletionVectorMetas().values()) { + dataFileToIndexFileMeta.put(dvMeta.dataFileName(), indexFileMeta); } } } for (DataFileMeta file : dataFiles) { IndexFileMeta indexFileMeta = dataFileToIndexFileMeta.get(file.fileName()); if (indexFileMeta != null) { - Map> ranges = indexFileMeta.deletionVectorsRanges(); - if (ranges != null && ranges.containsKey(file.fileName())) { - Pair range = ranges.get(file.fileName()); + LinkedHashMap dvMetas = + indexFileMeta.deletionVectorMetas(); + if (dvMetas != null && dvMetas.containsKey(file.fileName())) { deletionFiles.add( new DeletionFile( indexFileHandler.filePath(indexFileMeta).toString(), - range.getKey(), - range.getValue())); + dvMetas.get(file.fileName()).offset(), + dvMetas.get(file.fileName()).length(), + dvMetas.get(file.fileName()).cardinality())); continue; } } diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/snapshot/TimeTravelUtil.java b/paimon-core/src/main/java/org/apache/paimon/table/source/snapshot/TimeTravelUtil.java new file mode 100644 index 000000000000..4c8b41aa4215 --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/table/source/snapshot/TimeTravelUtil.java @@ -0,0 +1,127 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.table.source.snapshot; + +import org.apache.paimon.CoreOptions; +import org.apache.paimon.Snapshot; +import org.apache.paimon.table.FileStoreTable; +import org.apache.paimon.utils.Preconditions; +import org.apache.paimon.utils.SnapshotManager; +import org.apache.paimon.utils.SnapshotNotExistException; +import org.apache.paimon.utils.TagManager; + +import java.util.ArrayList; +import java.util.List; + +import static org.apache.paimon.CoreOptions.SCAN_SNAPSHOT_ID; + +/** The util class of resolve snapshot from scan params for time travel. */ +public class TimeTravelUtil { + + private static final String[] SCAN_KEYS = { + CoreOptions.SCAN_SNAPSHOT_ID.key(), + CoreOptions.SCAN_TAG_NAME.key(), + CoreOptions.SCAN_WATERMARK.key(), + CoreOptions.SCAN_TIMESTAMP_MILLIS.key() + }; + + public static Snapshot resolveSnapshot(FileStoreTable table) { + return resolveSnapshotFromOptions(table.coreOptions(), table.snapshotManager()); + } + + public static Snapshot resolveSnapshotFromOptions( + CoreOptions options, SnapshotManager snapshotManager) { + List scanHandleKey = new ArrayList<>(1); + for (String key : SCAN_KEYS) { + if (options.toConfiguration().containsKey(key)) { + scanHandleKey.add(key); + } + } + + if (scanHandleKey.size() == 0) { + return snapshotManager.latestSnapshot(); + } + + Preconditions.checkArgument( + scanHandleKey.size() == 1, + String.format( + "Only one of the following parameters may be set : [%s, %s, %s, %s]", + CoreOptions.SCAN_SNAPSHOT_ID.key(), + CoreOptions.SCAN_TAG_NAME.key(), + CoreOptions.SCAN_WATERMARK.key(), + CoreOptions.SCAN_TIMESTAMP_MILLIS.key())); + + String key = scanHandleKey.get(0); + Snapshot snapshot = null; + if (key.equals(CoreOptions.SCAN_SNAPSHOT_ID.key())) { + snapshot = resolveSnapshotBySnapshotId(snapshotManager, options); + } else if (key.equals(CoreOptions.SCAN_WATERMARK.key())) { + snapshot = resolveSnapshotByWatermark(snapshotManager, options); + } else if (key.equals(CoreOptions.SCAN_TIMESTAMP_MILLIS.key())) { + snapshot = resolveSnapshotByTimestamp(snapshotManager, options); + } else if (key.equals(CoreOptions.SCAN_TAG_NAME.key())) { + snapshot = resolveSnapshotByTagName(snapshotManager, options); + } + + if (snapshot == null) { + snapshot = snapshotManager.latestSnapshot(); + } + return snapshot; + } + + private static Snapshot resolveSnapshotBySnapshotId( + SnapshotManager snapshotManager, CoreOptions options) { + Long snapshotId = options.scanSnapshotId(); + if (snapshotId != null) { + if (!snapshotManager.snapshotExists(snapshotId)) { + Long earliestSnapshotId = snapshotManager.earliestSnapshotId(); + Long latestSnapshotId = snapshotManager.latestSnapshotId(); + throw new SnapshotNotExistException( + String.format( + "Specified parameter %s = %s is not exist, you can set it in range from %s to %s.", + SCAN_SNAPSHOT_ID.key(), + snapshotId, + earliestSnapshotId, + latestSnapshotId)); + } + return snapshotManager.snapshot(snapshotId); + } + return null; + } + + private static Snapshot resolveSnapshotByTimestamp( + SnapshotManager snapshotManager, CoreOptions options) { + Long timestamp = options.scanTimestampMills(); + return snapshotManager.earlierOrEqualTimeMills(timestamp); + } + + private static Snapshot resolveSnapshotByWatermark( + SnapshotManager snapshotManager, CoreOptions options) { + Long watermark = options.scanWatermark(); + return snapshotManager.laterOrEqualWatermark(watermark); + } + + private static Snapshot resolveSnapshotByTagName( + SnapshotManager snapshotManager, CoreOptions options) { + String tagName = options.scanTagName(); + TagManager tagManager = + new TagManager(snapshotManager.fileIO(), snapshotManager.tablePath()); + return tagManager.taggedSnapshot(tagName); + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/splitread/IncrementalChangelogReadProvider.java b/paimon-core/src/main/java/org/apache/paimon/table/source/splitread/IncrementalChangelogReadProvider.java index 308c09d14204..eb41d02669fc 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/source/splitread/IncrementalChangelogReadProvider.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/source/splitread/IncrementalChangelogReadProvider.java @@ -60,20 +60,20 @@ private SplitRead create(Supplier supplier) { ConcatRecordReader.create( () -> new ReverseReader( - read.createNoMergeReader( + read.createMergeReader( split.partition(), split.bucket(), split.beforeFiles(), split.beforeDeletionFiles() .orElse(null), - true)), + false)), () -> - read.createNoMergeReader( + read.createMergeReader( split.partition(), split.bucket(), split.dataFiles(), split.deletionFiles().orElse(null), - true)); + false)); return unwrap(reader); }; diff --git a/paimon-core/src/main/java/org/apache/paimon/table/system/AggregationFieldsTable.java b/paimon-core/src/main/java/org/apache/paimon/table/system/AggregationFieldsTable.java index a88bde9e5d72..8c0eed4d6b8b 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/system/AggregationFieldsTable.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/system/AggregationFieldsTable.java @@ -18,7 +18,6 @@ package org.apache.paimon.table.system; -import org.apache.paimon.CoreOptions; import org.apache.paimon.data.BinaryString; import org.apache.paimon.data.GenericRow; import org.apache.paimon.data.InternalRow; @@ -27,7 +26,6 @@ import org.apache.paimon.fs.Path; import org.apache.paimon.predicate.Predicate; import org.apache.paimon.reader.RecordReader; -import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.schema.TableSchema; import org.apache.paimon.table.FileStoreTable; import org.apache.paimon.table.ReadonlyTable; @@ -78,19 +76,13 @@ public class AggregationFieldsTable implements ReadonlyTable { private final FileIO fileIO; private final Path location; - private final String branch; - public AggregationFieldsTable(FileStoreTable dataTable) { - this( - dataTable.fileIO(), - dataTable.location(), - CoreOptions.branch(dataTable.schema().options())); - } + private final FileStoreTable dataTable; - public AggregationFieldsTable(FileIO fileIO, Path location, String branchName) { - this.fileIO = fileIO; - this.location = location; - this.branch = branchName; + public AggregationFieldsTable(FileStoreTable dataTable) { + this.fileIO = dataTable.fileIO(); + this.location = dataTable.location(); + this.dataTable = dataTable; } @Override @@ -120,7 +112,7 @@ public InnerTableRead newRead() { @Override public Table copy(Map dynamicOptions) { - return new AggregationFieldsTable(fileIO, location, branch); + return new AggregationFieldsTable(dataTable.copy(dynamicOptions)); } private class SchemasScan extends ReadOnceTableScan { @@ -196,8 +188,7 @@ public RecordReader createReader(Split split) { if (!(split instanceof AggregationSplit)) { throw new IllegalArgumentException("Unsupported split: " + split.getClass()); } - Path location = ((AggregationSplit) split).location; - TableSchema schemas = new SchemaManager(fileIO, location, branch).latest().get(); + TableSchema schemas = dataTable.schemaManager().latest().get(); Iterator rows = createInternalRowIterator(schemas); if (readType != null) { rows = diff --git a/paimon-core/src/main/java/org/apache/paimon/table/system/AuditLogTable.java b/paimon-core/src/main/java/org/apache/paimon/table/system/AuditLogTable.java index bbbea744191b..1cb967f8d1e2 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/system/AuditLogTable.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/system/AuditLogTable.java @@ -39,10 +39,11 @@ import org.apache.paimon.predicate.PredicateBuilder; import org.apache.paimon.predicate.PredicateReplaceVisitor; import org.apache.paimon.reader.RecordReader; +import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.table.DataTable; import org.apache.paimon.table.FileStoreTable; import org.apache.paimon.table.ReadonlyTable; -import org.apache.paimon.table.SystemFields; +import org.apache.paimon.table.SpecialFields; import org.apache.paimon.table.Table; import org.apache.paimon.table.source.DataTableScan; import org.apache.paimon.table.source.InnerTableRead; @@ -137,7 +138,7 @@ public String name() { @Override public RowType rowType() { List fields = new ArrayList<>(); - fields.add(SystemFields.ROW_KIND); + fields.add(SpecialFields.ROW_KIND); fields.addAll(wrapped.rowType().getFields()); return new RowType(fields); } @@ -187,6 +188,11 @@ public SnapshotManager snapshotManager() { return wrapped.snapshotManager(); } + @Override + public SchemaManager schemaManager() { + return wrapped.schemaManager(); + } + @Override public TagManager tagManager() { return wrapped.tagManager(); @@ -319,6 +325,12 @@ public SnapshotReader withLevelFilter(Filter levelFilter) { return this; } + @Override + public SnapshotReader enableValueFilter() { + wrapped.enableValueFilter(); + return this; + } + @Override public SnapshotReader withManifestEntryFilter(Filter filter) { wrapped.withManifestEntryFilter(filter); @@ -342,6 +354,12 @@ public SnapshotReader withDataFileNameFilter(Filter fileNameFilter) { return this; } + @Override + public SnapshotReader dropStats() { + wrapped.dropStats(); + return this; + } + @Override public SnapshotReader withShard(int indexOfThisSubtask, int numberOfParallelSubtasks) { wrapped.withShard(indexOfThisSubtask, numberOfParallelSubtasks); @@ -526,13 +544,13 @@ public DataTableScan withShard(int indexOfThisSubtask, int numberOfParallelSubta } } - private class AuditLogRead implements InnerTableRead { + class AuditLogRead implements InnerTableRead { - private final InnerTableRead dataRead; + protected final InnerTableRead dataRead; - private int[] readProjection; + protected int[] readProjection; - private AuditLogRead(InnerTableRead dataRead) { + protected AuditLogRead(InnerTableRead dataRead) { this.dataRead = dataRead.forceKeepDelete(); this.readProjection = defaultProjection(); } @@ -566,7 +584,7 @@ public InnerTableRead withReadType(RowType readType) { boolean rowKindAppeared = false; for (int i = 0; i < fields.size(); i++) { String fieldName = fields.get(i).name(); - if (fieldName.equals(SystemFields.ROW_KIND.name())) { + if (fieldName.equals(SpecialFields.ROW_KIND.name())) { rowKindAppeared = true; readProjection[i] = -1; } else { @@ -600,9 +618,9 @@ private InternalRow convertRow(InternalRow data) { } /** A {@link ProjectedRow} which returns row kind when mapping index is negative. */ - private static class AuditLogRow extends ProjectedRow { + static class AuditLogRow extends ProjectedRow { - private AuditLogRow(int[] indexMapping, InternalRow row) { + AuditLogRow(int[] indexMapping, InternalRow row) { super(indexMapping); replaceRow(row); } diff --git a/paimon-core/src/main/java/org/apache/paimon/table/system/BinlogTable.java b/paimon-core/src/main/java/org/apache/paimon/table/system/BinlogTable.java new file mode 100644 index 000000000000..08eea468ea70 --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/table/system/BinlogTable.java @@ -0,0 +1,159 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.table.system; + +import org.apache.paimon.data.GenericArray; +import org.apache.paimon.data.GenericRow; +import org.apache.paimon.data.InternalRow; +import org.apache.paimon.reader.PackChangelogReader; +import org.apache.paimon.reader.RecordReader; +import org.apache.paimon.table.FileStoreTable; +import org.apache.paimon.table.SpecialFields; +import org.apache.paimon.table.Table; +import org.apache.paimon.table.source.DataSplit; +import org.apache.paimon.table.source.InnerTableRead; +import org.apache.paimon.table.source.Split; +import org.apache.paimon.types.ArrayType; +import org.apache.paimon.types.DataField; +import org.apache.paimon.types.RowType; + +import javax.annotation.Nullable; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import static org.apache.paimon.catalog.Catalog.SYSTEM_TABLE_SPLITTER; + +/** + * A {@link Table} for reading binlog of table. The binlog format is as below. + * + *

    INSERT: [+I, [co1], [col2]] + * + *

    UPDATE: [+U, [co1_ub, col1_ua], [col2_ub, col2_ua]] + * + *

    DELETE: [-D, [co1], [col2]] + */ +public class BinlogTable extends AuditLogTable { + + public static final String BINLOG = "binlog"; + + private final FileStoreTable wrapped; + + public BinlogTable(FileStoreTable wrapped) { + super(wrapped); + this.wrapped = wrapped; + } + + @Override + public String name() { + return wrapped.name() + SYSTEM_TABLE_SPLITTER + BINLOG; + } + + @Override + public RowType rowType() { + List fields = new ArrayList<>(); + fields.add(SpecialFields.ROW_KIND); + for (DataField field : wrapped.rowType().getFields()) { + // convert to nullable + fields.add(field.newType(new ArrayType(field.type().nullable()))); + } + return new RowType(fields); + } + + @Override + public InnerTableRead newRead() { + return new BinlogRead(wrapped.newRead()); + } + + @Override + public Table copy(Map dynamicOptions) { + return new BinlogTable(wrapped.copy(dynamicOptions)); + } + + private class BinlogRead extends AuditLogRead { + + private BinlogRead(InnerTableRead dataRead) { + super(dataRead); + } + + @Override + public InnerTableRead withReadType(RowType readType) { + List fields = new ArrayList<>(); + for (DataField field : readType.getFields()) { + if (field.name().equals(SpecialFields.ROW_KIND.name())) { + fields.add(field); + } else { + fields.add(field.newType(((ArrayType) field.type()).getElementType())); + } + } + return super.withReadType(readType.copy(fields)); + } + + @Override + public RecordReader createReader(Split split) throws IOException { + DataSplit dataSplit = (DataSplit) split; + if (dataSplit.isStreaming()) { + return new PackChangelogReader( + dataRead.createReader(split), + (row1, row2) -> + new AuditLogRow( + readProjection, + convertToArray( + row1, row2, wrapped.rowType().fieldGetters())), + wrapped.rowType()); + } else { + return dataRead.createReader(split) + .transform( + (row) -> + new AuditLogRow( + readProjection, + convertToArray( + row, + null, + wrapped.rowType().fieldGetters()))); + } + } + + private InternalRow convertToArray( + InternalRow row1, + @Nullable InternalRow row2, + InternalRow.FieldGetter[] fieldGetters) { + GenericRow row = new GenericRow(row1.getFieldCount()); + for (int i = 0; i < row1.getFieldCount(); i++) { + Object o1 = fieldGetters[i].getFieldOrNull(row1); + Object o2; + if (row2 != null) { + o2 = fieldGetters[i].getFieldOrNull(row2); + row.setField(i, new GenericArray(new Object[] {o1, o2})); + } else { + row.setField(i, new GenericArray(new Object[] {o1})); + } + } + // If no row2 provided, then follow the row1 kind. + if (row2 == null) { + row.setRowKind(row1.getRowKind()); + } else { + row.setRowKind(row2.getRowKind()); + } + return row; + } + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/table/system/BranchesTable.java b/paimon-core/src/main/java/org/apache/paimon/table/system/BranchesTable.java index a055db6d58cd..384a2eee92c8 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/system/BranchesTable.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/system/BranchesTable.java @@ -18,7 +18,6 @@ package org.apache.paimon.table.system; -import org.apache.paimon.Snapshot; import org.apache.paimon.data.BinaryString; import org.apache.paimon.data.GenericRow; import org.apache.paimon.data.InternalRow; @@ -28,8 +27,6 @@ import org.apache.paimon.fs.Path; import org.apache.paimon.predicate.Predicate; import org.apache.paimon.reader.RecordReader; -import org.apache.paimon.schema.SchemaManager; -import org.apache.paimon.schema.TableSchema; import org.apache.paimon.table.FileStoreTable; import org.apache.paimon.table.FileStoreTableFactory; import org.apache.paimon.table.ReadonlyTable; @@ -40,7 +37,6 @@ import org.apache.paimon.table.source.SingletonSplit; import org.apache.paimon.table.source.Split; import org.apache.paimon.table.source.TableRead; -import org.apache.paimon.types.BigIntType; import org.apache.paimon.types.DataField; import org.apache.paimon.types.RowType; import org.apache.paimon.types.TimestampType; @@ -62,15 +58,11 @@ import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.Optional; -import java.util.SortedMap; import java.util.stream.Collectors; import static org.apache.paimon.catalog.Catalog.SYSTEM_TABLE_SPLITTER; import static org.apache.paimon.utils.BranchManager.BRANCH_PREFIX; -import static org.apache.paimon.utils.BranchManager.branchPath; import static org.apache.paimon.utils.FileUtils.listVersionedDirectories; -import static org.apache.paimon.utils.Preconditions.checkArgument; /** A {@link Table} for showing branches of table. */ public class BranchesTable implements ReadonlyTable { @@ -84,21 +76,17 @@ public class BranchesTable implements ReadonlyTable { Arrays.asList( new DataField( 0, "branch_name", SerializationUtils.newStringType(false)), - new DataField( - 1, "created_from_tag", SerializationUtils.newStringType(true)), - new DataField(2, "created_from_snapshot", new BigIntType(true)), - new DataField(3, "create_time", new TimestampType(false, 3)))); + new DataField(1, "create_time", new TimestampType(false, 3)))); private final FileIO fileIO; private final Path location; - public BranchesTable(FileStoreTable dataTable) { - this(dataTable.fileIO(), dataTable.location()); - } + private final FileStoreTable dataTable; - public BranchesTable(FileIO fileIO, Path location) { - this.fileIO = fileIO; - this.location = location; + public BranchesTable(FileStoreTable dataTable) { + this.fileIO = dataTable.fileIO(); + this.location = dataTable.location(); + this.dataTable = dataTable; } @Override @@ -113,7 +101,7 @@ public RowType rowType() { @Override public List primaryKeys() { - return Arrays.asList("branch_name", "tag_name"); + return Collections.singletonList("branch_name"); } @Override @@ -128,7 +116,7 @@ public InnerTableRead newRead() { @Override public Table copy(Map dynamicOptions) { - return new BranchesTable(fileIO, location); + return new BranchesTable(dataTable.copy(dynamicOptions)); } private class BranchesScan extends ReadOnceTableScan { @@ -227,7 +215,6 @@ public RecordReader createReader(Split split) { private List branches(FileStoreTable table) throws IOException { BranchManager branchManager = table.branchManager(); - SchemaManager schemaManager = new SchemaManager(fileIO, table.location()); List> paths = listVersionedDirectories(fileIO, branchManager.branchDirectory(), BRANCH_PREFIX) @@ -237,42 +224,10 @@ private List branches(FileStoreTable table) throws IOException { for (Pair path : paths) { String branchName = path.getLeft().getName().substring(BRANCH_PREFIX.length()); - String basedTag = null; - Long basedSnapshotId = null; long creationTime = path.getRight(); - - Optional tableSchema = - schemaManager.copyWithBranch(branchName).latest(); - if (tableSchema.isPresent()) { - FileStoreTable branchTable = - FileStoreTableFactory.create( - fileIO, new Path(branchPath(table.location(), branchName))); - SortedMap> snapshotTags = - branchTable.tagManager().tags(); - Long earliestSnapshotId = branchTable.snapshotManager().earliestSnapshotId(); - if (snapshotTags.isEmpty()) { - // create based on snapshotId - basedSnapshotId = earliestSnapshotId; - } else { - Snapshot snapshot = snapshotTags.firstKey(); - if (Objects.equals(earliestSnapshotId, snapshot.id())) { - // create based on tag - List tags = snapshotTags.get(snapshot); - checkArgument(tags.size() == 1); - basedTag = tags.get(0); - basedSnapshotId = snapshot.id(); - } else { - // create based on snapshotId - basedSnapshotId = earliestSnapshotId; - } - } - } - result.add( GenericRow.of( BinaryString.fromString(branchName), - BinaryString.fromString(basedTag), - basedSnapshotId, Timestamp.fromLocalDateTime( DateTimeUtils.toLocalDateTime(creationTime)))); } diff --git a/paimon-core/src/main/java/org/apache/paimon/table/system/BucketsTable.java b/paimon-core/src/main/java/org/apache/paimon/table/system/BucketsTable.java index 0ac42485068c..ccc260ef0b79 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/system/BucketsTable.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/system/BucketsTable.java @@ -40,6 +40,7 @@ import org.apache.paimon.types.DataTypes; import org.apache.paimon.types.RowType; import org.apache.paimon.utils.IteratorRecordReader; +import org.apache.paimon.utils.Pair; import org.apache.paimon.utils.ProjectedRow; import org.apache.paimon.utils.RowDataToObjectArrayConverter; import org.apache.paimon.utils.SerializationUtils; @@ -52,9 +53,11 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.Comparator; import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; import static org.apache.paimon.catalog.Catalog.SYSTEM_TABLE_SPLITTER; @@ -181,9 +184,24 @@ public RecordReader createReader(Split split) { new RowDataToObjectArrayConverter( fileStoreTable.schema().logicalPartitionType()); + // sorted by partition and bucket + List> bucketList = + buckets.stream() + .map( + entry -> + Pair.of( + Arrays.toString( + converter.convert(entry.partition())), + entry)) + .sorted( + Comparator.comparing( + (Pair p) -> p.getLeft()) + .thenComparing(p -> p.getRight().bucket())) + .collect(Collectors.toList()); + List results = new ArrayList<>(buckets.size()); - for (BucketEntry entry : buckets) { - results.add(toRow(entry, converter)); + for (Pair pair : bucketList) { + results.add(toRow(pair.getLeft(), pair.getRight())); } Iterator iterator = results.iterator(); @@ -198,13 +216,9 @@ public RecordReader createReader(Split split) { return new IteratorRecordReader<>(iterator); } - private GenericRow toRow( - BucketEntry entry, RowDataToObjectArrayConverter partitionConverter) { - BinaryString partitionId = - BinaryString.fromString( - Arrays.toString(partitionConverter.convert(entry.partition()))); + private GenericRow toRow(String partStr, BucketEntry entry) { return GenericRow.of( - partitionId, + BinaryString.fromString(partStr), entry.bucket(), entry.recordCount(), entry.fileSizeInBytes(), diff --git a/paimon-core/src/main/java/org/apache/paimon/table/system/CompactBucketsTable.java b/paimon-core/src/main/java/org/apache/paimon/table/system/CompactBucketsTable.java index ff40c9502eb7..31cecbfb15c2 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/system/CompactBucketsTable.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/system/CompactBucketsTable.java @@ -33,6 +33,7 @@ import org.apache.paimon.manifest.ManifestFileMeta; import org.apache.paimon.predicate.Predicate; import org.apache.paimon.reader.RecordReader; +import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.table.DataTable; import org.apache.paimon.table.FileStoreTable; import org.apache.paimon.table.ReadonlyTable; @@ -145,6 +146,11 @@ public SnapshotManager snapshotManager() { return wrapped.snapshotManager(); } + @Override + public SchemaManager schemaManager() { + return wrapped.schemaManager(); + } + @Override public TagManager tagManager() { return wrapped.tagManager(); diff --git a/paimon-core/src/main/java/org/apache/paimon/table/system/ConsumersTable.java b/paimon-core/src/main/java/org/apache/paimon/table/system/ConsumersTable.java index 9f7d12961e2f..7e4816b13510 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/system/ConsumersTable.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/system/ConsumersTable.java @@ -74,17 +74,13 @@ public class ConsumersTable implements ReadonlyTable { private final Path location; private final String branch; - public ConsumersTable(FileStoreTable dataTable) { - this( - dataTable.fileIO(), - dataTable.location(), - CoreOptions.branch(dataTable.schema().options())); - } + private final FileStoreTable dataTable; - public ConsumersTable(FileIO fileIO, Path location, String branchName) { - this.fileIO = fileIO; - this.location = location; - this.branch = branchName; + public ConsumersTable(FileStoreTable dataTable) { + this.fileIO = dataTable.fileIO(); + this.location = dataTable.location(); + this.branch = CoreOptions.branch(dataTable.schema().options()); + this.dataTable = dataTable; } @Override @@ -114,7 +110,7 @@ public InnerTableRead newRead() { @Override public Table copy(Map dynamicOptions) { - return new ConsumersTable(fileIO, location, branch); + return new ConsumersTable(dataTable.copy(dynamicOptions)); } private class ConsumersScan extends ReadOnceTableScan { diff --git a/paimon-core/src/main/java/org/apache/paimon/table/system/FileMonitorTable.java b/paimon-core/src/main/java/org/apache/paimon/table/system/FileMonitorTable.java index fc1bb2a5b167..522335aaa6c9 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/system/FileMonitorTable.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/system/FileMonitorTable.java @@ -34,6 +34,7 @@ import org.apache.paimon.manifest.ManifestFileMeta; import org.apache.paimon.predicate.Predicate; import org.apache.paimon.reader.RecordReader; +import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.table.DataTable; import org.apache.paimon.table.FileStoreTable; import org.apache.paimon.table.ReadonlyTable; @@ -131,6 +132,11 @@ public SnapshotManager snapshotManager() { return wrapped.snapshotManager(); } + @Override + public SchemaManager schemaManager() { + return wrapped.schemaManager(); + } + @Override public TagManager tagManager() { return wrapped.tagManager(); diff --git a/paimon-core/src/main/java/org/apache/paimon/table/system/FilesTable.java b/paimon-core/src/main/java/org/apache/paimon/table/system/FilesTable.java index ed06afd3a19a..6dcbb322d6d0 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/system/FilesTable.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/system/FilesTable.java @@ -27,6 +27,7 @@ import org.apache.paimon.disk.IOManager; import org.apache.paimon.io.DataFileMeta; import org.apache.paimon.io.DataFilePathFactory; +import org.apache.paimon.manifest.FileSource; import org.apache.paimon.predicate.Equal; import org.apache.paimon.predicate.LeafPredicate; import org.apache.paimon.predicate.LeafPredicateExtractor; @@ -34,9 +35,8 @@ import org.apache.paimon.reader.RecordReader; import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.schema.TableSchema; -import org.apache.paimon.stats.SimpleStats; -import org.apache.paimon.stats.SimpleStatsConverter; -import org.apache.paimon.stats.SimpleStatsConverters; +import org.apache.paimon.stats.SimpleStatsEvolution; +import org.apache.paimon.stats.SimpleStatsEvolutions; import org.apache.paimon.table.FileStoreTable; import org.apache.paimon.table.ReadonlyTable; import org.apache.paimon.table.Table; @@ -112,7 +112,8 @@ public class FilesTable implements ReadonlyTable { 12, "max_value_stats", SerializationUtils.newStringType(false)), new DataField(13, "min_sequence_number", new BigIntType(true)), new DataField(14, "max_sequence_number", new BigIntType(true)), - new DataField(15, "creation_time", DataTypes.TIMESTAMP_MILLIS()))); + new DataField(15, "creation_time", DataTypes.TIMESTAMP_MILLIS()), + new DataField(16, "file_source", DataTypes.STRING()))); private final FileStoreTable storeTable; @@ -142,8 +143,7 @@ public InnerTableScan newScan() { @Override public InnerTableRead newRead() { - return new FilesRead( - new SchemaManager(storeTable.fileIO(), storeTable.location()), storeTable); + return new FilesRead(storeTable.schemaManager(), storeTable); } @Override @@ -314,8 +314,8 @@ public RecordReader createReader(Split split) { List> iteratorList = new ArrayList<>(); // dataFilePlan.snapshotId indicates there's no files in the table, use the newest // schema id directly - SimpleStatsConverters simpleStatsConverters = - new SimpleStatsConverters( + SimpleStatsEvolutions simpleStatsEvolutions = + new SimpleStatsEvolutions( sid -> schemaManager.schema(sid).fields(), storeTable.schema().id()); RowDataToObjectArrayConverter partitionConverter = @@ -352,8 +352,7 @@ public RowDataToObjectArrayConverter apply(Long schemaId) { partitionConverter, keyConverters, file, - storeTable.getSchemaFieldStats(file), - simpleStatsConverters))); + simpleStatsEvolutions))); } Iterator rows = Iterators.concat(iteratorList.iterator()); if (readType != null) { @@ -372,10 +371,8 @@ private LazyGenericRow toRow( RowDataToObjectArrayConverter partitionConverter, Function keyConverters, DataFileMeta dataFileMeta, - SimpleStats stats, - SimpleStatsConverters simpleStatsConverters) { - StatsLazyGetter statsGetter = - new StatsLazyGetter(stats, dataFileMeta, simpleStatsConverters); + SimpleStatsEvolutions simpleStatsEvolutions) { + StatsLazyGetter statsGetter = new StatsLazyGetter(dataFileMeta, simpleStatsEvolutions); @SuppressWarnings("unchecked") Supplier[] fields = new Supplier[] { @@ -387,7 +384,9 @@ private LazyGenericRow toRow( partitionConverter.convert( dataSplit.partition()))), dataSplit::bucket, - () -> BinaryString.fromString(dataFileMeta.fileName()), + () -> + BinaryString.fromString( + dataSplit.bucketPath() + "/" + dataFileMeta.fileName()), () -> BinaryString.fromString( DataFilePathFactory.formatIdentifier( @@ -417,7 +416,13 @@ private LazyGenericRow toRow( () -> BinaryString.fromString(statsGetter.upperValueBounds().toString()), dataFileMeta::minSequenceNumber, dataFileMeta::maxSequenceNumber, - dataFileMeta::creationTime + dataFileMeta::creationTime, + () -> + BinaryString.fromString( + dataFileMeta + .fileSource() + .map(FileSource::toString) + .orElse(null)) }; return new LazyGenericRow(fields); @@ -426,32 +431,31 @@ private LazyGenericRow toRow( private static class StatsLazyGetter { - private final SimpleStats stats; private final DataFileMeta file; - private final SimpleStatsConverters simpleStatsConverters; + private final SimpleStatsEvolutions simpleStatsEvolutions; private Map lazyNullValueCounts; private Map lazyLowerValueBounds; private Map lazyUpperValueBounds; - private StatsLazyGetter( - SimpleStats stats, DataFileMeta file, SimpleStatsConverters simpleStatsConverters) { - this.stats = stats; + private StatsLazyGetter(DataFileMeta file, SimpleStatsEvolutions simpleStatsEvolutions) { this.file = file; - this.simpleStatsConverters = simpleStatsConverters; + this.simpleStatsEvolutions = simpleStatsEvolutions; } private void initialize() { - SimpleStatsConverter serializer = simpleStatsConverters.getOrCreate(file.schemaId()); + SimpleStatsEvolution evolution = simpleStatsEvolutions.getOrCreate(file.schemaId()); // Create value stats - InternalRow min = serializer.evolution(stats.minValues()); - InternalRow max = serializer.evolution(stats.maxValues()); - InternalArray nullCounts = serializer.evolution(stats.nullCounts(), file.rowCount()); + SimpleStatsEvolution.Result result = + evolution.evolution(file.valueStats(), file.rowCount(), file.valueStatsCols()); + InternalRow min = result.minValues(); + InternalRow max = result.maxValues(); + InternalArray nullCounts = result.nullCounts(); lazyNullValueCounts = new TreeMap<>(); lazyLowerValueBounds = new TreeMap<>(); lazyUpperValueBounds = new TreeMap<>(); for (int i = 0; i < min.getFieldCount(); i++) { - DataField field = simpleStatsConverters.tableDataFields().get(i); + DataField field = simpleStatsEvolutions.tableDataFields().get(i); String name = field.name(); DataType type = field.type(); lazyNullValueCounts.put( diff --git a/paimon-core/src/main/java/org/apache/paimon/table/system/ManifestsTable.java b/paimon-core/src/main/java/org/apache/paimon/table/system/ManifestsTable.java index 4e07c6c58fcd..d88636d02aab 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/system/ManifestsTable.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/system/ManifestsTable.java @@ -37,6 +37,7 @@ import org.apache.paimon.table.source.SingletonSplit; import org.apache.paimon.table.source.Split; import org.apache.paimon.table.source.TableRead; +import org.apache.paimon.table.source.snapshot.TimeTravelUtil; import org.apache.paimon.types.BigIntType; import org.apache.paimon.types.DataField; import org.apache.paimon.types.RowType; @@ -44,12 +45,12 @@ import org.apache.paimon.utils.IteratorRecordReader; import org.apache.paimon.utils.ProjectedRow; import org.apache.paimon.utils.SerializationUtils; -import org.apache.paimon.utils.SnapshotManager; -import org.apache.paimon.utils.SnapshotNotExistException; -import org.apache.paimon.utils.StringUtils; import org.apache.paimon.shade.guava30.com.google.common.collect.Iterators; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import java.util.Arrays; import java.util.Collections; import java.util.Iterator; @@ -60,6 +61,9 @@ /** A {@link Table} for showing committing snapshots of table. */ public class ManifestsTable implements ReadonlyTable { + + private static final Logger LOG = LoggerFactory.getLogger(ManifestsTable.class); + private static final long serialVersionUID = 1L; public static final String MANIFESTS = "manifests"; @@ -196,39 +200,18 @@ private InternalRow toRow(ManifestFileMeta manifestFileMeta) { } private static List allManifests(FileStoreTable dataTable) { - CoreOptions coreOptions = CoreOptions.fromMap(dataTable.options()); - SnapshotManager snapshotManager = dataTable.snapshotManager(); - Long snapshotId = coreOptions.scanSnapshotId(); - String tagName = coreOptions.scanTagName(); - Snapshot snapshot = null; - if (snapshotId != null) { - // reminder user with snapshot id range - if (!snapshotManager.snapshotExists(snapshotId)) { - Long earliestSnapshotId = snapshotManager.earliestSnapshotId(); - Long latestSnapshotId = snapshotManager.latestSnapshotId(); - throw new SnapshotNotExistException( - String.format( - "Specified scan.snapshot-id %s is not exist, you can set it in range from %s to %s", - snapshotId, earliestSnapshotId, latestSnapshotId)); - } - snapshot = snapshotManager.snapshot(snapshotId); - } else { - if (!StringUtils.isEmpty(tagName) && dataTable.tagManager().tagExists(tagName)) { - snapshot = dataTable.tagManager().tag(tagName).trimToSnapshot(); - } else { - snapshot = snapshotManager.latestSnapshot(); - } - } - + CoreOptions options = dataTable.coreOptions(); + Snapshot snapshot = TimeTravelUtil.resolveSnapshot(dataTable); if (snapshot == null) { + LOG.warn("Check if your snapshot is empty."); return Collections.emptyList(); } FileStorePathFactory fileStorePathFactory = dataTable.store().pathFactory(); ManifestList manifestList = new ManifestList.Factory( dataTable.fileIO(), - coreOptions.manifestFormat(), - coreOptions.manifestCompression(), + options.manifestFormat(), + options.manifestCompression(), fileStorePathFactory, null) .create(); diff --git a/paimon-core/src/main/java/org/apache/paimon/table/system/OptionsTable.java b/paimon-core/src/main/java/org/apache/paimon/table/system/OptionsTable.java index b4a3b82a2f5f..ed20896646b2 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/system/OptionsTable.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/system/OptionsTable.java @@ -18,7 +18,6 @@ package org.apache.paimon.table.system; -import org.apache.paimon.CoreOptions; import org.apache.paimon.data.BinaryString; import org.apache.paimon.data.GenericRow; import org.apache.paimon.data.InternalRow; @@ -27,7 +26,6 @@ import org.apache.paimon.fs.Path; import org.apache.paimon.predicate.Predicate; import org.apache.paimon.reader.RecordReader; -import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.table.FileStoreTable; import org.apache.paimon.table.ReadonlyTable; import org.apache.paimon.table.Table; @@ -44,7 +42,6 @@ import org.apache.paimon.shade.guava30.com.google.common.collect.Iterators; -import java.io.IOException; import java.util.Arrays; import java.util.Collections; import java.util.Iterator; @@ -70,19 +67,13 @@ public class OptionsTable implements ReadonlyTable { private final FileIO fileIO; private final Path location; - private final String branch; - public OptionsTable(FileStoreTable dataTable) { - this( - dataTable.fileIO(), - dataTable.location(), - CoreOptions.branch(dataTable.schema().options())); - } + private final FileStoreTable dataTable; - public OptionsTable(FileIO fileIO, Path location, String branchName) { - this.fileIO = fileIO; - this.location = location; - this.branch = branchName; + public OptionsTable(FileStoreTable dataTable) { + this.fileIO = dataTable.fileIO(); + this.location = dataTable.location(); + this.dataTable = dataTable; } @Override @@ -112,7 +103,7 @@ public InnerTableRead newRead() { @Override public Table copy(Map dynamicOptions) { - return new OptionsTable(fileIO, location, branch); + return new OptionsTable(dataTable.copy(dynamicOptions)); } private class OptionsScan extends ReadOnceTableScan { @@ -182,14 +173,20 @@ public TableRead withIOManager(IOManager ioManager) { } @Override - public RecordReader createReader(Split split) throws IOException { + public RecordReader createReader(Split split) { if (!(split instanceof OptionsSplit)) { throw new IllegalArgumentException("Unsupported split: " + split.getClass()); } - Path location = ((OptionsSplit) split).location; Iterator rows = Iterators.transform( - options(fileIO, location, branch).entrySet().iterator(), this::toRow); + dataTable + .schemaManager() + .latest() + .orElseThrow(() -> new RuntimeException("Table not exists.")) + .options() + .entrySet() + .iterator(), + this::toRow); if (readType != null) { rows = Iterators.transform( @@ -207,11 +204,4 @@ private InternalRow toRow(Map.Entry option) { BinaryString.fromString(option.getValue())); } } - - private static Map options(FileIO fileIO, Path location, String branchName) { - return new SchemaManager(fileIO, location, branchName) - .latest() - .orElseThrow(() -> new RuntimeException("Table not exists.")) - .options(); - } } diff --git a/paimon-core/src/main/java/org/apache/paimon/table/system/PartitionsTable.java b/paimon-core/src/main/java/org/apache/paimon/table/system/PartitionsTable.java index 5e966eabd9a7..7e0b1f1d7568 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/system/PartitionsTable.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/system/PartitionsTable.java @@ -40,6 +40,7 @@ import org.apache.paimon.types.DataTypes; import org.apache.paimon.types.RowType; import org.apache.paimon.utils.IteratorRecordReader; +import org.apache.paimon.utils.Pair; import org.apache.paimon.utils.ProjectedRow; import org.apache.paimon.utils.RowDataToObjectArrayConverter; import org.apache.paimon.utils.SerializationUtils; @@ -53,9 +54,11 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.Comparator; import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; import static org.apache.paimon.catalog.Catalog.SYSTEM_TABLE_SPLITTER; @@ -175,15 +178,27 @@ public RecordReader createReader(Split split) throws IOException { throw new IllegalArgumentException("Unsupported split: " + split.getClass()); } - List partitions = fileStoreTable.newSnapshotReader().partitionEntries(); + List partitions = fileStoreTable.newScan().listPartitionEntries(); RowDataToObjectArrayConverter converter = new RowDataToObjectArrayConverter( fileStoreTable.schema().logicalPartitionType()); + // sorted by partition + List> partitionList = + partitions.stream() + .map( + entry -> + Pair.of( + Arrays.toString( + converter.convert(entry.partition())), + entry)) + .sorted(Comparator.comparing(Pair::getLeft)) + .collect(Collectors.toList()); + List results = new ArrayList<>(partitions.size()); - for (PartitionEntry entry : partitions) { - results.add(toRow(entry, converter)); + for (Pair pair : partitionList) { + results.add(toRow(pair.getLeft(), pair.getRight())); } Iterator iterator = results.iterator(); @@ -198,13 +213,9 @@ public RecordReader createReader(Split split) throws IOException { return new IteratorRecordReader<>(iterator); } - private GenericRow toRow( - PartitionEntry entry, RowDataToObjectArrayConverter partitionConverter) { - BinaryString partitionId = - BinaryString.fromString( - Arrays.toString(partitionConverter.convert(entry.partition()))); + private GenericRow toRow(String partStr, PartitionEntry entry) { return GenericRow.of( - partitionId, + BinaryString.fromString(partStr), entry.recordCount(), entry.fileSizeInBytes(), entry.fileCount(), diff --git a/paimon-core/src/main/java/org/apache/paimon/table/system/ReadOptimizedTable.java b/paimon-core/src/main/java/org/apache/paimon/table/system/ReadOptimizedTable.java index e28ae3760534..5308005053c8 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/system/ReadOptimizedTable.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/system/ReadOptimizedTable.java @@ -26,6 +26,7 @@ import org.apache.paimon.manifest.ManifestEntry; import org.apache.paimon.manifest.ManifestFileMeta; import org.apache.paimon.operation.DefaultValueAssigner; +import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.table.DataTable; import org.apache.paimon.table.FileStoreTable; import org.apache.paimon.table.ReadonlyTable; @@ -120,7 +121,8 @@ public List primaryKeys() { public SnapshotReader newSnapshotReader() { if (wrapped.schema().primaryKeys().size() > 0) { return wrapped.newSnapshotReader() - .withLevelFilter(level -> level == coreOptions().numLevels() - 1); + .withLevelFilter(level -> level == coreOptions().numLevels() - 1) + .enableValueFilter(); } else { return wrapped.newSnapshotReader(); } @@ -164,6 +166,11 @@ public SnapshotManager snapshotManager() { return wrapped.snapshotManager(); } + @Override + public SchemaManager schemaManager() { + return wrapped.schemaManager(); + } + @Override public TagManager tagManager() { return wrapped.tagManager(); diff --git a/paimon-core/src/main/java/org/apache/paimon/table/system/SchemasTable.java b/paimon-core/src/main/java/org/apache/paimon/table/system/SchemasTable.java index aab6c1d876af..3cb0ff4783e9 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/system/SchemasTable.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/system/SchemasTable.java @@ -18,23 +18,23 @@ package org.apache.paimon.table.system; -import org.apache.paimon.CoreOptions; import org.apache.paimon.data.BinaryString; import org.apache.paimon.data.GenericRow; import org.apache.paimon.data.InternalRow; import org.apache.paimon.data.Timestamp; import org.apache.paimon.disk.IOManager; -import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.Path; import org.apache.paimon.predicate.And; import org.apache.paimon.predicate.CompoundPredicate; import org.apache.paimon.predicate.Equal; import org.apache.paimon.predicate.GreaterOrEqual; import org.apache.paimon.predicate.GreaterThan; +import org.apache.paimon.predicate.InPredicateVisitor; import org.apache.paimon.predicate.LeafPredicate; import org.apache.paimon.predicate.LeafPredicateExtractor; import org.apache.paimon.predicate.LessOrEqual; import org.apache.paimon.predicate.LessThan; +import org.apache.paimon.predicate.Or; import org.apache.paimon.predicate.Predicate; import org.apache.paimon.reader.RecordReader; import org.apache.paimon.schema.SchemaManager; @@ -59,18 +59,16 @@ import org.apache.paimon.shade.guava30.com.google.common.collect.Iterators; -import javax.annotation.Nullable; - import java.time.Instant; import java.time.LocalDateTime; import java.time.ZoneId; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.Iterator; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.Optional; import static org.apache.paimon.catalog.Catalog.SYSTEM_TABLE_SPLITTER; @@ -95,21 +93,13 @@ public class SchemasTable implements ReadonlyTable { new DataField(5, "comment", SerializationUtils.newStringType(true)), new DataField(6, "update_time", new TimestampType(false, 3)))); - private final FileIO fileIO; private final Path location; - private final String branch; - public SchemasTable(FileStoreTable dataTable) { - this( - dataTable.fileIO(), - dataTable.location(), - CoreOptions.branch(dataTable.schema().options())); - } + private final FileStoreTable dataTable; - public SchemasTable(FileIO fileIO, Path location, String branchName) { - this.fileIO = fileIO; - this.location = location; - this.branch = branchName; + public SchemasTable(FileStoreTable dataTable) { + this.location = dataTable.location(); + this.dataTable = dataTable; } @Override @@ -134,32 +124,24 @@ public InnerTableScan newScan() { @Override public InnerTableRead newRead() { - return new SchemasRead(fileIO); + return new SchemasRead(); } @Override public Table copy(Map dynamicOptions) { - return new SchemasTable(fileIO, location, branch); + return new SchemasTable(dataTable.copy(dynamicOptions)); } - private class SchemasScan extends ReadOnceTableScan { - private @Nullable LeafPredicate schemaId; + private static class SchemasScan extends ReadOnceTableScan { @Override - public InnerTableScan withFilter(Predicate predicate) { - if (predicate == null) { - return this; - } - - Map leafPredicates = - predicate.visit(LeafPredicateExtractor.INSTANCE); - schemaId = leafPredicates.get("schema_id"); - return this; + public Plan innerPlan() { + return () -> Collections.singletonList(new SchemasSplit()); } @Override - public Plan innerPlan() { - return () -> Collections.singletonList(new SchemasSplit(location, schemaId)); + public InnerTableScan withFilter(Predicate predicate) { + return this; } } @@ -168,45 +150,28 @@ private static class SchemasSplit extends SingletonSplit { private static final long serialVersionUID = 1L; - private final Path location; - - private final @Nullable LeafPredicate schemaId; - - private SchemasSplit(Path location, @Nullable LeafPredicate schemaId) { - this.location = location; - this.schemaId = schemaId; - } - + @Override public boolean equals(Object o) { if (this == o) { return true; } - if (o == null || getClass() != o.getClass()) { - return false; - } - SchemasSplit that = (SchemasSplit) o; - return Objects.equals(location, that.location) - && Objects.equals(schemaId, that.schemaId); + return o != null && getClass() == o.getClass(); } @Override public int hashCode() { - return Objects.hash(location, schemaId); + return 0; } } /** {@link TableRead} implementation for {@link SchemasTable}. */ private class SchemasRead implements InnerTableRead { - private final FileIO fileIO; private RowType readType; private Optional optionalFilterSchemaIdMax = Optional.empty(); private Optional optionalFilterSchemaIdMin = Optional.empty(); - - public SchemasRead(FileIO fileIO) { - this.fileIO = fileIO; - } + private final List schemaIds = new ArrayList<>(); @Override public InnerTableRead withFilter(Predicate predicate) { @@ -223,6 +188,18 @@ public InnerTableRead withFilter(Predicate predicate) { handleLeafPredicate(leaf, leafName); } } + + // optimize for IN filter + if ((compoundPredicate.function()) instanceof Or) { + InPredicateVisitor.extractInElements(predicate, leafName) + .ifPresent( + leafs -> + leafs.forEach( + leaf -> + schemaIds.add( + Long.parseLong( + leaf.toString())))); + } } else { handleLeafPredicate(predicate, leafName); } @@ -275,12 +252,16 @@ public RecordReader createReader(Split split) { if (!(split instanceof SchemasSplit)) { throw new IllegalArgumentException("Unsupported split: " + split.getClass()); } - SchemasSplit schemasSplit = (SchemasSplit) split; - Path location = schemasSplit.location; - SchemaManager manager = new SchemaManager(fileIO, location, branch); + SchemaManager manager = dataTable.schemaManager(); + + Collection tableSchemas; + if (!schemaIds.isEmpty()) { + tableSchemas = manager.schemasWithId(schemaIds); + } else { + tableSchemas = + manager.listWithRange(optionalFilterSchemaIdMax, optionalFilterSchemaIdMin); + } - Collection tableSchemas = - manager.listWithRange(optionalFilterSchemaIdMax, optionalFilterSchemaIdMin); Iterator rows = Iterators.transform(tableSchemas.iterator(), this::toRow); if (readType != null) { rows = diff --git a/paimon-core/src/main/java/org/apache/paimon/table/system/SinkTableLineageTable.java b/paimon-core/src/main/java/org/apache/paimon/table/system/SinkTableLineageTable.java deleted file mode 100644 index 71efce070471..000000000000 --- a/paimon-core/src/main/java/org/apache/paimon/table/system/SinkTableLineageTable.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.paimon.table.system; - -import org.apache.paimon.lineage.LineageMeta; -import org.apache.paimon.lineage.LineageMetaFactory; -import org.apache.paimon.options.Options; -import org.apache.paimon.table.Table; -import org.apache.paimon.table.source.InnerTableRead; - -import java.util.Map; - -/** - * This is a system table to display all the sink table lineages. - * - * 
    - *  For example:
- *     If we select * from sys.sink_table_lineage, we will get
- *     database_name       table_name       job_name      create_time
- *        default            test0            job1    2023-10-22 20:35:12
- *       database1           test1            job1    2023-10-28 21:35:52
- *          ...               ...             ...             ...
- *     We can write sql to fetch the information we need.
- *
    - */ -public class SinkTableLineageTable extends TableLineageTable { - - public static final String SINK_TABLE_LINEAGE = "sink_table_lineage"; - - public SinkTableLineageTable(LineageMetaFactory lineageMetaFactory, Options options) { - super(lineageMetaFactory, options); - } - - @Override - public InnerTableRead newRead() { - return new TableLineageRead(lineageMetaFactory, options, LineageMeta::sinkTableLineages); - } - - @Override - public String name() { - return SINK_TABLE_LINEAGE; - } - - @Override - public Table copy(Map dynamicOptions) { - return new SinkTableLineageTable(lineageMetaFactory, options); - } -} diff --git a/paimon-core/src/main/java/org/apache/paimon/table/system/SnapshotsTable.java b/paimon-core/src/main/java/org/apache/paimon/table/system/SnapshotsTable.java index 5bec2b109324..2af13ee937bd 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/system/SnapshotsTable.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/system/SnapshotsTable.java @@ -18,7 +18,6 @@ package org.apache.paimon.table.system; -import org.apache.paimon.CoreOptions; import org.apache.paimon.Snapshot; import org.apache.paimon.data.BinaryString; import org.apache.paimon.data.GenericRow; @@ -32,10 +31,12 @@ import org.apache.paimon.predicate.Equal; import org.apache.paimon.predicate.GreaterOrEqual; import org.apache.paimon.predicate.GreaterThan; +import org.apache.paimon.predicate.InPredicateVisitor; import org.apache.paimon.predicate.LeafPredicate; import org.apache.paimon.predicate.LeafPredicateExtractor; import org.apache.paimon.predicate.LessOrEqual; import org.apache.paimon.predicate.LessThan; +import org.apache.paimon.predicate.Or; import org.apache.paimon.predicate.Predicate; import org.apache.paimon.reader.RecordReader; import org.apache.paimon.table.FileStoreTable; @@ -62,6 +63,7 @@ import java.time.Instant; import java.time.LocalDateTime; import java.time.ZoneId; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.Iterator; @@ -108,24 +110,13 @@ public class SnapshotsTable implements ReadonlyTable { private final FileIO fileIO; private final Path location; - private final String branch; private final FileStoreTable dataTable; public SnapshotsTable(FileStoreTable dataTable) { - this( - dataTable.fileIO(), - dataTable.location(), - dataTable, - CoreOptions.branch(dataTable.schema().options())); - } - - public SnapshotsTable( - FileIO fileIO, Path location, FileStoreTable dataTable, String branchName) { - this.fileIO = fileIO; - this.location = location; + this.fileIO = dataTable.fileIO(); + this.location = dataTable.location(); this.dataTable = dataTable; - this.branch = branchName; } @Override @@ -155,7 +146,7 @@ public InnerTableRead newRead() { @Override public Table copy(Map dynamicOptions) { - return new SnapshotsTable(fileIO, location, dataTable.copy(dynamicOptions), branch); + return new SnapshotsTable(dataTable.copy(dynamicOptions)); } private class SnapshotsScan extends ReadOnceTableScan { @@ -206,6 +197,7 @@ private class SnapshotsRead implements InnerTableRead { private RowType readType; private Optional optionalFilterSnapshotIdMax = Optional.empty(); private Optional optionalFilterSnapshotIdMin = Optional.empty(); + private final List snapshotIds = new ArrayList<>(); public SnapshotsRead(FileIO fileIO) { this.fileIO = fileIO; @@ -220,12 +212,24 @@ public InnerTableRead withFilter(Predicate predicate) { String leafName = "snapshot_id"; if (predicate instanceof CompoundPredicate) { CompoundPredicate compoundPredicate = (CompoundPredicate) predicate; + List children = compoundPredicate.children(); if ((compoundPredicate.function()) instanceof And) { - List children = compoundPredicate.children(); for (Predicate leaf : children) { handleLeafPredicate(leaf, leafName); } } + + // optimize for IN filter + if ((compoundPredicate.function()) instanceof Or) { + InPredicateVisitor.extractInElements(predicate, leafName) + .ifPresent( + leafs -> + leafs.forEach( + leaf -> + snapshotIds.add( + Long.parseLong( + leaf.toString())))); + } } else { handleLeafPredicate(predicate, leafName); } @@ -282,11 +286,16 @@ public RecordReader createReader(Split split) throws IOException { if (!(split instanceof SnapshotsSplit)) { throw new IllegalArgumentException("Unsupported split: " + split.getClass()); } - SnapshotManager snapshotManager = - new SnapshotManager(fileIO, ((SnapshotsSplit) split).location, branch); - Iterator snapshots = - snapshotManager.snapshotsWithinRange( - optionalFilterSnapshotIdMax, optionalFilterSnapshotIdMin); + + SnapshotManager snapshotManager = dataTable.snapshotManager(); + Iterator snapshots; + if (!snapshotIds.isEmpty()) { + snapshots = snapshotManager.snapshotsWithId(snapshotIds); + } else { + snapshots = + snapshotManager.snapshotsWithinRange( + optionalFilterSnapshotIdMax, optionalFilterSnapshotIdMin); + } Iterator rows = Iterators.transform(snapshots, this::toRow); if (readType != null) { diff --git a/paimon-core/src/main/java/org/apache/paimon/table/system/SourceTableLineageTable.java b/paimon-core/src/main/java/org/apache/paimon/table/system/SourceTableLineageTable.java deleted file mode 100644 index 5d9904fa6675..000000000000 --- a/paimon-core/src/main/java/org/apache/paimon/table/system/SourceTableLineageTable.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.paimon.table.system; - -import org.apache.paimon.lineage.LineageMeta; -import org.apache.paimon.lineage.LineageMetaFactory; -import org.apache.paimon.options.Options; -import org.apache.paimon.table.Table; -import org.apache.paimon.table.source.InnerTableRead; - -import java.util.Map; - -/** - * This is a system table to display all the source table lineages. - * - * 
    - *  For example:
- *     If we select * from sys.source_table_lineage, we will get
- *     database_name       table_name       job_name      create_time
- *        default            test0            job1    2023-10-22 20:35:12
- *       database1           test1            job1    2023-10-28 21:35:52
- *          ...               ...             ...             ...
- *     We can write sql to fetch the information we need.
- *
    - */ -public class SourceTableLineageTable extends TableLineageTable { - - public static final String SOURCE_TABLE_LINEAGE = "source_table_lineage"; - - public SourceTableLineageTable(LineageMetaFactory lineageMetaFactory, Options options) { - super(lineageMetaFactory, options); - } - - @Override - public InnerTableRead newRead() { - return new TableLineageRead(lineageMetaFactory, options, LineageMeta::sourceTableLineages); - } - - @Override - public String name() { - return SOURCE_TABLE_LINEAGE; - } - - @Override - public Table copy(Map dynamicOptions) { - return new SourceTableLineageTable(lineageMetaFactory, options); - } -} diff --git a/paimon-core/src/main/java/org/apache/paimon/table/system/StatisticTable.java b/paimon-core/src/main/java/org/apache/paimon/table/system/StatisticTable.java index faaa654ebe48..f0180444988c 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/system/StatisticTable.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/system/StatisticTable.java @@ -22,7 +22,6 @@ import org.apache.paimon.data.GenericRow; import org.apache.paimon.data.InternalRow; import org.apache.paimon.disk.IOManager; -import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.Path; import org.apache.paimon.predicate.Predicate; import org.apache.paimon.reader.EmptyRecordReader; @@ -47,7 +46,6 @@ import org.apache.paimon.shade.guava30.com.google.common.collect.Iterators; -import java.io.IOException; import java.util.Arrays; import java.util.Collections; import java.util.Iterator; @@ -74,18 +72,9 @@ public class StatisticTable implements ReadonlyTable { new DataField(3, "mergedRecordSize", new BigIntType(true)), new DataField(4, "colstat", SerializationUtils.newStringType(true)))); - private final FileIO fileIO; - private final Path location; - private final FileStoreTable dataTable; public StatisticTable(FileStoreTable dataTable) { - this(dataTable.fileIO(), dataTable.location(), dataTable); - } - - public StatisticTable(FileIO fileIO, Path location, FileStoreTable dataTable) { - this.fileIO = fileIO; - this.location = location; this.dataTable = dataTable; } @@ -111,12 +100,12 @@ public InnerTableScan newScan() { @Override public InnerTableRead newRead() { - return new StatisticRead(fileIO, dataTable); + return new StatisticRead(dataTable); } @Override public Table copy(Map dynamicOptions) { - return new StatisticTable(fileIO, location, dataTable.copy(dynamicOptions)); + return new StatisticTable(dataTable.copy(dynamicOptions)); } private class StatisticScan extends ReadOnceTableScan { @@ -129,7 +118,9 @@ public InnerTableScan withFilter(Predicate predicate) { @Override public Plan innerPlan() { - return () -> Collections.singletonList(new StatisticTable.StatisticSplit(location)); + return () -> + Collections.singletonList( + new StatisticTable.StatisticSplit(dataTable.location())); } } @@ -163,13 +154,11 @@ public int hashCode() { private static class StatisticRead implements InnerTableRead { - private final FileIO fileIO; private RowType readType; private final FileStoreTable dataTable; - public StatisticRead(FileIO fileIO, FileStoreTable dataTable) { - this.fileIO = fileIO; + public StatisticRead(FileStoreTable dataTable) { this.dataTable = dataTable; } @@ -191,7 +180,7 @@ public TableRead withIOManager(IOManager ioManager) { } @Override - public RecordReader createReader(Split split) throws IOException { + public RecordReader createReader(Split split) { if (!(split instanceof StatisticTable.StatisticSplit)) { throw new IllegalArgumentException("Unsupported split: " + split.getClass()); } diff --git a/paimon-core/src/main/java/org/apache/paimon/table/system/SystemTableLoader.java b/paimon-core/src/main/java/org/apache/paimon/table/system/SystemTableLoader.java index a84f41ec1a51..763e4d121673 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/system/SystemTableLoader.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/system/SystemTableLoader.java @@ -20,7 +20,6 @@ import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.Path; -import org.apache.paimon.lineage.LineageMetaFactory; import org.apache.paimon.options.Options; import org.apache.paimon.table.FileStoreTable; import org.apache.paimon.table.Table; @@ -37,10 +36,10 @@ import java.util.function.Function; import java.util.function.Supplier; -import static org.apache.paimon.options.CatalogOptions.LINEAGE_META; import static org.apache.paimon.table.system.AggregationFieldsTable.AGGREGATION_FIELDS; import static org.apache.paimon.table.system.AllTableOptionsTable.ALL_TABLE_OPTIONS; import static org.apache.paimon.table.system.AuditLogTable.AUDIT_LOG; +import static org.apache.paimon.table.system.BinlogTable.BINLOG; import static org.apache.paimon.table.system.BranchesTable.BRANCHES; import static org.apache.paimon.table.system.BucketsTable.BUCKETS; import static org.apache.paimon.table.system.CatalogOptionsTable.CATALOG_OPTIONS; @@ -51,12 +50,9 @@ import static org.apache.paimon.table.system.PartitionsTable.PARTITIONS; import static org.apache.paimon.table.system.ReadOptimizedTable.READ_OPTIMIZED; import static org.apache.paimon.table.system.SchemasTable.SCHEMAS; -import static org.apache.paimon.table.system.SinkTableLineageTable.SINK_TABLE_LINEAGE; import static org.apache.paimon.table.system.SnapshotsTable.SNAPSHOTS; -import static org.apache.paimon.table.system.SourceTableLineageTable.SOURCE_TABLE_LINEAGE; import static org.apache.paimon.table.system.StatisticTable.STATISTICS; import static org.apache.paimon.table.system.TagsTable.TAGS; -import static org.apache.paimon.utils.Preconditions.checkNotNull; /** Loader to load system {@link Table}s. */ public class SystemTableLoader { @@ -77,6 +73,7 @@ public class SystemTableLoader { .put(READ_OPTIMIZED, ReadOptimizedTable::new) .put(AGGREGATION_FIELDS, AggregationFieldsTable::new) .put(STATISTICS, StatisticTable::new) + .put(BINLOG, BinlogTable::new) .build(); public static final List SYSTEM_TABLES = new ArrayList<>(SYSTEM_TABLE_LOADERS.keySet()); @@ -93,38 +90,18 @@ public static Table loadGlobal( String tableName, FileIO fileIO, Supplier>> allTablePaths, - Options catalogOptions, - @Nullable LineageMetaFactory lineageMetaFactory) { + Options catalogOptions) { switch (tableName.toLowerCase()) { case ALL_TABLE_OPTIONS: return new AllTableOptionsTable(fileIO, allTablePaths.get()); case CATALOG_OPTIONS: return new CatalogOptionsTable(catalogOptions); - case SOURCE_TABLE_LINEAGE: - { - checkNotNull( - lineageMetaFactory, - String.format( - "Lineage meta should be configured for catalog with %s", - LINEAGE_META.key())); - return new SourceTableLineageTable(lineageMetaFactory, catalogOptions); - } - case SINK_TABLE_LINEAGE: - { - checkNotNull( - lineageMetaFactory, - String.format( - "Lineage meta should be configured for catalog with %s", - LINEAGE_META.key())); - return new SinkTableLineageTable(lineageMetaFactory, catalogOptions); - } default: return null; } } public static List loadGlobalTableNames() { - return Arrays.asList( - ALL_TABLE_OPTIONS, CATALOG_OPTIONS, SOURCE_TABLE_LINEAGE, SINK_TABLE_LINEAGE); + return Arrays.asList(ALL_TABLE_OPTIONS, CATALOG_OPTIONS); } } diff --git a/paimon-core/src/main/java/org/apache/paimon/table/system/TableLineageTable.java b/paimon-core/src/main/java/org/apache/paimon/table/system/TableLineageTable.java deleted file mode 100644 index aeaf3ca3b133..000000000000 --- a/paimon-core/src/main/java/org/apache/paimon/table/system/TableLineageTable.java +++ /dev/null @@ -1,168 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.paimon.table.system; - -import org.apache.paimon.data.BinaryString; -import org.apache.paimon.data.GenericRow; -import org.apache.paimon.data.InternalRow; -import org.apache.paimon.disk.IOManager; -import org.apache.paimon.lineage.LineageMeta; -import org.apache.paimon.lineage.LineageMetaFactory; -import org.apache.paimon.lineage.TableLineageEntity; -import org.apache.paimon.options.Options; -import org.apache.paimon.predicate.Predicate; -import org.apache.paimon.reader.RecordReader; -import org.apache.paimon.table.ReadonlyTable; -import org.apache.paimon.table.source.InnerTableRead; -import org.apache.paimon.table.source.InnerTableScan; -import org.apache.paimon.table.source.ReadOnceTableScan; -import org.apache.paimon.table.source.Split; -import org.apache.paimon.table.source.TableRead; -import org.apache.paimon.types.DataField; -import org.apache.paimon.types.RowType; -import org.apache.paimon.types.TimestampType; -import org.apache.paimon.types.VarCharType; -import org.apache.paimon.utils.IteratorRecordReader; -import org.apache.paimon.utils.ProjectedRow; - -import org.apache.paimon.shade.guava30.com.google.common.collect.Iterators; - -import javax.annotation.Nullable; - -import java.io.IOException; -import java.util.Arrays; -import java.util.Collections; -import java.util.Iterator; -import java.util.List; -import java.util.function.BiFunction; - -import static org.apache.paimon.utils.Preconditions.checkNotNull; - -/** Base lineage table for source and sink table lineage. */ -public abstract class TableLineageTable implements ReadonlyTable { - protected final LineageMetaFactory lineageMetaFactory; - protected final Options options; - - public static final RowType TABLE_TYPE = - new RowType( - Arrays.asList( - new DataField( - 0, "database_name", new VarCharType(VarCharType.MAX_LENGTH)), - new DataField(1, "table_name", new VarCharType(VarCharType.MAX_LENGTH)), - new DataField(2, "job_name", new VarCharType(VarCharType.MAX_LENGTH)), - new DataField(3, "create_time", new TimestampType()))); - - protected TableLineageTable(LineageMetaFactory lineageMetaFactory, Options options) { - this.lineageMetaFactory = lineageMetaFactory; - this.options = options; - } - - @Override - public InnerTableScan newScan() { - return new ReadOnceTableScan() { - @Override - public InnerTableScan withFilter(Predicate predicate) { - return this; - } - - @Override - protected Plan innerPlan() { - /// TODO get the real row count for plan. - return () -> Collections.singletonList((Split) () -> 1L); - } - }; - } - - @Override - public RowType rowType() { - return TABLE_TYPE; - } - - @Override - public List primaryKeys() { - return Arrays.asList("database_name", "table_name", "job_name"); - } - - /** Table lineage read with lineage meta query. */ - protected static class TableLineageRead implements InnerTableRead { - private final LineageMetaFactory lineageMetaFactory; - private final Options options; - private final BiFunction> - tableLineageQuery; - @Nullable private Predicate predicate; - private RowType readType; - - protected TableLineageRead( - LineageMetaFactory lineageMetaFactory, - Options options, - BiFunction> - tableLineageQuery) { - this.lineageMetaFactory = lineageMetaFactory; - this.options = options; - this.tableLineageQuery = tableLineageQuery; - this.predicate = null; - } - - @Override - public InnerTableRead withFilter(Predicate predicate) { - this.predicate = predicate; - return this; - } - - @Override - public InnerTableRead withReadType(RowType readType) { - this.readType = readType; - return this; - } - - @Override - public TableRead withIOManager(IOManager ioManager) { - return this; - } - - @Override - public RecordReader createReader(Split split) throws IOException { - try (LineageMeta lineageMeta = lineageMetaFactory.create(() -> options)) { - Iterator sourceTableLineages = - tableLineageQuery.apply(lineageMeta, predicate); - return new IteratorRecordReader<>( - Iterators.transform( - sourceTableLineages, - entity -> { - checkNotNull(entity); - GenericRow row = - GenericRow.of( - BinaryString.fromString(entity.getDatabase()), - BinaryString.fromString(entity.getTable()), - BinaryString.fromString(entity.getJob()), - entity.getCreateTime()); - if (readType != null) { - return ProjectedRow.from( - readType, TableLineageTable.TABLE_TYPE) - .replaceRow(row); - } else { - return row; - } - })); - } catch (Exception e) { - throw new RuntimeException(e); - } - } - } -} diff --git a/paimon-core/src/main/java/org/apache/paimon/table/system/TagsTable.java b/paimon-core/src/main/java/org/apache/paimon/table/system/TagsTable.java index abce3f33027e..9aafdb5983fd 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/system/TagsTable.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/system/TagsTable.java @@ -26,9 +26,12 @@ import org.apache.paimon.disk.IOManager; import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.Path; +import org.apache.paimon.predicate.CompoundPredicate; import org.apache.paimon.predicate.Equal; +import org.apache.paimon.predicate.InPredicateVisitor; import org.apache.paimon.predicate.LeafPredicate; import org.apache.paimon.predicate.LeafPredicateExtractor; +import org.apache.paimon.predicate.Or; import org.apache.paimon.predicate.Predicate; import org.apache.paimon.reader.RecordReader; import org.apache.paimon.table.FileStoreTable; @@ -59,11 +62,11 @@ import java.util.Arrays; import java.util.Collections; import java.util.Iterator; -import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.TreeMap; import static org.apache.paimon.catalog.Catalog.SYSTEM_TABLE_SPLITTER; @@ -74,10 +77,12 @@ public class TagsTable implements ReadonlyTable { public static final String TAGS = "tags"; + private static final String TAG_NAME = "tag_name"; + public static final RowType TABLE_TYPE = new RowType( Arrays.asList( - new DataField(0, "tag_name", SerializationUtils.newStringType(false)), + new DataField(0, TAG_NAME, SerializationUtils.newStringType(false)), new DataField(1, "snapshot_id", new BigIntType(false)), new DataField(2, "schema_id", new BigIntType(false)), new DataField(3, "commit_time", new TimestampType(false, 3)), @@ -90,17 +95,13 @@ public class TagsTable implements ReadonlyTable { private final Path location; private final String branch; - public TagsTable(FileStoreTable dataTable) { - this( - dataTable.fileIO(), - dataTable.location(), - CoreOptions.branch(dataTable.schema().options())); - } + private final FileStoreTable dataTable; - public TagsTable(FileIO fileIO, Path location, String branchName) { - this.fileIO = fileIO; - this.location = location; - this.branch = branchName; + public TagsTable(FileStoreTable dataTable) { + this.fileIO = dataTable.fileIO(); + this.location = dataTable.location(); + this.branch = CoreOptions.branch(dataTable.schema().options()); + this.dataTable = dataTable; } @Override @@ -115,7 +116,7 @@ public RowType rowType() { @Override public List primaryKeys() { - return Collections.singletonList("tag_name"); + return Collections.singletonList(TAG_NAME); } @Override @@ -130,27 +131,24 @@ public InnerTableRead newRead() { @Override public Table copy(Map dynamicOptions) { - return new TagsTable(fileIO, location, branch); + return new TagsTable(dataTable.copy(dynamicOptions)); } private class TagsScan extends ReadOnceTableScan { - private @Nullable LeafPredicate tagName; + private @Nullable Predicate tagPredicate; @Override public InnerTableScan withFilter(Predicate predicate) { if (predicate == null) { return this; } - // TODO - Map leafPredicates = - predicate.visit(LeafPredicateExtractor.INSTANCE); - tagName = leafPredicates.get("tag_name"); + tagPredicate = predicate; return this; } @Override public Plan innerPlan() { - return () -> Collections.singletonList(new TagsSplit(location, tagName)); + return () -> Collections.singletonList(new TagsSplit(location, tagPredicate)); } } @@ -160,11 +158,11 @@ private static class TagsSplit extends SingletonSplit { private final Path location; - private final @Nullable LeafPredicate tagName; + private final @Nullable Predicate tagPredicate; - private TagsSplit(Path location, @Nullable LeafPredicate tagName) { + private TagsSplit(Path location, @Nullable Predicate tagPredicate) { this.location = location; - this.tagName = tagName; + this.tagPredicate = tagPredicate; } @Override @@ -176,7 +174,8 @@ public boolean equals(Object o) { return false; } TagsSplit that = (TagsSplit) o; - return Objects.equals(location, that.location) && Objects.equals(tagName, that.tagName); + return Objects.equals(location, that.location) + && Objects.equals(tagPredicate, that.tagPredicate); } @Override @@ -217,18 +216,44 @@ public RecordReader createReader(Split split) { throw new IllegalArgumentException("Unsupported split: " + split.getClass()); } Path location = ((TagsSplit) split).location; - LeafPredicate predicate = ((TagsSplit) split).tagName; + Predicate predicate = ((TagsSplit) split).tagPredicate; TagManager tagManager = new TagManager(fileIO, location, branch); - Map nameToSnapshot = new LinkedHashMap<>(); + Map nameToSnapshot = new TreeMap<>(); + Map predicateMap = new TreeMap<>(); + if (predicate != null) { + if (predicate instanceof LeafPredicate + && ((LeafPredicate) predicate).function() instanceof Equal + && ((LeafPredicate) predicate).literals().get(0) instanceof BinaryString + && predicate.visit(LeafPredicateExtractor.INSTANCE).get(TAG_NAME) != null) { + String equalValue = ((LeafPredicate) predicate).literals().get(0).toString(); + if (tagManager.tagExists(equalValue)) { + predicateMap.put(equalValue, tagManager.tag(equalValue)); + } + } - if (predicate != null - && predicate.function() instanceof Equal - && predicate.literals().get(0) instanceof BinaryString) { - String equalValue = predicate.literals().get(0).toString(); - if (tagManager.tagExists(equalValue)) { - nameToSnapshot.put(equalValue, tagManager.tag(equalValue)); + if (predicate instanceof CompoundPredicate) { + CompoundPredicate compoundPredicate = (CompoundPredicate) predicate; + // optimize for IN filter + if ((compoundPredicate.function()) instanceof Or) { + InPredicateVisitor.extractInElements(predicate, TAG_NAME) + .ifPresent( + leafs -> + leafs.forEach( + leaf -> { + String leftName = leaf.toString(); + if (tagManager.tagExists(leftName)) { + predicateMap.put( + leftName, + tagManager.tag(leftName)); + } + })); + } } + } + + if (!predicateMap.isEmpty()) { + nameToSnapshot.putAll(predicateMap); } else { for (Pair tag : tagManager.tagObjects()) { nameToSnapshot.put(tag.getValue(), tag.getKey()); diff --git a/paimon-core/src/main/java/org/apache/paimon/tag/Tag.java b/paimon-core/src/main/java/org/apache/paimon/tag/Tag.java index f1ac879d33a7..53641a2eb69f 100644 --- a/paimon-core/src/main/java/org/apache/paimon/tag/Tag.java +++ b/paimon-core/src/main/java/org/apache/paimon/tag/Tag.java @@ -33,6 +33,7 @@ import java.io.FileNotFoundException; import java.io.IOException; +import java.io.UncheckedIOException; import java.time.Duration; import java.time.LocalDateTime; import java.util.Map; @@ -113,29 +114,6 @@ public String toJson() { return JsonSerdeUtil.toJson(this); } - public static Tag fromJson(String json) { - return JsonSerdeUtil.fromJson(json, Tag.class); - } - - public static Tag fromPath(FileIO fileIO, Path path) { - try { - String json = fileIO.readFileUtf8(path); - return Tag.fromJson(json); - } catch (IOException e) { - throw new RuntimeException("Fails to read tag from path " + path, e); - } - } - - @Nullable - public static Tag safelyFromPath(FileIO fileIO, Path path) throws IOException { - try { - String json = fileIO.readFileUtf8(path); - return Tag.fromJson(json); - } catch (FileNotFoundException e) { - return null; - } - } - public static Tag fromSnapshotAndTagTtl( Snapshot snapshot, Duration tagTimeRetained, LocalDateTime tagCreateTime) { return new Tag( @@ -201,4 +179,28 @@ public boolean equals(Object o) { return Objects.equals(tagCreateTime, that.tagCreateTime) && Objects.equals(tagTimeRetained, that.tagTimeRetained); } + + // =================== Utils for reading ========================= + + public static Tag fromJson(String json) { + return JsonSerdeUtil.fromJson(json, Tag.class); + } + + public static Tag fromPath(FileIO fileIO, Path path) { + try { + return tryFromPath(fileIO, path); + } catch (FileNotFoundException e) { + throw new RuntimeException(e.getMessage(), e); + } + } + + public static Tag tryFromPath(FileIO fileIO, Path path) throws FileNotFoundException { + try { + return fromJson(fileIO.readFileUtf8(path)); + } catch (FileNotFoundException e) { + throw e; + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } } diff --git a/paimon-core/src/main/java/org/apache/paimon/tag/TagAutoCreation.java b/paimon-core/src/main/java/org/apache/paimon/tag/TagAutoCreation.java index 58241033f5fb..3989786bd277 100644 --- a/paimon-core/src/main/java/org/apache/paimon/tag/TagAutoCreation.java +++ b/paimon-core/src/main/java/org/apache/paimon/tag/TagAutoCreation.java @@ -150,22 +150,28 @@ public void run() { private void tryToCreateTags(Snapshot snapshot) { Optional timeOptional = timeExtractor.extract(snapshot.timeMillis(), snapshot.watermark()); + LOG.info("Starting to create a tag for snapshot {}.", snapshot.id()); if (!timeOptional.isPresent()) { return; } LocalDateTime time = timeOptional.get(); + LOG.info("The time of snapshot {} is {}.", snapshot.id(), time); + LOG.info("The next tag time is {}.", nextTag); if (nextTag == null || isAfterOrEqual(time.minus(delay), periodHandler.nextTagTime(nextTag))) { LocalDateTime thisTag = periodHandler.normalizeToPreviousTag(time); + LOG.info("Create tag for snapshot {} with time {}.", snapshot.id(), thisTag); if (automaticCompletion && nextTag != null) { thisTag = nextTag; } String tagName = periodHandler.timeToTag(thisTag); + LOG.info("The tag name is {}.", tagName); if (!tagManager.tagExists(tagName)) { tagManager.createTag(snapshot, tagName, defaultTimeRetained, callbacks); } nextTag = periodHandler.nextTagTime(thisTag); + LOG.info("The next tag time after this is {}.", nextTag); if (numRetainedMax != null) { // only handle auto-created tags here diff --git a/paimon-core/src/main/java/org/apache/paimon/tag/TagAutoManager.java b/paimon-core/src/main/java/org/apache/paimon/tag/TagAutoManager.java index 387a3e746adc..817c20af4612 100644 --- a/paimon-core/src/main/java/org/apache/paimon/tag/TagAutoManager.java +++ b/paimon-core/src/main/java/org/apache/paimon/tag/TagAutoManager.java @@ -42,7 +42,7 @@ public void run() { tagAutoCreation.run(); } if (tagTimeExpire != null) { - tagTimeExpire.run(); + tagTimeExpire.expire(); } } @@ -52,6 +52,7 @@ public static TagAutoManager create( TagManager tagManager, TagDeletion tagDeletion, List callbacks) { + TagTimeExtractor extractor = TagTimeExtractor.createForAutoTag(options); return new TagAutoManager( @@ -65,4 +66,8 @@ public static TagAutoManager create( public TagAutoCreation getTagAutoCreation() { return tagAutoCreation; } + + public TagTimeExpire getTagTimeExpire() { + return tagTimeExpire; + } } diff --git a/paimon-core/src/main/java/org/apache/paimon/tag/TagTimeExpire.java b/paimon-core/src/main/java/org/apache/paimon/tag/TagTimeExpire.java index d4797c0cb056..0b57dfa884e9 100644 --- a/paimon-core/src/main/java/org/apache/paimon/tag/TagTimeExpire.java +++ b/paimon-core/src/main/java/org/apache/paimon/tag/TagTimeExpire.java @@ -18,8 +18,10 @@ package org.apache.paimon.tag; +import org.apache.paimon.fs.FileStatus; import org.apache.paimon.operation.TagDeletion; import org.apache.paimon.table.sink.TagCallback; +import org.apache.paimon.utils.DateTimeUtils; import org.apache.paimon.utils.Pair; import org.apache.paimon.utils.SnapshotManager; import org.apache.paimon.utils.TagManager; @@ -27,8 +29,10 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.IOException; import java.time.Duration; import java.time.LocalDateTime; +import java.util.ArrayList; import java.util.List; /** A manager to expire tags by time. */ @@ -41,6 +45,8 @@ public class TagTimeExpire { private final TagDeletion tagDeletion; private final List callbacks; + private LocalDateTime olderThanTime; + private TagTimeExpire( SnapshotManager snapshotManager, TagManager tagManager, @@ -52,24 +58,53 @@ private TagTimeExpire( this.callbacks = callbacks; } - public void run() { + public List expire() { List> tags = tagManager.tagObjects(); + List expired = new ArrayList<>(); for (Pair pair : tags) { Tag tag = pair.getLeft(); String tagName = pair.getRight(); LocalDateTime createTime = tag.getTagCreateTime(); Duration timeRetained = tag.getTagTimeRetained(); if (createTime == null || timeRetained == null) { - continue; + if (olderThanTime != null) { + FileStatus tagFileStatus; + try { + tagFileStatus = + snapshotManager.fileIO().getFileStatus(tagManager.tagPath(tagName)); + } catch (IOException e) { + LOG.warn( + "Tag path {} not exist, skip expire it.", + tagManager.tagPath(tagName)); + continue; + } + createTime = DateTimeUtils.toLocalDateTime(tagFileStatus.getModificationTime()); + } else { + continue; + } } - if (LocalDateTime.now().isAfter(createTime.plus(timeRetained))) { + boolean isReachTimeRetained = + timeRetained != null + && LocalDateTime.now().isAfter(createTime.plus(timeRetained)); + boolean isOlderThan = olderThanTime != null && olderThanTime.isAfter(createTime); + if (isReachTimeRetained || isOlderThan) { LOG.info( - "Delete tag {}, because its existence time has reached its timeRetained of {}.", + "Delete tag {}, because its existence time has reached its timeRetained of {} or" + + " its createTime {} is olderThan olderThanTime {}.", tagName, - timeRetained); + timeRetained, + createTime, + olderThanTime); tagManager.deleteTag(tagName, tagDeletion, snapshotManager, callbacks); + expired.add(tagName); } } + return expired; + } + + public TagTimeExpire withOlderThanTime(LocalDateTime olderThanTime) { + this.olderThanTime = olderThanTime; + return this; } public static TagTimeExpire create( diff --git a/paimon-core/src/main/java/org/apache/paimon/utils/BranchManager.java b/paimon-core/src/main/java/org/apache/paimon/utils/BranchManager.java index c2793de37799..2ea5f542f4e5 100644 --- a/paimon-core/src/main/java/org/apache/paimon/utils/BranchManager.java +++ b/paimon-core/src/main/java/org/apache/paimon/utils/BranchManager.java @@ -23,6 +23,7 @@ import org.apache.paimon.fs.Path; import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.schema.TableSchema; +import org.apache.paimon.tag.Tag; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -94,10 +95,7 @@ public void createBranch(String branchName) { try { TableSchema latestSchema = schemaManager.latest().get(); - fileIO.copyFile( - schemaManager.toSchemaPath(latestSchema.id()), - schemaManager.copyWithBranch(branchName).toSchemaPath(latestSchema.id()), - true); + copySchemasToBranch(branchName, latestSchema.id()); } catch (IOException e) { throw new RuntimeException( String.format( @@ -123,10 +121,7 @@ public void createBranch(String branchName, String tagName) { snapshotManager.snapshotPath(snapshot.id()), snapshotManager.copyWithBranch(branchName).snapshotPath(snapshot.id()), true); - fileIO.copyFile( - schemaManager.toSchemaPath(snapshot.schemaId()), - schemaManager.copyWithBranch(branchName).toSchemaPath(snapshot.schemaId()), - true); + copySchemasToBranch(branchName, snapshot.schemaId()); } catch (IOException e) { throw new RuntimeException( String.format( @@ -184,7 +179,7 @@ public void fastForward(String branchName) { List deleteSchemaPaths = schemaManager.schemaPaths(id -> id >= earliestSchemaId); List deleteTagPaths = tagManager.tagPaths( - path -> Snapshot.fromPath(fileIO, path).id() >= earliestSnapshotId); + path -> Tag.fromPath(fileIO, path).id() >= earliestSnapshotId); List deletePaths = Stream.of(deleteSnapshotPaths, deleteSchemaPaths, deleteTagPaths) @@ -207,6 +202,7 @@ public void fastForward(String branchName) { tagManager.copyWithBranch(branchName).tagDirectory(), tagManager.tagDirectory(), true); + snapshotManager.invalidateCache(); } catch (IOException e) { throw new RuntimeException( String.format( @@ -249,4 +245,15 @@ private void validateBranch(String branchName) { "Branch name cannot be pure numeric string but is '%s'.", branchName); } + + private void copySchemasToBranch(String branchName, long schemaId) throws IOException { + for (int i = 0; i <= schemaId; i++) { + if (schemaManager.schemaExists(i)) { + fileIO.copyFile( + schemaManager.toSchemaPath(i), + schemaManager.copyWithBranch(branchName).toSchemaPath(i), + true); + } + } + } } diff --git a/paimon-core/src/main/java/org/apache/paimon/utils/FileStorePathFactory.java b/paimon-core/src/main/java/org/apache/paimon/utils/FileStorePathFactory.java index df442383a3fa..f255762cfd3c 100644 --- a/paimon-core/src/main/java/org/apache/paimon/utils/FileStorePathFactory.java +++ b/paimon-core/src/main/java/org/apache/paimon/utils/FileStorePathFactory.java @@ -24,6 +24,7 @@ import org.apache.paimon.io.DataFilePathFactory; import org.apache.paimon.types.RowType; +import javax.annotation.Nullable; import javax.annotation.concurrent.ThreadSafe; import java.util.List; @@ -43,6 +44,10 @@ public class FileStorePathFactory { private final String formatIdentifier; private final String dataFilePrefix; private final String changelogFilePrefix; + private final boolean fileSuffixIncludeCompression; + private final String fileCompression; + + @Nullable private final String dataFilePathDirectory; private final AtomicInteger manifestFileCount; private final AtomicInteger manifestListCount; @@ -56,14 +61,22 @@ public FileStorePathFactory( String defaultPartValue, String formatIdentifier, String dataFilePrefix, - String changelogFilePrefix) { + String changelogFilePrefix, + boolean legacyPartitionName, + boolean fileSuffixIncludeCompression, + String fileCompression, + @Nullable String dataFilePathDirectory) { this.root = root; + this.dataFilePathDirectory = dataFilePathDirectory; this.uuid = UUID.randomUUID().toString(); - this.partitionComputer = getPartitionComputer(partitionType, defaultPartValue); + this.partitionComputer = + getPartitionComputer(partitionType, defaultPartValue, legacyPartitionName); this.formatIdentifier = formatIdentifier; this.dataFilePrefix = dataFilePrefix; this.changelogFilePrefix = changelogFilePrefix; + this.fileSuffixIncludeCompression = fileSuffixIncludeCompression; + this.fileCompression = fileCompression; this.manifestFileCount = new AtomicInteger(0); this.manifestListCount = new AtomicInteger(0); @@ -78,9 +91,10 @@ public Path root() { @VisibleForTesting public static InternalRowPartitionComputer getPartitionComputer( - RowType partitionType, String defaultPartValue) { + RowType partitionType, String defaultPartValue, boolean legacyPartitionName) { String[] partitionColumns = partitionType.getFieldNames().toArray(new String[0]); - return new InternalRowPartitionComputer(defaultPartValue, partitionType, partitionColumns); + return new InternalRowPartitionComputer( + defaultPartValue, partitionType, partitionColumns, legacyPartitionName); } public Path newManifestFile() { @@ -110,20 +124,25 @@ public DataFilePathFactory createDataFilePathFactory(BinaryRow partition, int bu bucketPath(partition, bucket), formatIdentifier, dataFilePrefix, - changelogFilePrefix); + changelogFilePrefix, + fileSuffixIncludeCompression, + fileCompression); } public Path bucketPath(BinaryRow partition, int bucket) { - return new Path(root + "/" + relativePartitionAndBucketPath(partition, bucket)); + return new Path(root, relativeBucketPath(partition, bucket)); } - public Path relativePartitionAndBucketPath(BinaryRow partition, int bucket) { + public Path relativeBucketPath(BinaryRow partition, int bucket) { + Path relativeBucketPath = new Path(BUCKET_PATH_PREFIX + bucket); String partitionPath = getPartitionString(partition); - String fullPath = - partitionPath.isEmpty() - ? BUCKET_PATH_PREFIX + bucket - : partitionPath + "/" + BUCKET_PATH_PREFIX + bucket; - return new Path(fullPath); + if (!partitionPath.isEmpty()) { + relativeBucketPath = new Path(partitionPath, relativeBucketPath); + } + if (dataFilePathDirectory != null) { + relativeBucketPath = new Path(dataFilePathDirectory, relativeBucketPath); + } + return relativeBucketPath; } /** IMPORTANT: This method is NOT THREAD SAFE. */ diff --git a/paimon-core/src/main/java/org/apache/paimon/utils/BulkFormatMapping.java b/paimon-core/src/main/java/org/apache/paimon/utils/FormatReaderMapping.java similarity index 58% rename from paimon-core/src/main/java/org/apache/paimon/utils/BulkFormatMapping.java rename to paimon-core/src/main/java/org/apache/paimon/utils/FormatReaderMapping.java index 037622f95f1e..f6c6287f51b4 100644 --- a/paimon-core/src/main/java/org/apache/paimon/utils/BulkFormatMapping.java +++ b/paimon-core/src/main/java/org/apache/paimon/utils/FormatReaderMapping.java @@ -26,6 +26,7 @@ import org.apache.paimon.schema.IndexCastMapping; import org.apache.paimon.schema.SchemaEvolutionUtil; import org.apache.paimon.schema.TableSchema; +import org.apache.paimon.table.SpecialFields; import org.apache.paimon.types.ArrayType; import org.apache.paimon.types.DataField; import org.apache.paimon.types.DataType; @@ -35,37 +36,66 @@ import javax.annotation.Nullable; import java.util.ArrayList; +import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.function.Function; import static org.apache.paimon.predicate.PredicateBuilder.excludePredicateWithFields; +import static org.apache.paimon.table.SpecialFields.KEY_FIELD_ID_START; -/** Class with index mapping and bulk format. */ -public class BulkFormatMapping { +/** Class with index mapping and format reader. */ +public class FormatReaderMapping { + // Index mapping from data schema fields to table schema fields, this is used to realize paimon + // schema evolution. And it combines trimeedKeyMapping, which maps key fields to the value + // fields @Nullable private final int[] indexMapping; + // help indexMapping to cast different data type @Nullable private final CastFieldGetter[] castMapping; + // partition fields mapping, add partition fields to the read fields @Nullable private final Pair partitionPair; - private final FormatReaderFactory bulkFormat; + private final FormatReaderFactory readerFactory; private final TableSchema dataSchema; private final List dataFilters; - public BulkFormatMapping( + public FormatReaderMapping( @Nullable int[] indexMapping, @Nullable CastFieldGetter[] castMapping, + @Nullable int[] trimmedKeyMapping, @Nullable Pair partitionPair, - FormatReaderFactory bulkFormat, + FormatReaderFactory readerFactory, TableSchema dataSchema, List dataFilters) { - this.indexMapping = indexMapping; + this.indexMapping = combine(indexMapping, trimmedKeyMapping); this.castMapping = castMapping; - this.bulkFormat = bulkFormat; + this.readerFactory = readerFactory; this.partitionPair = partitionPair; this.dataSchema = dataSchema; this.dataFilters = dataFilters; } + private int[] combine(@Nullable int[] indexMapping, @Nullable int[] trimmedKeyMapping) { + if (indexMapping == null) { + return trimmedKeyMapping; + } + if (trimmedKeyMapping == null) { + return indexMapping; + } + + int[] combined = new int[indexMapping.length]; + + for (int i = 0; i < indexMapping.length; i++) { + if (indexMapping[i] < 0) { + combined[i] = indexMapping[i]; + } else { + combined[i] = trimmedKeyMapping[indexMapping[i]]; + } + } + return combined; + } + @Nullable public int[] getIndexMapping() { return indexMapping; @@ -82,7 +112,7 @@ public Pair getPartitionPair() { } public FormatReaderFactory getReaderFactory() { - return bulkFormat; + return readerFactory; } public TableSchema getDataSchema() { @@ -93,15 +123,15 @@ public List getDataFilters() { return dataFilters; } - /** Builder for {@link BulkFormatMapping}. */ - public static class BulkFormatMappingBuilder { + /** Builder for {@link FormatReaderMapping}. */ + public static class Builder { private final FileFormatDiscover formatDiscover; private final List readTableFields; private final Function> fieldsExtractor; @Nullable private final List filters; - public BulkFormatMappingBuilder( + public Builder( FileFormatDiscover formatDiscover, List readTableFields, Function> fieldsExtractor, @@ -112,44 +142,103 @@ public BulkFormatMappingBuilder( this.filters = filters; } - public BulkFormatMapping build( + /** + * There are three steps here to build {@link FormatReaderMapping}: + * + *

    1. Calculate the readDataFields, which is what we intend to read from the data schema. + * Meanwhile, generate the indexCastMapping, which is used to map the index of the + * readDataFields to the index of the data schema. + * + *

    2. Calculate the mapping to trim _KEY_ fields. For example: we want _KEY_a, _KEY_b, + * _FIELD_SEQUENCE, _ROW_KIND, a, b, c, d, e, f, g from the data, but actually we don't need + * to read _KEY_a and a, _KEY_b and b the same time, so we need to trim them. So we mapping + * it: read before: _KEY_a, _KEY_b, _FIELD_SEQUENCE, _ROW_KIND, a, b, c, d, e, f, g read + * after: a, b, _FIELD_SEQUENCE, _ROW_KIND, c, d, e, f, g and the mapping is + * [0,1,2,3,0,1,4,5,6,7,8], it converts the [read after] columns to [read before] columns. + * + *

    3. We want read much fewer fields than readDataFields, so we kick out the partition + * fields. We generate the partitionMappingAndFieldsWithoutPartitionPair which helps reduce + * the real read fields and tell us how to map it back. + */ + public FormatReaderMapping build( String formatIdentifier, TableSchema tableSchema, TableSchema dataSchema) { - List readDataFields = readDataFields(dataSchema); - + // extract the whole data fields in logic. + List allDataFields = fieldsExtractor.apply(dataSchema); + List readDataFields = readDataFields(allDataFields); // build index cast mapping IndexCastMapping indexCastMapping = SchemaEvolutionUtil.createIndexCastMapping(readTableFields, readDataFields); + // map from key fields reading to value fields reading + Pair trimmedKeyPair = trimKeyFields(readDataFields, allDataFields); + // build partition mapping and filter partition fields Pair, List> partitionMappingAndFieldsWithoutPartitionPair = - PartitionUtils.constructPartitionMapping(dataSchema, readDataFields); + PartitionUtils.constructPartitionMapping( + dataSchema, trimmedKeyPair.getRight().getFields()); Pair partitionMapping = partitionMappingAndFieldsWithoutPartitionPair.getLeft(); - // build read row type - RowType readDataRowType = + RowType readRowType = new RowType(partitionMappingAndFieldsWithoutPartitionPair.getRight()); // build read filters List readFilters = readFilters(filters, tableSchema, dataSchema); - return new BulkFormatMapping( + return new FormatReaderMapping( indexCastMapping.getIndexMapping(), indexCastMapping.getCastMapping(), + trimmedKeyPair.getLeft(), partitionMapping, formatDiscover .discover(formatIdentifier) - .createReaderFactory(readDataRowType, readFilters), + .createReaderFactory(readRowType, readFilters), dataSchema, readFilters); } - private List readDataFields(TableSchema dataSchema) { - List dataFields = fieldsExtractor.apply(dataSchema); + static Pair trimKeyFields( + List fieldsWithoutPartition, List fields) { + int[] map = new int[fieldsWithoutPartition.size()]; + List trimmedFields = new ArrayList<>(); + Map fieldMap = new HashMap<>(); + Map positionMap = new HashMap<>(); + + for (DataField field : fields) { + fieldMap.put(field.id(), field); + } + + for (int i = 0; i < fieldsWithoutPartition.size(); i++) { + DataField field = fieldsWithoutPartition.get(i); + boolean keyField = SpecialFields.isKeyField(field.name()); + int id = keyField ? field.id() - KEY_FIELD_ID_START : field.id(); + // field in data schema + DataField f = fieldMap.get(id); + + if (f != null) { + if (positionMap.containsKey(id)) { + map[i] = positionMap.get(id); + } else { + map[i] = positionMap.computeIfAbsent(id, k -> trimmedFields.size()); + // If the target field is not key field, we remain what it is, because it + // may be projected. Example: the target field is a row type, but only read + // the few fields in it. If we simply trimmedFields.add(f), we will read + // more fields than we need. + trimmedFields.add(keyField ? f : field); + } + } else { + throw new RuntimeException("Can't find field with id: " + id + " in fields."); + } + } + + return Pair.of(map, new RowType(trimmedFields)); + } + + private List readDataFields(List allDataFields) { List readDataFields = new ArrayList<>(); - for (DataField dataField : dataFields) { + for (DataField dataField : allDataFields) { readTableFields.stream() .filter(f -> f.id() == dataField.id()) .findFirst() diff --git a/paimon-core/src/main/java/org/apache/paimon/utils/KeyComparatorSupplier.java b/paimon-core/src/main/java/org/apache/paimon/utils/KeyComparatorSupplier.java index 6f90cef01a20..25fd07b7cad2 100644 --- a/paimon-core/src/main/java/org/apache/paimon/utils/KeyComparatorSupplier.java +++ b/paimon-core/src/main/java/org/apache/paimon/utils/KeyComparatorSupplier.java @@ -45,6 +45,6 @@ public KeyComparatorSupplier(RowType keyType) { @Override public RecordComparator get() { - return newRecordComparator(inputTypes, sortFields); + return newRecordComparator(inputTypes, sortFields, true); } } diff --git a/paimon-core/src/main/java/org/apache/paimon/utils/ManifestReadThreadPool.java b/paimon-core/src/main/java/org/apache/paimon/utils/ManifestReadThreadPool.java index d967e778fe99..49fcfc8bd909 100644 --- a/paimon-core/src/main/java/org/apache/paimon/utils/ManifestReadThreadPool.java +++ b/paimon-core/src/main/java/org/apache/paimon/utils/ManifestReadThreadPool.java @@ -54,9 +54,9 @@ public static Iterable sequentialBatchedExecute( } /** This method aims to parallel process tasks with randomly but return values sequentially. */ - public static Iterator randomlyExecute( + public static Iterator randomlyExecuteSequentialReturn( Function> processor, List input, @Nullable Integer threadNum) { ThreadPoolExecutor executor = getExecutorService(threadNum); - return ThreadPoolUtils.randomlyExecute(executor, processor, input); + return ThreadPoolUtils.randomlyExecuteSequentialReturn(executor, processor, input); } } diff --git a/paimon-core/src/main/java/org/apache/paimon/utils/NextSnapshotFetcher.java b/paimon-core/src/main/java/org/apache/paimon/utils/NextSnapshotFetcher.java index 021673950d9c..d0a317df5379 100644 --- a/paimon-core/src/main/java/org/apache/paimon/utils/NextSnapshotFetcher.java +++ b/paimon-core/src/main/java/org/apache/paimon/utils/NextSnapshotFetcher.java @@ -45,8 +45,18 @@ public Snapshot getNextSnapshot(long nextSnapshotId) { } Long earliestSnapshotId = snapshotManager.earliestSnapshotId(); + Long latestSnapshotId = snapshotManager.latestSnapshotId(); // No snapshot now if (earliestSnapshotId == null || earliestSnapshotId <= nextSnapshotId) { + if ((earliestSnapshotId == null && nextSnapshotId > 1) + || (latestSnapshotId != null && nextSnapshotId > latestSnapshotId + 1)) { + throw new OutOfRangeException( + String.format( + "The next expected snapshot is too big! Most possible cause might be the table had been recreated." + + "The next snapshot id is %d, while the latest snapshot id is %s", + nextSnapshotId, latestSnapshotId)); + } + LOG.debug( "Next snapshot id {} does not exist, wait for the snapshot generation.", nextSnapshotId); diff --git a/paimon-core/src/main/java/org/apache/paimon/utils/ObjectsCache.java b/paimon-core/src/main/java/org/apache/paimon/utils/ObjectsCache.java index 8c490e008baa..1c9d9664f22f 100644 --- a/paimon-core/src/main/java/org/apache/paimon/utils/ObjectsCache.java +++ b/paimon-core/src/main/java/org/apache/paimon/utils/ObjectsCache.java @@ -66,11 +66,12 @@ public List read( K key, @Nullable Long fileSize, Filter loadFilter, - Filter readFilter) + Filter readFilter, + Filter readVFilter) throws IOException { Segments segments = cache.getIfPresents(key); if (segments != null) { - return readFromSegments(segments, readFilter); + return readFromSegments(segments, readFilter, readVFilter); } else { if (fileSize == null) { fileSize = fileSizeFunction.apply(key); @@ -78,15 +79,16 @@ public List read( if (fileSize <= cache.maxElementSize()) { segments = readSegments(key, fileSize, loadFilter); cache.put(key, segments); - return readFromSegments(segments, readFilter); + return readFromSegments(segments, readFilter, readVFilter); } else { return readFromIterator( - reader.apply(key, fileSize), projectedSerializer, readFilter); + reader.apply(key, fileSize), projectedSerializer, readFilter, readVFilter); } } } - private List readFromSegments(Segments segments, Filter readFilter) + private List readFromSegments( + Segments segments, Filter readFilter, Filter readVFilter) throws IOException { InternalRowSerializer formatSerializer = this.formatSerializer.get(); List entries = new ArrayList<>(); @@ -98,7 +100,10 @@ private List readFromSegments(Segments segments, Filter readFilt try { formatSerializer.mapFromPages(binaryRow, view); if (readFilter.test(binaryRow)) { - entries.add(projectedSerializer.fromRow(binaryRow)); + V v = projectedSerializer.fromRow(binaryRow); + if (readVFilter.test(v)) { + entries.add(v); + } } } catch (EOFException e) { return entries; diff --git a/paimon-core/src/main/java/org/apache/paimon/utils/ObjectsFile.java b/paimon-core/src/main/java/org/apache/paimon/utils/ObjectsFile.java index 190d29d6243f..b0bea8e66a82 100644 --- a/paimon-core/src/main/java/org/apache/paimon/utils/ObjectsFile.java +++ b/paimon-core/src/main/java/org/apache/paimon/utils/ObjectsFile.java @@ -94,7 +94,8 @@ public List read(String fileName) { } public List read(String fileName, @Nullable Long fileSize) { - return read(fileName, fileSize, Filter.alwaysTrue(), Filter.alwaysTrue()); + return read( + fileName, fileSize, Filter.alwaysTrue(), Filter.alwaysTrue(), Filter.alwaysTrue()); } public List readWithIOException(String fileName) throws IOException { @@ -103,7 +104,8 @@ public List readWithIOException(String fileName) throws IOException { public List readWithIOException(String fileName, @Nullable Long fileSize) throws IOException { - return readWithIOException(fileName, fileSize, Filter.alwaysTrue(), Filter.alwaysTrue()); + return readWithIOException( + fileName, fileSize, Filter.alwaysTrue(), Filter.alwaysTrue(), Filter.alwaysTrue()); } public boolean exists(String fileName) { @@ -118,11 +120,12 @@ public List read( String fileName, @Nullable Long fileSize, Filter loadFilter, - Filter readFilter) { + Filter readFilter, + Filter readTFilter) { try { - return readWithIOException(fileName, fileSize, loadFilter, readFilter); + return readWithIOException(fileName, fileSize, loadFilter, readFilter, readTFilter); } catch (IOException e) { - throw new RuntimeException("Failed to read manifest list " + fileName, e); + throw new RuntimeException("Failed to read " + fileName, e); } } @@ -130,14 +133,16 @@ private List readWithIOException( String fileName, @Nullable Long fileSize, Filter loadFilter, - Filter readFilter) + Filter readFilter, + Filter readTFilter) throws IOException { Path path = pathFactory.toPath(fileName); if (cache != null) { - return cache.read(path, fileSize, loadFilter, readFilter); + return cache.read(path, fileSize, loadFilter, readFilter, readTFilter); } - return readFromIterator(createIterator(path, fileSize), serializer, readFilter); + return readFromIterator( + createIterator(path, fileSize), serializer, readFilter, readTFilter); } public String writeWithoutRolling(Collection records) { @@ -184,13 +189,17 @@ public void delete(String fileName) { public static List readFromIterator( CloseableIterator inputIterator, ObjectSerializer serializer, - Filter readFilter) { + Filter readFilter, + Filter readVFilter) { try (CloseableIterator iterator = inputIterator) { List result = new ArrayList<>(); while (iterator.hasNext()) { InternalRow row = iterator.next(); if (readFilter.test(row)) { - result.add(serializer.fromRow(row)); + V v = serializer.fromRow(row); + if (readVFilter.test(v)) { + result.add(v); + } } } return result; diff --git a/paimon-core/src/main/java/org/apache/paimon/utils/SnapshotManager.java b/paimon-core/src/main/java/org/apache/paimon/utils/SnapshotManager.java index 4dda63960fdb..49da83bfe48a 100644 --- a/paimon-core/src/main/java/org/apache/paimon/utils/SnapshotManager.java +++ b/paimon-core/src/main/java/org/apache/paimon/utils/SnapshotManager.java @@ -24,6 +24,8 @@ import org.apache.paimon.fs.FileStatus; import org.apache.paimon.fs.Path; +import org.apache.paimon.shade.caffeine2.com.github.benmanes.caffeine.cache.Cache; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -42,8 +44,11 @@ import java.util.List; import java.util.Optional; import java.util.Set; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.TimeUnit; import java.util.function.BinaryOperator; +import java.util.function.Consumer; import java.util.function.Function; import java.util.function.Predicate; import java.util.stream.Collectors; @@ -52,6 +57,8 @@ import static org.apache.paimon.utils.BranchManager.DEFAULT_MAIN_BRANCH; import static org.apache.paimon.utils.BranchManager.branchPath; import static org.apache.paimon.utils.FileUtils.listVersionedFiles; +import static org.apache.paimon.utils.ThreadPoolUtils.createCachedThreadPool; +import static org.apache.paimon.utils.ThreadPoolUtils.randomlyOnlyExecute; /** Manager for {@link Snapshot}, providing utility methods related to paths and snapshot hints. */ public class SnapshotManager implements Serializable { @@ -70,16 +77,26 @@ public class SnapshotManager implements Serializable { private final FileIO fileIO; private final Path tablePath; private final String branch; + @Nullable private final Cache cache; public SnapshotManager(FileIO fileIO, Path tablePath) { this(fileIO, tablePath, DEFAULT_MAIN_BRANCH); } /** Specify the default branch for data writing. */ - public SnapshotManager(FileIO fileIO, Path tablePath, String branchName) { + public SnapshotManager(FileIO fileIO, Path tablePath, @Nullable String branchName) { + this(fileIO, tablePath, branchName, null); + } + + public SnapshotManager( + FileIO fileIO, + Path tablePath, + @Nullable String branchName, + @Nullable Cache cache) { this.fileIO = fileIO; this.tablePath = tablePath; this.branch = BranchManager.normalizeBranch(branchName); + this.cache = cache; } public SnapshotManager copyWithBranch(String branchName) { @@ -116,20 +133,34 @@ public Path snapshotDirectory() { return new Path(branchPath(tablePath, branch) + "/snapshot"); } + public void invalidateCache() { + if (cache != null) { + cache.invalidateAll(); + } + } + public Snapshot snapshot(long snapshotId) { - Path snapshotPath = snapshotPath(snapshotId); - return Snapshot.fromPath(fileIO, snapshotPath); + Path path = snapshotPath(snapshotId); + Snapshot snapshot = cache == null ? null : cache.getIfPresent(path); + if (snapshot == null) { + snapshot = Snapshot.fromPath(fileIO, path); + if (cache != null) { + cache.put(path, snapshot); + } + } + return snapshot; } public Snapshot tryGetSnapshot(long snapshotId) throws FileNotFoundException { - try { - Path snapshotPath = snapshotPath(snapshotId); - return Snapshot.fromJson(fileIO.readFileUtf8(snapshotPath)); - } catch (FileNotFoundException fileNotFoundException) { - throw fileNotFoundException; - } catch (IOException ioException) { - throw new RuntimeException(ioException); + Path path = snapshotPath(snapshotId); + Snapshot snapshot = cache == null ? null : cache.getIfPresent(path); + if (snapshot == null) { + snapshot = Snapshot.tryFromPath(fileIO, path); + if (cache != null) { + cache.put(path, snapshot); + } } + return snapshot; } public Changelog changelog(long snapshotId) { @@ -336,6 +367,65 @@ private Snapshot changelogOrSnapshot(long snapshotId) { return finalSnapshot; } + public @Nullable Snapshot earlierOrEqualWatermark(long watermark) { + Long earliest = earliestSnapshotId(); + Long latest = latestSnapshotId(); + // If latest == Long.MIN_VALUE don't need next binary search for watermark + // which can reduce IO cost with snapshot + if (earliest == null || latest == null || snapshot(latest).watermark() == Long.MIN_VALUE) { + return null; + } + Long earliestWatermark = null; + // find the first snapshot with watermark + if ((earliestWatermark = snapshot(earliest).watermark()) == null) { + while (earliest < latest) { + earliest++; + earliestWatermark = snapshot(earliest).watermark(); + if (earliestWatermark != null) { + break; + } + } + } + if (earliestWatermark == null) { + return null; + } + + if (earliestWatermark >= watermark) { + return snapshot(earliest); + } + Snapshot finalSnapshot = null; + + while (earliest <= latest) { + long mid = earliest + (latest - earliest) / 2; // Avoid overflow + Snapshot snapshot = snapshot(mid); + Long commitWatermark = snapshot.watermark(); + if (commitWatermark == null) { + // find the first snapshot with watermark + while (mid >= earliest) { + mid--; + commitWatermark = snapshot(mid).watermark(); + if (commitWatermark != null) { + break; + } + } + } + if (commitWatermark == null) { + earliest = mid + 1; + } else { + if (commitWatermark > watermark) { + latest = mid - 1; // Search in the left half + } else if (commitWatermark < watermark) { + earliest = mid + 1; // Search in the right half + finalSnapshot = snapshot; + } else { + finalSnapshot = snapshot; // Found the exact match + break; + } + } + } + return finalSnapshot; + } + public @Nullable Snapshot laterOrEqualWatermark(long watermark) { Long earliest = earliestSnapshotId(); Long latest = latestSnapshotId(); @@ -413,9 +503,15 @@ public List snapshotPaths(Predicate predicate) throws IOException { .collect(Collectors.toList()); } + public Iterator snapshotsWithId(List snapshotIds) { + return snapshotIds.stream() + .map(this::snapshot) + .sorted(Comparator.comparingLong(Snapshot::id)) + .iterator(); + } + public Iterator snapshotsWithinRange( - Optional optionalMaxSnapshotId, Optional optionalMinSnapshotId) - throws IOException { + Optional optionalMaxSnapshotId, Optional optionalMinSnapshotId) { Long lowerBoundSnapshotId = earliestSnapshotId(); Long upperBoundSnapshotId = latestSnapshotId(); Long lowerId; @@ -457,7 +553,7 @@ public Iterator snapshotsWithinRange( public Iterator changelogs() throws IOException { return listVersionedFiles(fileIO, changelogDirectory(), CHANGELOG_PREFIX) - .map(snapshotId -> changelog(snapshotId)) + .map(this::changelog) .sorted(Comparator.comparingLong(Changelog::id)) .iterator(); } @@ -469,16 +565,19 @@ public Iterator changelogs() throws IOException { public List safelyGetAllSnapshots() throws IOException { List paths = listVersionedFiles(fileIO, snapshotDirectory(), SNAPSHOT_PREFIX) - .map(id -> snapshotPath(id)) + .map(this::snapshotPath) .collect(Collectors.toList()); - List snapshots = new ArrayList<>(); - for (Path path : paths) { - try { - snapshots.add(Snapshot.fromJson(fileIO.readFileUtf8(path))); - } catch (FileNotFoundException ignored) { - } - } + List snapshots = Collections.synchronizedList(new ArrayList<>(paths.size())); + collectSnapshots( + path -> { + try { + // do not pollution cache + snapshots.add(Snapshot.tryFromPath(fileIO, path)); + } catch (FileNotFoundException ignored) { + } + }, + paths); return snapshots; } @@ -486,34 +585,53 @@ public List safelyGetAllSnapshots() throws IOException { public List safelyGetAllChangelogs() throws IOException { List paths = listVersionedFiles(fileIO, changelogDirectory(), CHANGELOG_PREFIX) - .map(id -> longLivedChangelogPath(id)) + .map(this::longLivedChangelogPath) .collect(Collectors.toList()); - List changelogs = new ArrayList<>(); - for (Path path : paths) { - try { - String json = fileIO.readFileUtf8(path); - changelogs.add(Changelog.fromJson(json)); - } catch (FileNotFoundException ignored) { - } - } + List changelogs = Collections.synchronizedList(new ArrayList<>(paths.size())); + collectSnapshots( + path -> { + try { + changelogs.add(Changelog.fromJson(fileIO.readFileUtf8(path))); + } catch (IOException e) { + if (!(e instanceof FileNotFoundException)) { + throw new RuntimeException(e); + } + } + }, + paths); return changelogs; } + private void collectSnapshots(Consumer pathConsumer, List paths) + throws IOException { + ExecutorService executor = + createCachedThreadPool( + Runtime.getRuntime().availableProcessors(), "SNAPSHOT_COLLECTOR"); + + try { + randomlyOnlyExecute(executor, pathConsumer, paths); + } catch (RuntimeException e) { + throw new IOException(e); + } finally { + executor.shutdown(); + } + } + /** * Try to get non snapshot files. If any error occurred, just ignore it and return an empty * result. */ - public List tryGetNonSnapshotFiles(Predicate fileStatusFilter) { + public List> tryGetNonSnapshotFiles(Predicate fileStatusFilter) { return listPathWithFilter(snapshotDirectory(), fileStatusFilter, nonSnapshotFileFilter()); } - public List tryGetNonChangelogFiles(Predicate fileStatusFilter) { + public List> tryGetNonChangelogFiles(Predicate fileStatusFilter) { return listPathWithFilter(changelogDirectory(), fileStatusFilter, nonChangelogFileFilter()); } - private List listPathWithFilter( + private List> listPathWithFilter( Path directory, Predicate fileStatusFilter, Predicate fileFilter) { try { FileStatus[] statuses = fileIO.listStatus(directory); @@ -523,8 +641,8 @@ private List listPathWithFilter( return Arrays.stream(statuses) .filter(fileStatusFilter) - .map(FileStatus::getPath) - .filter(fileFilter) + .filter(status -> fileFilter.test(status.getPath())) + .map(status -> Pair.of(status.getPath(), status.getLen())) .collect(Collectors.toList()); } catch (IOException ignored) { return Collections.emptyList(); @@ -723,23 +841,6 @@ private Long findByListFiles(BinaryOperator reducer, Path dir, String pref return listVersionedFiles(fileIO, dir, prefix).reduce(reducer).orElse(null); } - /** - * Find the overlapping snapshots between sortedSnapshots and range of [beginInclusive, - * endExclusive). - */ - public static List findOverlappedSnapshots( - List sortedSnapshots, long beginInclusive, long endExclusive) { - List overlappedSnapshots = new ArrayList<>(); - int right = findPreviousSnapshot(sortedSnapshots, endExclusive); - if (right >= 0) { - int left = Math.max(findPreviousOrEqualSnapshot(sortedSnapshots, beginInclusive), 0); - for (int i = left; i <= right; i++) { - overlappedSnapshots.add(sortedSnapshots.get(i)); - } - } - return overlappedSnapshots; - } - public static int findPreviousSnapshot(List sortedSnapshots, long targetSnapshotId) { for (int i = sortedSnapshots.size() - 1; i >= 0; i--) { if (sortedSnapshots.get(i).id() < targetSnapshotId) { @@ -749,7 +850,7 @@ public static int findPreviousSnapshot(List sortedSnapshots, long targ return -1; } - private static int findPreviousOrEqualSnapshot( + public static int findPreviousOrEqualSnapshot( List sortedSnapshots, long targetSnapshotId) { for (int i = sortedSnapshots.size() - 1; i >= 0; i--) { if (sortedSnapshots.get(i).id() <= targetSnapshotId) { @@ -783,6 +884,23 @@ public void commitEarliestHint(long snapshotId) throws IOException { private void commitHint(long snapshotId, String fileName, Path dir) throws IOException { Path hintFile = new Path(dir, fileName); - fileIO.overwriteFileUtf8(hintFile, String.valueOf(snapshotId)); + int loopTime = 3; + while (loopTime-- > 0) { + try { + fileIO.overwriteFileUtf8(hintFile, String.valueOf(snapshotId)); + return; + } catch (IOException e) { + try { + Thread.sleep(ThreadLocalRandom.current().nextInt(1000) + 500); + } catch (InterruptedException ex) { + Thread.currentThread().interrupt(); + // throw root cause + throw new RuntimeException(e); + } + if (loopTime == 0) { + throw e; + } + } + } } } diff --git a/paimon-core/src/main/java/org/apache/paimon/utils/StatsCollectorFactories.java b/paimon-core/src/main/java/org/apache/paimon/utils/StatsCollectorFactories.java index 6d93224e5600..abb1d686073f 100644 --- a/paimon-core/src/main/java/org/apache/paimon/utils/StatsCollectorFactories.java +++ b/paimon-core/src/main/java/org/apache/paimon/utils/StatsCollectorFactories.java @@ -21,7 +21,10 @@ import org.apache.paimon.CoreOptions; import org.apache.paimon.options.Options; import org.apache.paimon.statistics.SimpleColStatsCollector; +import org.apache.paimon.statistics.TruncateSimpleColStatsCollector; +import org.apache.paimon.table.SpecialFields; +import java.util.Collections; import java.util.List; import static org.apache.paimon.CoreOptions.FIELDS_PREFIX; @@ -33,19 +36,29 @@ public class StatsCollectorFactories { public static SimpleColStatsCollector.Factory[] createStatsFactories( CoreOptions options, List fields) { + return createStatsFactories(options, fields, Collections.emptyList()); + } + + public static SimpleColStatsCollector.Factory[] createStatsFactories( + CoreOptions options, List fields, List keyNames) { Options cfg = options.toConfiguration(); SimpleColStatsCollector.Factory[] modes = new SimpleColStatsCollector.Factory[fields.size()]; for (int i = 0; i < fields.size(); i++) { + String field = fields.get(i); String fieldMode = cfg.get( - key(String.format( - "%s.%s.%s", - FIELDS_PREFIX, fields.get(i), STATS_MODE_SUFFIX)) + key(String.format("%s.%s.%s", FIELDS_PREFIX, field, STATS_MODE_SUFFIX)) .stringType() .noDefaultValue()); if (fieldMode != null) { modes[i] = SimpleColStatsCollector.from(fieldMode); + } else if (SpecialFields.isSystemField(field) + || + // If we config DATA_FILE_THIN_MODE to true, we need to maintain the + // stats for key fields. + keyNames.contains(SpecialFields.KEY_FIELD_PREFIX + field)) { + modes[i] = () -> new TruncateSimpleColStatsCollector(128); } else { modes[i] = SimpleColStatsCollector.from(cfg.get(CoreOptions.METADATA_STATS_MODE)); } diff --git a/paimon-core/src/main/java/org/apache/paimon/utils/TagManager.java b/paimon-core/src/main/java/org/apache/paimon/utils/TagManager.java index c3a674bc5eaf..4019395d8d65 100644 --- a/paimon-core/src/main/java/org/apache/paimon/utils/TagManager.java +++ b/paimon-core/src/main/java/org/apache/paimon/utils/TagManager.java @@ -23,7 +23,7 @@ import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.FileStatus; import org.apache.paimon.fs.Path; -import org.apache.paimon.manifest.ManifestEntry; +import org.apache.paimon.manifest.ExpireFileEntry; import org.apache.paimon.operation.TagDeletion; import org.apache.paimon.table.sink.TagCallback; import org.apache.paimon.tag.Tag; @@ -97,13 +97,26 @@ public List tagPaths(Predicate predicate) throws IOException { /** Create a tag from given snapshot and save it in the storage. */ public void createTag( - Snapshot snapshot, - String tagName, - @Nullable Duration timeRetained, - List callbacks) { + Snapshot snapshot, String tagName, Duration timeRetained, List callbacks) { + checkArgument( + !StringUtils.isNullOrWhitespaceOnly(tagName), "Tag name '%s' is blank.", tagName); + checkArgument(!tagExists(tagName), "Tag name '%s' already exists.", tagName); + createOrReplaceTag(snapshot, tagName, timeRetained, callbacks); + } + + /** Replace a tag from given snapshot and save it in the storage. */ + public void replaceTag(Snapshot snapshot, String tagName, Duration timeRetained) { checkArgument( !StringUtils.isNullOrWhitespaceOnly(tagName), "Tag name '%s' is blank.", tagName); + checkArgument(tagExists(tagName), "Tag name '%s' does not exist.", tagName); + createOrReplaceTag(snapshot, tagName, timeRetained, null); + } + public void createOrReplaceTag( + Snapshot snapshot, + String tagName, + @Nullable Duration timeRetained, + @Nullable List callbacks) { // When timeRetained is not defined, please do not write the tagCreatorTime field, // as this will cause older versions (<= 0.7) of readers to be unable to read this // tag. @@ -117,15 +130,7 @@ public void createTag( Path tagPath = tagPath(tagName); try { - if (tagExists(tagName)) { - Snapshot tagged = taggedSnapshot(tagName); - Preconditions.checkArgument( - tagged.id() == snapshot.id(), "Tag name '%s' already exists.", tagName); - // update tag metadata into for the same snapshot of the same tag name. - fileIO.overwriteFileUtf8(tagPath, content); - } else { - fileIO.writeFile(tagPath, content, false); - } + fileIO.overwriteFileUtf8(tagPath, content); } catch (IOException e) { throw new RuntimeException( String.format( @@ -135,11 +140,13 @@ public void createTag( e); } - try { - callbacks.forEach(callback -> callback.notifyCreation(tagName)); - } finally { - for (TagCallback tagCallback : callbacks) { - IOUtils.closeQuietly(tagCallback); + if (callbacks != null) { + try { + callbacks.forEach(callback -> callback.notifyCreation(tagName)); + } finally { + for (TagCallback tagCallback : callbacks) { + IOUtils.closeQuietly(tagCallback); + } } } } @@ -248,7 +255,7 @@ private void doClean( skippedSnapshots.add(right); // delete data files and empty directories - Predicate dataFileSkipper = null; + Predicate dataFileSkipper = null; boolean success = true; try { dataFileSkipper = tagDeletion.dataFileSkipper(skippedSnapshots); @@ -346,7 +353,7 @@ public SortedMap> tags(Predicate filter) { // If the tag file is not found, it might be deleted by // other processes, so just skip this tag try { - Snapshot snapshot = Snapshot.fromJson(fileIO.readFileUtf8(path)); + Snapshot snapshot = Tag.tryFromPath(fileIO, path).trimToSnapshot(); tags.computeIfAbsent(snapshot, s -> new ArrayList<>()).add(tagName); } catch (FileNotFoundException ignored) { } @@ -364,9 +371,9 @@ public List> tagObjects() { List> tags = new ArrayList<>(); for (Path path : paths) { String tagName = path.getName().substring(TAG_PREFIX.length()); - Tag tag = Tag.safelyFromPath(fileIO, path); - if (tag != null) { - tags.add(Pair.of(tag, tagName)); + try { + tags.add(Pair.of(Tag.tryFromPath(fileIO, path), tagName)); + } catch (FileNotFoundException ignored) { } } return tags; diff --git a/paimon-core/src/main/java/org/apache/paimon/utils/UserDefinedSeqComparator.java b/paimon-core/src/main/java/org/apache/paimon/utils/UserDefinedSeqComparator.java index ec7a00bcb3b2..5e5eb293d0de 100644 --- a/paimon-core/src/main/java/org/apache/paimon/utils/UserDefinedSeqComparator.java +++ b/paimon-core/src/main/java/org/apache/paimon/utils/UserDefinedSeqComparator.java @@ -51,11 +51,13 @@ public int compare(InternalRow o1, InternalRow o2) { @Nullable public static UserDefinedSeqComparator create(RowType rowType, CoreOptions options) { - return create(rowType, options.sequenceField()); + return create( + rowType, options.sequenceField(), options.sequenceFieldSortOrderIsAscending()); } @Nullable - public static UserDefinedSeqComparator create(RowType rowType, List sequenceFields) { + public static UserDefinedSeqComparator create( + RowType rowType, List sequenceFields, boolean isAscendingOrder) { if (sequenceFields.isEmpty()) { return null; } @@ -63,17 +65,19 @@ public static UserDefinedSeqComparator create(RowType rowType, List sequ List fieldNames = rowType.getFieldNames(); int[] fields = sequenceFields.stream().mapToInt(fieldNames::indexOf).toArray(); - return create(rowType, fields); + return create(rowType, fields, isAscendingOrder); } @Nullable - public static UserDefinedSeqComparator create(RowType rowType, int[] sequenceFields) { + public static UserDefinedSeqComparator create( + RowType rowType, int[] sequenceFields, boolean isAscendingOrder) { if (sequenceFields.length == 0) { return null; } RecordComparator comparator = - CodeGenUtils.newRecordComparator(rowType.getFieldTypes(), sequenceFields); + CodeGenUtils.newRecordComparator( + rowType.getFieldTypes(), sequenceFields, isAscendingOrder); return new UserDefinedSeqComparator(sequenceFields, comparator); } } diff --git a/paimon-core/src/main/java/org/apache/paimon/view/View.java b/paimon-core/src/main/java/org/apache/paimon/view/View.java new file mode 100644 index 000000000000..87f56764244b --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/view/View.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.view; + +import org.apache.paimon.types.RowType; + +import java.util.Map; +import java.util.Optional; + +/** Interface for view definition. */ +public interface View { + + /** A name to identify this view. */ + String name(); + + /** Full name (including database) to identify this view. */ + String fullName(); + + /** Returns the row type of this view. */ + RowType rowType(); + + /** Returns the view representation. */ + String query(); + + /** Optional comment of this view. */ + Optional comment(); + + /** Options of this view. */ + Map options(); + + /** Copy this view with adding dynamic options. */ + View copy(Map dynamicOptions); +} diff --git a/paimon-core/src/main/java/org/apache/paimon/view/ViewImpl.java b/paimon-core/src/main/java/org/apache/paimon/view/ViewImpl.java new file mode 100644 index 000000000000..1cd48d4ce445 --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/view/ViewImpl.java @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.view; + +import org.apache.paimon.catalog.Identifier; +import org.apache.paimon.types.RowType; + +import javax.annotation.Nullable; + +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; + +/** Implementation of {@link View}. */ +public class ViewImpl implements View { + + private final Identifier identifier; + private final RowType rowType; + private final String query; + @Nullable private final String comment; + private final Map options; + + public ViewImpl( + Identifier identifier, + RowType rowType, + String query, + @Nullable String comment, + Map options) { + this.identifier = identifier; + this.rowType = rowType; + this.query = query; + this.comment = comment; + this.options = options; + } + + @Override + public String name() { + return identifier.getObjectName(); + } + + @Override + public String fullName() { + return identifier.getFullName(); + } + + @Override + public RowType rowType() { + return rowType; + } + + @Override + public String query() { + return query; + } + + @Override + public Optional comment() { + return Optional.ofNullable(comment); + } + + @Override + public Map options() { + return options; + } + + @Override + public View copy(Map dynamicOptions) { + Map newOptions = new HashMap<>(options); + newOptions.putAll(dynamicOptions); + return new ViewImpl(identifier, rowType, query, comment, newOptions); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + ViewImpl view = (ViewImpl) o; + return Objects.equals(identifier, view.identifier) + && Objects.equals(rowType, view.rowType) + && Objects.equals(query, view.query) + && Objects.equals(comment, view.comment) + && Objects.equals(options, view.options); + } + + @Override + public int hashCode() { + return Objects.hash(identifier, rowType, query, comment, options); + } +} diff --git a/paimon-core/src/main/resources/META-INF/NOTICE b/paimon-core/src/main/resources/META-INF/NOTICE new file mode 100644 index 000000000000..dd2479b1d6e7 --- /dev/null +++ b/paimon-core/src/main/resources/META-INF/NOTICE @@ -0,0 +1,8 @@ +paimon-core +Copyright 2023-2024 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +This project bundles the following dependencies under the Apache Software License 2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt) +- com.squareup.okhttp3:okhttp:4.12.0 \ No newline at end of file diff --git a/paimon-core/src/main/resources/META-INF/services/org.apache.paimon.factories.Factory b/paimon-core/src/main/resources/META-INF/services/org.apache.paimon.factories.Factory index 0f87c96b0d4e..6416edd720f8 100644 --- a/paimon-core/src/main/resources/META-INF/services/org.apache.paimon.factories.Factory +++ b/paimon-core/src/main/resources/META-INF/services/org.apache.paimon.factories.Factory @@ -16,3 +16,26 @@ org.apache.paimon.catalog.FileSystemCatalogFactory org.apache.paimon.jdbc.JdbcCatalogFactory org.apache.paimon.jdbc.JdbcCatalogLockFactory +org.apache.paimon.mergetree.compact.aggregate.factory.FieldBoolAndAggFactory +org.apache.paimon.mergetree.compact.aggregate.factory.FieldBoolOrAggFactory +org.apache.paimon.mergetree.compact.aggregate.factory.FieldCollectAggFactory +org.apache.paimon.mergetree.compact.aggregate.factory.FieldFirstNonNullValueAggFactory +org.apache.paimon.mergetree.compact.aggregate.factory.FieldFirstNonNullValueAggLegacyFactory +org.apache.paimon.mergetree.compact.aggregate.factory.FieldFirstValueAggFactory +org.apache.paimon.mergetree.compact.aggregate.factory.FieldHllSketchAggFactory +org.apache.paimon.mergetree.compact.aggregate.factory.FieldLastNonNullValueAggFactory +org.apache.paimon.mergetree.compact.aggregate.factory.FieldLastValueAggFactory +org.apache.paimon.mergetree.compact.aggregate.factory.FieldListaggAggFactory +org.apache.paimon.mergetree.compact.aggregate.factory.FieldMaxAggFactory +org.apache.paimon.mergetree.compact.aggregate.factory.FieldMergeMapAggFactory +org.apache.paimon.mergetree.compact.aggregate.factory.FieldMinAggFactory +org.apache.paimon.mergetree.compact.aggregate.factory.FieldNestedUpdateAggFactory +org.apache.paimon.mergetree.compact.aggregate.factory.FieldPrimaryKeyAggFactory +org.apache.paimon.mergetree.compact.aggregate.factory.FieldProductAggFactory +org.apache.paimon.mergetree.compact.aggregate.factory.FieldRoaringBitmap32AggFactory +org.apache.paimon.mergetree.compact.aggregate.factory.FieldRoaringBitmap64AggFactory +org.apache.paimon.mergetree.compact.aggregate.factory.FieldSumAggFactory +org.apache.paimon.mergetree.compact.aggregate.factory.FieldThetaSketchAggFactory +org.apache.paimon.rest.RESTCatalogFactory +org.apache.paimon.rest.auth.BearTokenCredentialsProviderFactory +org.apache.paimon.rest.auth.BearTokenFileCredentialsProviderFactory diff --git a/paimon-core/src/test/java/org/apache/paimon/TestFileStore.java b/paimon-core/src/test/java/org/apache/paimon/TestFileStore.java index 303879337780..0d8ea5f4a49a 100644 --- a/paimon-core/src/test/java/org/apache/paimon/TestFileStore.java +++ b/paimon-core/src/test/java/org/apache/paimon/TestFileStore.java @@ -26,10 +26,12 @@ import org.apache.paimon.index.IndexFileMeta; import org.apache.paimon.io.DataFileMeta; import org.apache.paimon.io.IndexIncrement; +import org.apache.paimon.manifest.FileEntry; import org.apache.paimon.manifest.FileKind; import org.apache.paimon.manifest.FileSource; import org.apache.paimon.manifest.ManifestCommittable; import org.apache.paimon.manifest.ManifestEntry; +import org.apache.paimon.manifest.ManifestFile; import org.apache.paimon.manifest.ManifestFileMeta; import org.apache.paimon.manifest.ManifestList; import org.apache.paimon.memory.HeapMemorySegmentPool; @@ -38,7 +40,6 @@ import org.apache.paimon.operation.AbstractFileStoreWrite; import org.apache.paimon.operation.FileStoreCommit; import org.apache.paimon.operation.FileStoreCommitImpl; -import org.apache.paimon.operation.FileStoreScan; import org.apache.paimon.operation.SplitRead; import org.apache.paimon.options.ExpireConfig; import org.apache.paimon.options.MemorySize; @@ -222,7 +223,8 @@ public List commitData( null, Collections.emptyList(), (commit, committable) -> { - logOffsets.forEach(committable::addLogOffset); + logOffsets.forEach( + (bucket, offset) -> committable.addLogOffset(bucket, offset, false)); commit.commit(committable, Collections.emptyMap()); }); } @@ -560,29 +562,41 @@ public Set getFilesInUse(long snapshotId) { return getFilesInUse( snapshotId, snapshotManager(), - newScan(), fileIO, pathFactory(), - manifestListFactory().create()); + manifestListFactory().create(), + manifestFileFactory().create()); } public static Set getFilesInUse( long snapshotId, SnapshotManager snapshotManager, - FileStoreScan scan, FileIO fileIO, FileStorePathFactory pathFactory, - ManifestList manifestList) { + ManifestList manifestList, + ManifestFile manifestFile) { Set result = new HashSet<>(); if (snapshotManager.snapshotExists(snapshotId)) { - result.addAll( + Set files = getSnapshotFileInUse( - snapshotId, snapshotManager, scan, fileIO, pathFactory, manifestList)); + snapshotId, + snapshotManager, + fileIO, + pathFactory, + manifestList, + manifestFile); + result.addAll(files); } else if (snapshotManager.longLivedChangelogExists(snapshotId)) { - result.addAll( + Set files = getChangelogFileInUse( - snapshotId, snapshotManager, scan, fileIO, pathFactory, manifestList)); + snapshotId, + snapshotManager, + fileIO, + pathFactory, + manifestList, + manifestFile); + result.addAll(files); } else { throw new RuntimeException( String.format("The snapshot %s does not exist.", snapshotId)); @@ -594,10 +608,10 @@ public static Set getFilesInUse( private static Set getSnapshotFileInUse( long snapshotId, SnapshotManager snapshotManager, - FileStoreScan scan, FileIO fileIO, FileStorePathFactory pathFactory, - ManifestList manifestList) { + ManifestList manifestList, + ManifestFile manifestFile) { Set result = new HashSet<>(); SchemaManager schemaManager = new SchemaManager(fileIO, snapshotManager.tablePath()); CoreOptions options = new CoreOptions(schemaManager.latest().get().options()); @@ -624,7 +638,11 @@ private static Set getSnapshotFileInUse( manifests.forEach(m -> result.add(pathFactory.toManifestFilePath(m.fileName()))); // data file - List entries = scan.withManifestList(manifests).plan().files(); + List entries = + manifests.stream() + .flatMap(m -> manifestFile.read(m.fileName()).stream()) + .collect(Collectors.toList()); + entries = new ArrayList<>(FileEntry.mergeEntries(entries)); for (ManifestEntry entry : entries) { result.add( new Path( @@ -640,7 +658,9 @@ private static Set getSnapshotFileInUse( // use list. if (changelogDecoupled && !produceChangelog) { entries = - scan.withManifestList(manifestList.readDeltaManifests(snapshot)).plan().files(); + manifestList.readDeltaManifests(snapshot).stream() + .flatMap(m -> manifestFile.read(m.fileName()).stream()) + .collect(Collectors.toList()); for (ManifestEntry entry : entries) { // append delete file are delayed to delete if (entry.kind() == FileKind.DELETE @@ -660,15 +680,13 @@ private static Set getSnapshotFileInUse( private static Set getChangelogFileInUse( long changelogId, SnapshotManager snapshotManager, - FileStoreScan scan, FileIO fileIO, FileStorePathFactory pathFactory, - ManifestList manifestList) { + ManifestList manifestList, + ManifestFile manifestFile) { Set result = new HashSet<>(); SchemaManager schemaManager = new SchemaManager(fileIO, snapshotManager.tablePath()); CoreOptions options = new CoreOptions(schemaManager.latest().get().options()); - boolean produceChangelog = - options.changelogProducer() != CoreOptions.ChangelogProducer.NONE; Path changelogPath = snapshotManager.longLivedChangelogPath(changelogId); Changelog changelog = Changelog.fromPath(fileIO, changelogPath); @@ -676,35 +694,27 @@ private static Set getChangelogFileInUse( // changelog file result.add(changelogPath); - // manifest lists - if (!produceChangelog) { - result.add(pathFactory.toManifestListPath(changelog.baseManifestList())); - result.add(pathFactory.toManifestListPath(changelog.deltaManifestList())); - } - if (changelog.changelogManifestList() != null) { - result.add(pathFactory.toManifestListPath(changelog.changelogManifestList())); - } - - // manifests - List manifests = - new ArrayList<>(manifestList.readChangelogManifests(changelog)); - if (!produceChangelog) { - manifests.addAll(manifestList.readDataManifests(changelog)); - } - - manifests.forEach(m -> result.add(pathFactory.toManifestFilePath(m.fileName()))); - // data file // not all manifests contains useful data file // (1) produceChangelog = 'true': data file in changelog manifests // (2) produceChangelog = 'false': 'APPEND' data file in delta manifests // delta file - if (!produceChangelog) { - for (ManifestEntry entry : - scan.withManifestList(manifestList.readDeltaManifests(changelog)) - .plan() - .files()) { + if (options.changelogProducer() == CoreOptions.ChangelogProducer.NONE) { + // TODO why we need to keep base manifests? + result.add(pathFactory.toManifestListPath(changelog.baseManifestList())); + manifestList + .readDataManifests(changelog) + .forEach(m -> result.add(pathFactory.toManifestFilePath(m.fileName()))); + + result.add(pathFactory.toManifestListPath(changelog.deltaManifestList())); + List manifests = manifestList.readDeltaManifests(changelog); + manifests.forEach(m -> result.add(pathFactory.toManifestFilePath(m.fileName()))); + List files = + manifests.stream() + .flatMap(m -> manifestFile.read(m.fileName()).stream()) + .collect(Collectors.toList()); + for (ManifestEntry entry : files) { if (entry.file().fileSource().orElse(FileSource.APPEND) == FileSource.APPEND) { result.add( new Path( @@ -712,12 +722,15 @@ private static Set getChangelogFileInUse( entry.file().fileName())); } } - } else { - // changelog - for (ManifestEntry entry : - scan.withManifestList(manifestList.readChangelogManifests(changelog)) - .plan() - .files()) { + } else if (changelog.changelogManifestList() != null) { + result.add(pathFactory.toManifestListPath(changelog.changelogManifestList())); + List manifests = manifestList.readChangelogManifests(changelog); + manifests.forEach(m -> result.add(pathFactory.toManifestFilePath(m.fileName()))); + List files = + manifests.stream() + .flatMap(m -> manifestFile.read(m.fileName()).stream()) + .collect(Collectors.toList()); + for (ManifestEntry entry : files) { result.add( new Path( pathFactory.bucketPath(entry.partition(), entry.bucket()), diff --git a/paimon-core/src/test/java/org/apache/paimon/TestKeyValueGenerator.java b/paimon-core/src/test/java/org/apache/paimon/TestKeyValueGenerator.java index 72e826db7a1d..587204cd7616 100644 --- a/paimon-core/src/test/java/org/apache/paimon/TestKeyValueGenerator.java +++ b/paimon-core/src/test/java/org/apache/paimon/TestKeyValueGenerator.java @@ -30,7 +30,7 @@ import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.schema.TableSchema; import org.apache.paimon.table.SchemaEvolutionTableTestBase; -import org.apache.paimon.table.SystemFields; +import org.apache.paimon.table.SpecialFields; import org.apache.paimon.types.ArrayType; import org.apache.paimon.types.BigIntType; import org.apache.paimon.types.DataField; @@ -95,10 +95,12 @@ public class TestKeyValueGenerator { public static final RowType KEY_TYPE = RowType.of( new DataField( - 2 + SystemFields.KEY_FIELD_ID_START, "key_shopId", new IntType(false)), + 2 + SpecialFields.KEY_FIELD_ID_START, + SpecialFields.KEY_FIELD_PREFIX + "shopId", + new IntType(false)), new DataField( - 3 + SystemFields.KEY_FIELD_ID_START, - "key_orderId", + 3 + SpecialFields.KEY_FIELD_ID_START, + SpecialFields.KEY_FIELD_PREFIX + "orderId", new BigIntType(false))); public static final InternalRowSerializer DEFAULT_ROW_SERIALIZER = @@ -281,7 +283,7 @@ public BinaryRow getPartition(KeyValue kv) { public static List getPrimaryKeys(GeneratorMode mode) { List trimmedPk = KEY_TYPE.getFieldNames().stream() - .map(f -> f.replaceFirst("key_", "")) + .map(f -> f.replaceFirst(SpecialFields.KEY_FIELD_PREFIX, "")) .collect(Collectors.toList()); if (mode != NON_PARTITIONED) { trimmedPk = new ArrayList<>(trimmedPk); @@ -393,8 +395,8 @@ public List keyFields(TableSchema schema) { .map( f -> new DataField( - f.id() + SystemFields.KEY_FIELD_ID_START, - "key_" + f.name(), + f.id() + SpecialFields.KEY_FIELD_ID_START, + SpecialFields.KEY_FIELD_PREFIX + f.name(), f.type(), f.description())) .collect(Collectors.toList()); diff --git a/paimon-core/src/test/java/org/apache/paimon/append/AppendOnlyWriterTest.java b/paimon-core/src/test/java/org/apache/paimon/append/AppendOnlyWriterTest.java index a4ba849ad7c0..a9012ed89b34 100644 --- a/paimon-core/src/test/java/org/apache/paimon/append/AppendOnlyWriterTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/append/AppendOnlyWriterTest.java @@ -137,7 +137,7 @@ public void testSingleWrite() throws Exception { new SimpleColStats[] { initStats(1, 1, 0), initStats("AAA", "AAA", 0), initStats(PART, PART, 0) }; - assertThat(meta.valueStats()).isEqualTo(STATS_SERIALIZER.toBinary(expected)); + assertThat(meta.valueStats()).isEqualTo(STATS_SERIALIZER.toBinaryAllMode(expected)); assertThat(meta.minSequenceNumber()).isEqualTo(0); assertThat(meta.maxSequenceNumber()).isEqualTo(0); @@ -200,7 +200,7 @@ public void testMultipleCommits() throws Exception { initStats(String.format("%03d", start), String.format("%03d", end - 1), 0), initStats(PART, PART, 0) }; - assertThat(meta.valueStats()).isEqualTo(STATS_SERIALIZER.toBinary(expected)); + assertThat(meta.valueStats()).isEqualTo(STATS_SERIALIZER.toBinaryAllMode(expected)); assertThat(meta.minSequenceNumber()).isEqualTo(start); assertThat(meta.maxSequenceNumber()).isEqualTo(end - 1); @@ -243,7 +243,7 @@ public void testRollingWrite() throws Exception { initStats(String.format("%03d", min), String.format("%03d", max), 0), initStats(PART, PART, 0) }; - assertThat(meta.valueStats()).isEqualTo(STATS_SERIALIZER.toBinary(expected)); + assertThat(meta.valueStats()).isEqualTo(STATS_SERIALIZER.toBinaryAllMode(expected)); assertThat(meta.minSequenceNumber()).isEqualTo(min); assertThat(meta.maxSequenceNumber()).isEqualTo(max); @@ -522,7 +522,9 @@ private DataFilePathFactory createPathFactory() { new Path(tempDir + "/dt=" + PART + "/bucket-0"), CoreOptions.FILE_FORMAT.defaultValue().toString(), CoreOptions.DATA_FILE_PREFIX.defaultValue(), - CoreOptions.CHANGELOG_FILE_PREFIX.defaultValue()); + CoreOptions.CHANGELOG_FILE_PREFIX.defaultValue(), + CoreOptions.FILE_SUFFIX_INCLUDE_COMPRESSION.defaultValue(), + CoreOptions.FILE_COMPRESSION.defaultValue()); } private AppendOnlyWriter createEmptyWriter(long targetFileSize) { @@ -633,7 +635,8 @@ private Pair> createWriter( options, AppendOnlyWriterTest.SCHEMA.getFieldNames()), MemorySize.MAX_VALUE, new FileIndexOptions(), - true); + true, + false); writer.setMemoryPool( new HeapMemorySegmentPool(options.writeBufferSize(), options.pageSize())); return Pair.of(writer, compactManager.allFiles()); @@ -649,7 +652,7 @@ private DataFileMeta generateCompactAfter(List toCompact) throws I fileName, toCompact.stream().mapToLong(DataFileMeta::fileSize).sum(), toCompact.stream().mapToLong(DataFileMeta::rowCount).sum(), - STATS_SERIALIZER.toBinary( + STATS_SERIALIZER.toBinaryAllMode( new SimpleColStats[] { initStats( toCompact.get(0).valueStats().minValues().getInt(0), @@ -674,6 +677,9 @@ private DataFileMeta generateCompactAfter(List toCompact) throws I minSeq, maxSeq, toCompact.get(0).schemaId(), - FileSource.APPEND); + Collections.emptyList(), + null, + FileSource.APPEND, + null); } } diff --git a/paimon-core/src/test/java/org/apache/paimon/append/UnawareAppendTableCompactionCoordinatorTest.java b/paimon-core/src/test/java/org/apache/paimon/append/UnawareAppendTableCompactionCoordinatorTest.java index d69773900954..9bb461ffe151 100644 --- a/paimon-core/src/test/java/org/apache/paimon/append/UnawareAppendTableCompactionCoordinatorTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/append/UnawareAppendTableCompactionCoordinatorTest.java @@ -29,6 +29,7 @@ import org.apache.paimon.schema.TableSchema; import org.apache.paimon.table.FileStoreTable; import org.apache.paimon.table.FileStoreTableFactory; +import org.apache.paimon.table.source.EndOfScanException; import org.apache.paimon.types.DataTypes; import org.junit.jupiter.api.BeforeEach; @@ -43,7 +44,9 @@ import static org.apache.paimon.mergetree.compact.MergeTreeCompactManagerTest.row; import static org.apache.paimon.stats.StatsTestUtils.newSimpleStats; +import static org.apache.paimon.testutils.assertj.PaimonAssertions.anyCauseMatches; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; /** Tests for {@link UnawareAppendTableCompactionCoordinator}. */ public class UnawareAppendTableCompactionCoordinatorTest { @@ -135,6 +138,14 @@ public void testAgeGrowUp() { .isEqualTo(0); } + @Test + public void testBatchScanEmptyTable() { + compactionCoordinator = + new UnawareAppendTableCompactionCoordinator(appendOnlyFileStoreTable, false); + assertThatThrownBy(() -> compactionCoordinator.scan()) + .satisfies(anyCauseMatches(EndOfScanException.class)); + } + private void assertTasks(List files, int taskNum) { compactionCoordinator.notifyNewFiles(partition, files); List tasks = compactionCoordinator.compactPlan(); @@ -190,6 +201,7 @@ private DataFileMeta newFile(long fileSize) { 0, 0L, null, - FileSource.APPEND); + FileSource.APPEND, + null); } } diff --git a/paimon-core/src/test/java/org/apache/paimon/catalog/CachingCatalogTest.java b/paimon-core/src/test/java/org/apache/paimon/catalog/CachingCatalogTest.java index d1f7eeb8a56d..4792e33c932b 100644 --- a/paimon-core/src/test/java/org/apache/paimon/catalog/CachingCatalogTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/catalog/CachingCatalogTest.java @@ -18,10 +18,13 @@ package org.apache.paimon.catalog; +import org.apache.paimon.Snapshot; import org.apache.paimon.data.GenericRow; import org.apache.paimon.fs.Path; +import org.apache.paimon.manifest.PartitionEntry; import org.apache.paimon.options.MemorySize; import org.apache.paimon.options.Options; +import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaChange; import org.apache.paimon.table.Table; import org.apache.paimon.table.sink.BatchTableCommit; @@ -32,6 +35,8 @@ import org.apache.paimon.table.source.TableScan; import org.apache.paimon.table.system.SystemTableLoader; import org.apache.paimon.types.DataTypes; +import org.apache.paimon.types.RowType; +import org.apache.paimon.types.VarCharType; import org.apache.paimon.utils.FakeTicker; import org.apache.paimon.shade.caffeine2.com.github.benmanes.caffeine.cache.Cache; @@ -43,14 +48,15 @@ import java.io.FileNotFoundException; import java.time.Duration; import java.util.ArrayList; -import java.util.Arrays; +import java.util.Collections; import java.util.List; -import java.util.Optional; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; +import static java.util.Collections.emptyList; +import static java.util.Collections.singletonList; import static org.apache.paimon.data.BinaryString.fromString; import static org.apache.paimon.options.CatalogOptions.CACHE_MANIFEST_MAX_MEMORY; import static org.apache.paimon.options.CatalogOptions.CACHE_MANIFEST_SMALL_FILE_MEMORY; @@ -93,14 +99,49 @@ public void testInvalidateSystemTablesIfBaseTableIsModified() throws Exception { @Test public void testInvalidateSysTablesIfBaseTableIsDropped() throws Exception { - Catalog catalog = new CachingCatalog(this.catalog); + TestableCachingCatalog catalog = + new TestableCachingCatalog(this.catalog, EXPIRATION_TTL, ticker); Identifier tableIdent = new Identifier("db", "tbl"); catalog.createTable(new Identifier("db", "tbl"), DEFAULT_TABLE_SCHEMA, false); Identifier sysIdent = new Identifier("db", "tbl$files"); + // get system table will only cache the origin table catalog.getTable(sysIdent); + assertThat(catalog.tableCache.asMap()).containsKey(tableIdent); + assertThat(catalog.tableCache.asMap()).doesNotContainKey(sysIdent); + // test case sensitivity + Identifier sysIdent1 = new Identifier("db", "tbl$SNAPSHOTS"); + catalog.getTable(sysIdent1); + assertThat(catalog.tableCache.asMap()).doesNotContainKey(sysIdent1); + catalog.dropTable(tableIdent, false); + assertThat(catalog.tableCache.asMap()).doesNotContainKey(tableIdent); assertThatThrownBy(() -> catalog.getTable(sysIdent)) .hasMessage("Table db.tbl does not exist."); + assertThatThrownBy(() -> catalog.getTable(sysIdent1)) + .hasMessage("Table db.tbl does not exist."); + } + + @Test + public void testInvalidateBranchIfBaseTableIsDropped() throws Exception { + TestableCachingCatalog catalog = + new TestableCachingCatalog(this.catalog, EXPIRATION_TTL, ticker); + Identifier tableIdent = new Identifier("db", "tbl"); + catalog.createTable(new Identifier("db", "tbl"), DEFAULT_TABLE_SCHEMA, false); + catalog.getTable(tableIdent).createBranch("b1"); + + Identifier branchIdent = new Identifier("db", "tbl$branch_b1"); + Identifier branchSysIdent = new Identifier("db", "tbl$branch_b1$FILES"); + // get system table will only cache the origin table + catalog.getTable(branchSysIdent); + assertThat(catalog.tableCache.asMap()).containsKey(branchIdent); + assertThat(catalog.tableCache.asMap()).doesNotContainKey(branchSysIdent); + + catalog.dropTable(tableIdent, false); + assertThat(catalog.tableCache.asMap()).doesNotContainKey(branchIdent); + assertThatThrownBy(() -> catalog.getTable(branchIdent)) + .hasMessage("Table db.tbl$branch_b1 does not exist."); + assertThatThrownBy(() -> catalog.getTable(branchSysIdent)) + .hasMessage("Table db.tbl$branch_b1 does not exist."); } @Test @@ -113,15 +154,15 @@ public void testTableExpiresAfterInterval() throws Exception { Table table = catalog.getTable(tableIdent); // Ensure table is cached with full ttl remaining upon creation - assertThat(catalog.cache().asMap()).containsKey(tableIdent); + assertThat(catalog.tableCache().asMap()).containsKey(tableIdent); assertThat(catalog.remainingAgeFor(tableIdent)).isPresent().get().isEqualTo(EXPIRATION_TTL); ticker.advance(HALF_OF_EXPIRATION); - assertThat(catalog.cache().asMap()).containsKey(tableIdent); + assertThat(catalog.tableCache().asMap()).containsKey(tableIdent); assertThat(catalog.ageOf(tableIdent)).isPresent().get().isEqualTo(HALF_OF_EXPIRATION); ticker.advance(HALF_OF_EXPIRATION.plus(Duration.ofSeconds(10))); - assertThat(catalog.cache().asMap()).doesNotContainKey(tableIdent); + assertThat(catalog.tableCache().asMap()).doesNotContainKey(tableIdent); assertThat(catalog.getTable(tableIdent)) .as("CachingCatalog should return a new instance after expiration") .isNotSameAs(table); @@ -135,11 +176,11 @@ public void testCatalogExpirationTtlRefreshesAfterAccessViaCatalog() throws Exce Identifier tableIdent = new Identifier("db", "tbl"); catalog.createTable(tableIdent, DEFAULT_TABLE_SCHEMA, false); catalog.getTable(tableIdent); - assertThat(catalog.cache().asMap()).containsKey(tableIdent); + assertThat(catalog.tableCache().asMap()).containsKey(tableIdent); assertThat(catalog.ageOf(tableIdent)).isPresent().get().isEqualTo(Duration.ZERO); ticker.advance(HALF_OF_EXPIRATION); - assertThat(catalog.cache().asMap()).containsKey(tableIdent); + assertThat(catalog.tableCache().asMap()).containsKey(tableIdent); assertThat(catalog.ageOf(tableIdent)).isPresent().get().isEqualTo(HALF_OF_EXPIRATION); assertThat(catalog.remainingAgeFor(tableIdent)) .isPresent() @@ -148,7 +189,7 @@ public void testCatalogExpirationTtlRefreshesAfterAccessViaCatalog() throws Exce Duration oneMinute = Duration.ofMinutes(1L); ticker.advance(oneMinute); - assertThat(catalog.cache().asMap()).containsKey(tableIdent); + assertThat(catalog.tableCache().asMap()).containsKey(tableIdent); assertThat(catalog.ageOf(tableIdent)) .isPresent() .get() @@ -168,56 +209,28 @@ public void testCatalogExpirationTtlRefreshesAfterAccessViaCatalog() throws Exce } @Test - public void testCacheExpirationEagerlyRemovesSysTables() throws Exception { + public void testPartitionCache() throws Exception { TestableCachingCatalog catalog = new TestableCachingCatalog(this.catalog, EXPIRATION_TTL, ticker); Identifier tableIdent = new Identifier("db", "tbl"); - catalog.createTable(tableIdent, DEFAULT_TABLE_SCHEMA, false); - Table table = catalog.getTable(tableIdent); - assertThat(catalog.cache().asMap()).containsKey(tableIdent); - assertThat(catalog.ageOf(tableIdent)).get().isEqualTo(Duration.ZERO); - - ticker.advance(HALF_OF_EXPIRATION); - assertThat(catalog.cache().asMap()).containsKey(tableIdent); - assertThat(catalog.ageOf(tableIdent)).get().isEqualTo(HALF_OF_EXPIRATION); - - for (Identifier sysTable : sysTables(tableIdent)) { - catalog.getTable(sysTable); - } - assertThat(catalog.cache().asMap()).containsKeys(sysTables(tableIdent)); - assertThat(Arrays.stream(sysTables(tableIdent)).map(catalog::ageOf)) - .isNotEmpty() - .allMatch(age -> age.isPresent() && age.get().equals(Duration.ZERO)); - - assertThat(catalog.remainingAgeFor(tableIdent)) - .as("Loading a non-cached sys table should refresh the main table's age") - .isEqualTo(Optional.of(EXPIRATION_TTL)); - - // Move time forward and access already cached sys tables. - ticker.advance(HALF_OF_EXPIRATION); - for (Identifier sysTable : sysTables(tableIdent)) { - catalog.getTable(sysTable); - } - assertThat(Arrays.stream(sysTables(tableIdent)).map(catalog::ageOf)) - .isNotEmpty() - .allMatch(age -> age.isPresent() && age.get().equals(Duration.ZERO)); - - assertThat(catalog.remainingAgeFor(tableIdent)) - .as("Accessing a cached sys table should not affect the main table's age") - .isEqualTo(Optional.of(HALF_OF_EXPIRATION)); - - // Move time forward so the data table drops. - ticker.advance(HALF_OF_EXPIRATION); - assertThat(catalog.cache().asMap()).doesNotContainKey(tableIdent); - - Arrays.stream(sysTables(tableIdent)) - .forEach( - sysTable -> - assertThat(catalog.cache().asMap()) - .as( - "When a data table expires, its sys tables should expire regardless of age") - .doesNotContainKeys(sysTable)); + Schema schema = + new Schema( + RowType.of(VarCharType.STRING_TYPE, VarCharType.STRING_TYPE).getFields(), + singletonList("f0"), + emptyList(), + Collections.emptyMap(), + ""); + catalog.createTable(tableIdent, schema, false); + List partitionEntryList = catalog.listPartitions(tableIdent); + assertThat(catalog.partitionCache().asMap()).containsKey(tableIdent); + catalog.invalidateTable(tableIdent); + catalog.refreshPartitions(tableIdent); + assertThat(catalog.partitionCache().asMap()).containsKey(tableIdent); + List partitionEntryListFromCache = + catalog.partitionCache().getIfPresent(tableIdent); + assertThat(partitionEntryListFromCache).isNotNull(); + assertThat(partitionEntryListFromCache).containsAll(partitionEntryList); } @Test @@ -233,7 +246,7 @@ public void testDeadlock() throws Exception { createdTables.add(tableIdent); } - Cache cache = catalog.cache(); + Cache cache = catalog.tableCache(); AtomicInteger cacheGetCount = new AtomicInteger(0); AtomicInteger cacheCleanupCount = new AtomicInteger(0); ExecutorService executor = Executors.newFixedThreadPool(numThreads); @@ -288,10 +301,10 @@ public void testInvalidateTableForChainedCachingCatalogs() throws Exception { Identifier tableIdent = new Identifier("db", "tbl"); catalog.createTable(tableIdent, DEFAULT_TABLE_SCHEMA, false); catalog.getTable(tableIdent); - assertThat(catalog.cache().asMap()).containsKey(tableIdent); + assertThat(catalog.tableCache().asMap()).containsKey(tableIdent); catalog.dropTable(tableIdent, false); - assertThat(catalog.cache().asMap()).doesNotContainKey(tableIdent); - assertThat(wrappedCatalog.cache().asMap()).doesNotContainKey(tableIdent); + assertThat(catalog.tableCache().asMap()).doesNotContainKey(tableIdent); + assertThat(wrappedCatalog.tableCache().asMap()).doesNotContainKey(tableIdent); } public static Identifier[] sysTables(Identifier tableIdent) { @@ -300,6 +313,31 @@ public static Identifier[] sysTables(Identifier tableIdent) { .toArray(Identifier[]::new); } + @Test + public void testSnapshotCache() throws Exception { + TestableCachingCatalog wrappedCatalog = + new TestableCachingCatalog(this.catalog, EXPIRATION_TTL, ticker); + Identifier tableIdent = new Identifier("db", "tbl"); + wrappedCatalog.createTable(tableIdent, DEFAULT_TABLE_SCHEMA, false); + Table table = wrappedCatalog.getTable(tableIdent); + + // write + BatchWriteBuilder writeBuilder = table.newBatchWriteBuilder(); + try (BatchTableWrite write = writeBuilder.newWrite(); + BatchTableCommit commit = writeBuilder.newCommit()) { + write.write(GenericRow.of(1, fromString("1"), fromString("1"))); + write.write(GenericRow.of(2, fromString("2"), fromString("2"))); + commit.commit(write.prepareCommit()); + } + + Snapshot snapshot = table.snapshot(1); + assertThat(snapshot).isSameAs(table.snapshot(1)); + + // copy + Snapshot copied = table.copy(Collections.singletonMap("a", "b")).snapshot(1); + assertThat(copied).isSameAs(snapshot); + } + @Test public void testManifestCache() throws Exception { innerTestManifestCache(Long.MAX_VALUE); @@ -313,7 +351,9 @@ private void innerTestManifestCache(long manifestCacheThreshold) throws Exceptio this.catalog, Duration.ofSeconds(10), MemorySize.ofMebiBytes(1), - manifestCacheThreshold); + manifestCacheThreshold, + 0L, + 10); Identifier tableIdent = new Identifier("db", "tbl"); catalog.dropTable(tableIdent, true); catalog.createTable(tableIdent, DEFAULT_TABLE_SCHEMA, false); @@ -330,7 +370,8 @@ private void innerTestManifestCache(long manifestCacheThreshold) throws Exceptio // repeat read for (int i = 0; i < 5; i++) { - table = catalog.getTable(tableIdent); + // test copy too + table = catalog.getTable(tableIdent).copy(Collections.singletonMap("a", "b")); ReadBuilder readBuilder = table.newReadBuilder(); TableScan scan = readBuilder.newScan(); TableRead read = readBuilder.newRead(); diff --git a/paimon-core/src/test/java/org/apache/paimon/catalog/CatalogTestBase.java b/paimon-core/src/test/java/org/apache/paimon/catalog/CatalogTestBase.java index c4470b2283f3..98a9b92c5c38 100644 --- a/paimon-core/src/test/java/org/apache/paimon/catalog/CatalogTestBase.java +++ b/paimon-core/src/test/java/org/apache/paimon/catalog/CatalogTestBase.java @@ -25,22 +25,29 @@ import org.apache.paimon.options.Options; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaChange; +import org.apache.paimon.table.FormatTable; import org.apache.paimon.table.Table; import org.apache.paimon.types.DataField; import org.apache.paimon.types.DataTypes; import org.apache.paimon.types.RowType; +import org.apache.paimon.view.View; +import org.apache.paimon.view.ViewImpl; import org.apache.paimon.shade.guava30.com.google.common.collect.Lists; import org.apache.paimon.shade.guava30.com.google.common.collect.Maps; import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; import static org.apache.paimon.testutils.assertj.PaimonAssertions.anyCauseMatches; import static org.assertj.core.api.Assertions.assertThat; @@ -107,23 +114,22 @@ public void testListDatabases() throws Exception { } @Test - public void testDatabaseExistsWhenExists() throws Exception { - // Database exists returns true when the database exists + public void testDuplicatedDatabaseAfterCreatingTable() throws Exception { catalog.createDatabase("test_db", false); - boolean exists = catalog.databaseExists("test_db"); - assertThat(exists).isTrue(); + Identifier identifier = Identifier.create("test_db", "new_table"); + Schema schema = Schema.newBuilder().column("pk1", DataTypes.INT()).build(); + catalog.createTable(identifier, schema, false); - // Database exists returns false when the database does not exist - exists = catalog.databaseExists("non_existing_db"); - assertThat(exists).isFalse(); + List databases = catalog.listDatabases(); + List distinctDatabases = databases.stream().distinct().collect(Collectors.toList()); + Assertions.assertEquals(distinctDatabases.size(), databases.size()); } @Test public void testCreateDatabase() throws Exception { // Create database creates a new database when it does not exist catalog.createDatabase("new_db", false); - boolean exists = catalog.databaseExists("new_db"); - assertThat(exists).isTrue(); + catalog.getDatabase("new_db"); catalog.createDatabase("existing_db", false); @@ -144,8 +150,8 @@ public void testDropDatabase() throws Exception { // Drop database deletes the database when it exists and there are no tables catalog.createDatabase("db_to_drop", false); catalog.dropDatabase("db_to_drop", false, false); - boolean exists = catalog.databaseExists("db_to_drop"); - assertThat(exists).isFalse(); + assertThatThrownBy(() -> catalog.getDatabase("db_to_drop")) + .isInstanceOf(Catalog.DatabaseNotExistException.class); // Drop database does not throw exception when database does not exist and ignoreIfNotExists // is true @@ -158,8 +164,8 @@ public void testDropDatabase() throws Exception { catalog.createTable(Identifier.create("db_to_drop", "table2"), DEFAULT_TABLE_SCHEMA, false); catalog.dropDatabase("db_to_drop", false, true); - exists = catalog.databaseExists("db_to_drop"); - assertThat(exists).isFalse(); + assertThatThrownBy(() -> catalog.getDatabase("db_to_drop")) + .isInstanceOf(Catalog.DatabaseNotExistException.class); // Drop database throws DatabaseNotEmptyException when cascade is false and there are tables // in the database @@ -188,21 +194,6 @@ public void testListTables() throws Exception { assertThat(tables).containsExactlyInAnyOrder("table1", "table2", "table3"); } - @Test - public void testTableExists() throws Exception { - // Table exists returns true when the table exists in the database - catalog.createDatabase("test_db", false); - Identifier identifier = Identifier.create("test_db", "test_table"); - catalog.createTable(identifier, DEFAULT_TABLE_SCHEMA, false); - - boolean exists = catalog.tableExists(identifier); - assertThat(exists).isTrue(); - - // Table exists returns false when the table does not exist in the database - exists = catalog.tableExists(Identifier.create("non_existing_db", "non_existing_table")); - assertThat(exists).isFalse(); - } - @Test public void testCreateTable() throws Exception { catalog.createDatabase("test_db", false); @@ -234,8 +225,7 @@ public void testCreateTable() throws Exception { schema.options().remove(CoreOptions.AUTO_CREATE.key()); catalog.createTable(identifier, schema, false); - boolean exists = catalog.tableExists(identifier); - assertThat(exists).isTrue(); + catalog.getTable(identifier); // Create table throws Exception when table is system table assertThatExceptionOfType(IllegalArgumentException.class) @@ -302,6 +292,19 @@ public void testCreateTable() throws Exception { ""), true)) .doesNotThrowAnyException(); + // Create table throws IleaArgumentException when some table options are not set correctly + schema.options() + .put( + CoreOptions.MERGE_ENGINE.key(), + CoreOptions.MergeEngine.DEDUPLICATE.toString()); + schema.options().put(CoreOptions.IGNORE_DELETE.key(), "max"); + assertThatCode( + () -> + catalog.createTable( + Identifier.create("test_db", "wrong_table"), schema, false)) + .hasRootCauseInstanceOf(IllegalArgumentException.class) + .hasRootCauseMessage( + "Unrecognized option for boolean: max. Expected either true or false(case insensitive)"); } @Test @@ -364,8 +367,8 @@ public void testDropTable() throws Exception { Identifier identifier = Identifier.create("test_db", "table_to_drop"); catalog.createTable(identifier, DEFAULT_TABLE_SCHEMA, false); catalog.dropTable(identifier, false); - boolean exists = catalog.tableExists(identifier); - assertThat(exists).isFalse(); + assertThatThrownBy(() -> catalog.getTable(identifier)) + .isInstanceOf(Catalog.TableNotExistException.class); // Drop table throws Exception when table is system table assertThatExceptionOfType(IllegalArgumentException.class) @@ -397,8 +400,9 @@ public void testRenameTable() throws Exception { catalog.createTable(fromTable, DEFAULT_TABLE_SCHEMA, false); Identifier toTable = Identifier.create("test_db", "new_table"); catalog.renameTable(fromTable, toTable, false); - assertThat(catalog.tableExists(fromTable)).isFalse(); - assertThat(catalog.tableExists(toTable)).isTrue(); + assertThatThrownBy(() -> catalog.getTable(fromTable)) + .isInstanceOf(Catalog.TableNotExistException.class); + catalog.getTable(toTable); // Rename table throws Exception when original or target table is system table assertThatExceptionOfType(IllegalArgumentException.class) @@ -509,7 +513,9 @@ public void testAlterTableRenameColumn() throws Exception { catalog.createTable( identifier, new Schema( - Lists.newArrayList(new DataField(0, "col1", DataTypes.STRING())), + Lists.newArrayList( + new DataField(0, "col1", DataTypes.STRING()), + new DataField(1, "col2", DataTypes.STRING())), Collections.emptyList(), Collections.emptyList(), Maps.newHashMap(), @@ -521,7 +527,7 @@ public void testAlterTableRenameColumn() throws Exception { false); Table table = catalog.getTable(identifier); - assertThat(table.rowType().getFields()).hasSize(1); + assertThat(table.rowType().getFields()).hasSize(2); assertThat(table.rowType().getFieldIndex("col1")).isLessThan(0); assertThat(table.rowType().getFieldIndex("new_col1")).isEqualTo(0); @@ -532,12 +538,12 @@ public void testAlterTableRenameColumn() throws Exception { catalog.alterTable( identifier, Lists.newArrayList( - SchemaChange.renameColumn("col1", "new_col1")), + SchemaChange.renameColumn("col2", "new_col1")), false)) .satisfies( anyCauseMatches( Catalog.ColumnAlreadyExistException.class, - "Column col1 already exists in the test_db.test_table table.")); + "Column new_col1 already exists in the test_db.test_table table.")); // Alter table renames a column throws ColumnNotExistException when column does not exist assertThatThrownBy( @@ -551,7 +557,7 @@ public void testAlterTableRenameColumn() throws Exception { .satisfies( anyCauseMatches( Catalog.ColumnNotExistException.class, - "Column [non_existing_col] does not exist in the test_db.test_table table.")); + "Column non_existing_col does not exist in the test_db.test_table table.")); } @Test @@ -657,7 +663,7 @@ public void testAlterTableUpdateColumnType() throws Exception { .satisfies( anyCauseMatches( Catalog.ColumnNotExistException.class, - "Column [non_existing_col] does not exist in the test_db.test_table table.")); + "Column non_existing_col does not exist in the test_db.test_table table.")); // Alter table update a column type throws Exception when column is partition columns assertThatThrownBy( () -> @@ -669,8 +675,8 @@ public void testAlterTableUpdateColumnType() throws Exception { false)) .satisfies( anyCauseMatches( - IllegalArgumentException.class, - "Cannot update partition column [dt] type in the table")); + UnsupportedOperationException.class, + "Cannot update partition column: [dt]")); } @Test @@ -728,7 +734,7 @@ public void testAlterTableUpdateColumnComment() throws Exception { .satisfies( anyCauseMatches( Catalog.ColumnNotExistException.class, - "Column [non_existing_col] does not exist in the test_db.test_table table.")); + "Column non_existing_col does not exist in the test_db.test_table table.")); } @Test @@ -784,7 +790,7 @@ public void testAlterTableUpdateColumnNullability() throws Exception { .satisfies( anyCauseMatches( Catalog.ColumnNotExistException.class, - "Column [non_existing_col] does not exist in the test_db.test_table table.")); + "Column non_existing_col does not exist in the test_db.test_table table.")); // Alter table update a column nullability throws Exception when column is pk columns assertThatThrownBy( @@ -831,4 +837,127 @@ public void testAlterTableUpdateComment() throws Exception { table = catalog.getTable(identifier); assertThat(table.comment().isPresent()).isFalse(); } + + protected boolean supportsView() { + return false; + } + + @Test + public void testView() throws Exception { + if (!supportsView()) { + return; + } + + Identifier identifier = new Identifier("view_db", "my_view"); + RowType rowType = + RowType.builder() + .field("str", DataTypes.STRING()) + .field("int", DataTypes.INT()) + .build(); + String query = "SELECT * FROM OTHER_TABLE"; + String comment = "it is my view"; + Map options = new HashMap<>(); + options.put("key1", "v1"); + options.put("key2", "v2"); + View view = new ViewImpl(identifier, rowType, query, comment, options); + + assertThatThrownBy(() -> catalog.createView(identifier, view, false)) + .isInstanceOf(Catalog.DatabaseNotExistException.class); + + assertThatThrownBy(() -> catalog.listViews(identifier.getDatabaseName())) + .isInstanceOf(Catalog.DatabaseNotExistException.class); + + catalog.createDatabase(identifier.getDatabaseName(), false); + + assertThatThrownBy(() -> catalog.getView(identifier)) + .isInstanceOf(Catalog.ViewNotExistException.class); + + catalog.createView(identifier, view, false); + + View catalogView = catalog.getView(identifier); + assertThat(catalogView.fullName()).isEqualTo(view.fullName()); + assertThat(catalogView.rowType()).isEqualTo(view.rowType()); + assertThat(catalogView.query()).isEqualTo(view.query()); + assertThat(catalogView.comment()).isEqualTo(view.comment()); + assertThat(catalogView.options()).containsAllEntriesOf(view.options()); + + List views = catalog.listViews(identifier.getDatabaseName()); + assertThat(views).containsOnly(identifier.getObjectName()); + + catalog.createView(identifier, view, true); + assertThatThrownBy(() -> catalog.createView(identifier, view, false)) + .isInstanceOf(Catalog.ViewAlreadyExistException.class); + + Identifier newIdentifier = new Identifier("view_db", "new_view"); + catalog.renameView(new Identifier("view_db", "unknown"), newIdentifier, true); + assertThatThrownBy( + () -> + catalog.renameView( + new Identifier("view_db", "unknown"), newIdentifier, false)) + .isInstanceOf(Catalog.ViewNotExistException.class); + catalog.renameView(identifier, newIdentifier, false); + + catalog.dropView(newIdentifier, false); + catalog.dropView(newIdentifier, true); + assertThatThrownBy(() -> catalog.dropView(newIdentifier, false)) + .isInstanceOf(Catalog.ViewNotExistException.class); + } + + protected boolean supportsFormatTable() { + return false; + } + + @Test + public void testFormatTable() throws Exception { + if (!supportsFormatTable()) { + return; + } + + Identifier identifier = new Identifier("format_db", "my_format"); + catalog.createDatabase(identifier.getDatabaseName(), false); + + // create table + Schema schema = + Schema.newBuilder() + .column("str", DataTypes.STRING()) + .column("int", DataTypes.INT()) + .option("type", "format-table") + .option("file.format", "csv") + .build(); + catalog.createTable(identifier, schema, false); + assertThat(catalog.listTables(identifier.getDatabaseName())) + .contains(identifier.getTableName()); + assertThat(catalog.getTable(identifier)).isInstanceOf(FormatTable.class); + + // alter table + SchemaChange schemaChange = SchemaChange.addColumn("new_col", DataTypes.STRING()); + assertThatThrownBy(() -> catalog.alterTable(identifier, schemaChange, false)) + .isInstanceOf(UnsupportedOperationException.class) + .hasMessage("Only data table support alter table."); + + // drop table + catalog.dropTable(identifier, false); + assertThatThrownBy(() -> catalog.getTable(identifier)) + .isInstanceOf(Catalog.TableNotExistException.class); + + // rename table + catalog.createTable(identifier, schema, false); + Identifier newIdentifier = new Identifier("format_db", "new_format"); + catalog.renameTable(identifier, newIdentifier, false); + assertThatThrownBy(() -> catalog.getTable(identifier)) + .isInstanceOf(Catalog.TableNotExistException.class); + assertThat(catalog.getTable(newIdentifier)).isInstanceOf(FormatTable.class); + } + + @Test + public void testTableUUID() throws Exception { + catalog.createDatabase("test_db", false); + Identifier identifier = Identifier.create("test_db", "test_table"); + catalog.createTable(identifier, DEFAULT_TABLE_SCHEMA, false); + Table table = catalog.getTable(identifier); + String uuid = table.uuid(); + assertThat(uuid).startsWith(identifier.getFullName() + "."); + assertThat(Long.parseLong(uuid.substring((identifier.getFullName() + ".").length()))) + .isGreaterThan(0); + } } diff --git a/paimon-core/src/test/java/org/apache/paimon/catalog/FileSystemCatalogTest.java b/paimon-core/src/test/java/org/apache/paimon/catalog/FileSystemCatalogTest.java index 67fbd2718a36..303a9d8733d4 100644 --- a/paimon-core/src/test/java/org/apache/paimon/catalog/FileSystemCatalogTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/catalog/FileSystemCatalogTest.java @@ -19,14 +19,52 @@ package org.apache.paimon.catalog; import org.apache.paimon.fs.Path; +import org.apache.paimon.options.CatalogOptions; +import org.apache.paimon.options.Options; +import org.apache.paimon.schema.Schema; +import org.apache.paimon.types.DataTypes; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; -class FileSystemCatalogTest extends CatalogTestBase { +import static org.assertj.core.api.Assertions.assertThatExceptionOfType; + +/** Tests for {@link FileSystemCatalog}. */ +public class FileSystemCatalogTest extends CatalogTestBase { @BeforeEach public void setUp() throws Exception { super.setUp(); - catalog = new FileSystemCatalog(fileIO, new Path(warehouse)); + Options catalogOptions = new Options(); + catalogOptions.set(CatalogOptions.ALLOW_UPPER_CASE, false); + catalog = new FileSystemCatalog(fileIO, new Path(warehouse), catalogOptions); + } + + @Test + public void testCreateTableAllowUpperCase() throws Exception { + catalog.createDatabase("test_db", false); + Identifier identifier = Identifier.create("test_db", "new_table"); + Schema schema = + Schema.newBuilder() + .column("Pk1", DataTypes.INT()) + .column("pk2", DataTypes.STRING()) + .column("pk3", DataTypes.STRING()) + .column( + "Col1", + DataTypes.ROW( + DataTypes.STRING(), + DataTypes.BIGINT(), + DataTypes.TIMESTAMP(), + DataTypes.ARRAY(DataTypes.STRING()))) + .column("col2", DataTypes.MAP(DataTypes.STRING(), DataTypes.BIGINT())) + .column("col3", DataTypes.ARRAY(DataTypes.ROW(DataTypes.STRING()))) + .partitionKeys("Pk1", "pk2") + .primaryKey("Pk1", "pk2", "pk3") + .build(); + + // Create table throws Exception if using uppercase when 'allow-upper-case' is false + assertThatExceptionOfType(IllegalArgumentException.class) + .isThrownBy(() -> catalog.createTable(identifier, schema, false)) + .withMessage("Field name [Pk1, Col1] cannot contain upper case in the catalog."); } } diff --git a/paimon-core/src/test/java/org/apache/paimon/catalog/TestableCachingCatalog.java b/paimon-core/src/test/java/org/apache/paimon/catalog/TestableCachingCatalog.java index 159f5edaef1f..1d4a9b0e8a58 100644 --- a/paimon-core/src/test/java/org/apache/paimon/catalog/TestableCachingCatalog.java +++ b/paimon-core/src/test/java/org/apache/paimon/catalog/TestableCachingCatalog.java @@ -18,6 +18,7 @@ package org.apache.paimon.catalog; +import org.apache.paimon.manifest.PartitionEntry; import org.apache.paimon.options.MemorySize; import org.apache.paimon.table.Table; @@ -25,6 +26,7 @@ import org.apache.paimon.shade.caffeine2.com.github.benmanes.caffeine.cache.Ticker; import java.time.Duration; +import java.util.List; import java.util.Optional; /** @@ -36,18 +38,29 @@ public class TestableCachingCatalog extends CachingCatalog { private final Duration cacheExpirationInterval; public TestableCachingCatalog(Catalog catalog, Duration expirationInterval, Ticker ticker) { - super(catalog, expirationInterval, MemorySize.ZERO, Long.MAX_VALUE, ticker); + super( + catalog, + expirationInterval, + MemorySize.ZERO, + Long.MAX_VALUE, + Long.MAX_VALUE, + Integer.MAX_VALUE, + ticker); this.cacheExpirationInterval = expirationInterval; } - public Cache cache() { + public Cache tableCache() { // cleanUp must be called as tests apply assertions directly on the underlying map, but - // metadata - // table map entries are cleaned up asynchronously. + // metadata table map entries are cleaned up asynchronously. tableCache.cleanUp(); return tableCache; } + public Cache> partitionCache() { + partitionCache.cleanUp(); + return partitionCache; + } + public Optional ageOf(Identifier identifier) { return tableCache.policy().expireAfterAccess().get().ageOf(identifier); } diff --git a/paimon-core/src/test/java/org/apache/paimon/codegen/CodeGenUtilsTest.java b/paimon-core/src/test/java/org/apache/paimon/codegen/CodeGenUtilsTest.java index b7e416af137b..c0da56f54454 100644 --- a/paimon-core/src/test/java/org/apache/paimon/codegen/CodeGenUtilsTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/codegen/CodeGenUtilsTest.java @@ -71,14 +71,15 @@ public void testNormalizedKeyComputerCodegenCacheMiss() { @Test public void testRecordComparatorCodegenCache() { assertClassEquals( - () -> newRecordComparator(Arrays.asList(STRING(), INT()), new int[] {0, 1})); + () -> newRecordComparator(Arrays.asList(STRING(), INT()), new int[] {0, 1}, true)); } @Test public void testRecordComparatorCodegenCacheMiss() { assertClassNotEquals( - newRecordComparator(Arrays.asList(STRING(), INT()), new int[] {0, 1}), - newRecordComparator(Arrays.asList(STRING(), INT(), DOUBLE()), new int[] {0, 1, 2})); + newRecordComparator(Arrays.asList(STRING(), INT()), new int[] {0, 1}, true), + newRecordComparator( + Arrays.asList(STRING(), INT(), DOUBLE()), new int[] {0, 1, 2}, true)); } @Test @@ -96,7 +97,7 @@ public void testRecordEqualiserCodegenCacheMiss() { @Test public void testHybridNotEqual() { assertClassNotEquals( - newRecordComparator(Arrays.asList(STRING(), INT()), new int[] {0, 1}), + newRecordComparator(Arrays.asList(STRING(), INT()), new int[] {0, 1}, true), newNormalizedKeyComputer(Arrays.asList(STRING(), INT()), new int[] {0, 1})); } diff --git a/paimon-core/src/test/java/org/apache/paimon/crosspartition/IndexBootstrapTest.java b/paimon-core/src/test/java/org/apache/paimon/crosspartition/IndexBootstrapTest.java index d9d8e1f69bcf..be4147735614 100644 --- a/paimon-core/src/test/java/org/apache/paimon/crosspartition/IndexBootstrapTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/crosspartition/IndexBootstrapTest.java @@ -159,7 +159,8 @@ private static DataFileMeta newFile(long timeMillis) { .toLocalDateTime()), 0L, null, - FileSource.APPEND); + FileSource.APPEND, + null); } private Pair row(int pt, int col, int pk, int bucket) { diff --git a/paimon-core/src/test/java/org/apache/paimon/deletionvectors/append/AppendDeletionFileMaintainerTest.java b/paimon-core/src/test/java/org/apache/paimon/deletionvectors/append/AppendDeletionFileMaintainerTest.java index 6c674352b8d3..a52819c80515 100644 --- a/paimon-core/src/test/java/org/apache/paimon/deletionvectors/append/AppendDeletionFileMaintainerTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/deletionvectors/append/AppendDeletionFileMaintainerTest.java @@ -23,12 +23,12 @@ import org.apache.paimon.deletionvectors.DeletionVector; import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.local.LocalFileIO; +import org.apache.paimon.index.DeletionVectorMeta; import org.apache.paimon.index.IndexFileMeta; import org.apache.paimon.manifest.FileKind; import org.apache.paimon.manifest.IndexManifestEntry; import org.apache.paimon.table.sink.CommitMessageImpl; import org.apache.paimon.table.source.DeletionFile; -import org.apache.paimon.utils.Pair; import org.apache.paimon.utils.PathFactory; import org.junit.jupiter.api.Test; @@ -94,7 +94,7 @@ public void test() throws Exception { assertThat(res.size()).isEqualTo(3); IndexManifestEntry entry = res.stream().filter(file -> file.kind() == FileKind.ADD).findAny().get(); - assertThat(entry.indexFile().deletionVectorsRanges().containsKey("f2")).isTrue(); + assertThat(entry.indexFile().deletionVectorMetas().containsKey("f2")).isTrue(); entry = res.stream() .filter(file -> file.kind() == FileKind.DELETE) @@ -117,14 +117,15 @@ private Map createDeletionFileMapFromIndexFileMetas( PathFactory indexPathFactory, List fileMetas) { Map dataFileToDeletionFiles = new HashMap<>(); for (IndexFileMeta indexFileMeta : fileMetas) { - for (Map.Entry> range : - indexFileMeta.deletionVectorsRanges().entrySet()) { + for (Map.Entry dvMeta : + indexFileMeta.deletionVectorMetas().entrySet()) { dataFileToDeletionFiles.put( - range.getKey(), + dvMeta.getKey(), new DeletionFile( indexPathFactory.toPath(indexFileMeta.fileName()).toString(), - range.getValue().getLeft(), - range.getValue().getRight())); + dvMeta.getValue().offset(), + dvMeta.getValue().length(), + dvMeta.getValue().cardinality())); } } return dataFileToDeletionFiles; diff --git a/paimon-core/src/test/java/org/apache/paimon/format/FileFormatSuffixTest.java b/paimon-core/src/test/java/org/apache/paimon/format/FileFormatSuffixTest.java index 4aee88298926..c29519ce8b9b 100644 --- a/paimon-core/src/test/java/org/apache/paimon/format/FileFormatSuffixTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/format/FileFormatSuffixTest.java @@ -70,7 +70,9 @@ public void testFileSuffix(@TempDir java.nio.file.Path tempDir) throws Exception new Path(tempDir + "/dt=1/bucket-1"), format, CoreOptions.DATA_FILE_PREFIX.defaultValue(), - CoreOptions.CHANGELOG_FILE_PREFIX.defaultValue()); + CoreOptions.CHANGELOG_FILE_PREFIX.defaultValue(), + CoreOptions.FILE_SUFFIX_INCLUDE_COMPRESSION.defaultValue(), + CoreOptions.FILE_COMPRESSION.defaultValue()); FileFormat fileFormat = FileFormat.fromIdentifier(format, new Options()); LinkedList toCompact = new LinkedList<>(); CoreOptions options = new CoreOptions(new HashMap<>()); @@ -97,7 +99,8 @@ public void testFileSuffix(@TempDir java.nio.file.Path tempDir) throws Exception options, SCHEMA.getFieldNames()), MemorySize.MAX_VALUE, new FileIndexOptions(), - true); + true, + false); appendOnlyWriter.setMemoryPool( new HeapMemorySegmentPool(options.writeBufferSize(), options.pageSize())); appendOnlyWriter.write( diff --git a/paimon-core/src/test/java/org/apache/paimon/format/ThinModeReadWriteTest.java b/paimon-core/src/test/java/org/apache/paimon/format/ThinModeReadWriteTest.java new file mode 100644 index 000000000000..3f8015b33b2d --- /dev/null +++ b/paimon-core/src/test/java/org/apache/paimon/format/ThinModeReadWriteTest.java @@ -0,0 +1,158 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.format; + +import org.apache.paimon.data.Decimal; +import org.apache.paimon.data.GenericRow; +import org.apache.paimon.data.InternalRow; +import org.apache.paimon.data.Timestamp; +import org.apache.paimon.manifest.FileKind; +import org.apache.paimon.manifest.ManifestEntry; +import org.apache.paimon.schema.Schema; +import org.apache.paimon.table.FileStoreTable; +import org.apache.paimon.table.Table; +import org.apache.paimon.table.TableTestBase; +import org.apache.paimon.types.DataTypes; + +import org.assertj.core.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.List; + +/** This class test the compatibility and effectiveness of storage thin mode. */ +public class ThinModeReadWriteTest extends TableTestBase { + + private Table createTable(String format, Boolean thinMode) throws Exception { + catalog.createTable(identifier(), schema(format, thinMode), true); + return catalog.getTable(identifier()); + } + + private Schema schema(String format, Boolean thinMode) { + Schema.Builder schemaBuilder = Schema.newBuilder(); + schemaBuilder.column("f0", DataTypes.INT()); + schemaBuilder.column("f1", DataTypes.INT()); + schemaBuilder.column("f2", DataTypes.SMALLINT()); + schemaBuilder.column("f3", DataTypes.STRING()); + schemaBuilder.column("f4", DataTypes.DOUBLE()); + schemaBuilder.column("f5", DataTypes.CHAR(100)); + schemaBuilder.column("f6", DataTypes.VARCHAR(100)); + schemaBuilder.column("f7", DataTypes.BOOLEAN()); + schemaBuilder.column("f8", DataTypes.INT()); + schemaBuilder.column("f9", DataTypes.TIME()); + schemaBuilder.column("f10", DataTypes.TIMESTAMP()); + schemaBuilder.column("f11", DataTypes.DECIMAL(10, 2)); + schemaBuilder.column("f12", DataTypes.BYTES()); + schemaBuilder.column("f13", DataTypes.FLOAT()); + schemaBuilder.column("f14", DataTypes.BINARY(100)); + schemaBuilder.column("f15", DataTypes.VARBINARY(100)); + schemaBuilder.primaryKey( + "f0", "f1", "f2", "f3", "f4", "f5", "f6", "f7", "f8", "f9", "f10", "f11", "f12", + "f13"); + schemaBuilder.option("bucket", "1"); + schemaBuilder.option("bucket-key", "f1"); + schemaBuilder.option("file.format", format); + schemaBuilder.option("data-file.thin-mode", thinMode.toString()); + return schemaBuilder.build(); + } + + @Test + public void testThinModeWorks() throws Exception { + + InternalRow[] datas = datas(200000); + + Table table = createTable("orc", true); + write(table, datas); + + long size1 = tableSize(table); + dropTableDefault(); + + table = createTable("orc", false); + write(table, datas); + long size2 = tableSize(table); + dropTableDefault(); + + Assertions.assertThat(size2).isGreaterThan(size1); + } + + @Test + public void testAllFormatReadWrite() throws Exception { + testFormat("orc"); + testFormat("parquet"); + testFormat("avro"); + } + + private void testFormat(String format) throws Exception { + testReadWrite(format, true); + testReadWrite(format, true); + testReadWrite(format, false); + testReadWrite(format, false); + } + + private void testReadWrite(String format, boolean writeThin) throws Exception { + Table table = createTable(format, writeThin); + + InternalRow[] datas = datas(2000); + + write(table, datas); + + List readed = read(table); + + Assertions.assertThat(readed).containsExactlyInAnyOrder(datas); + dropTableDefault(); + } + + InternalRow[] datas(int i) { + InternalRow[] arrays = new InternalRow[i]; + for (int j = 0; j < i; j++) { + arrays[j] = data(); + } + return arrays; + } + + protected InternalRow data() { + return GenericRow.of( + RANDOM.nextInt(), + RANDOM.nextInt(), + (short) RANDOM.nextInt(), + randomString(), + RANDOM.nextDouble(), + randomString(), + randomString(), + RANDOM.nextBoolean(), + RANDOM.nextInt(), + RANDOM.nextInt(), + Timestamp.now(), + Decimal.zero(10, 2), + randomBytes(), + (float) RANDOM.nextDouble(), + randomBytes(), + randomBytes()); + } + + public static long tableSize(Table table) { + long count = 0; + List files = + ((FileStoreTable) table).store().newScan().plan().files(FileKind.ADD); + for (ManifestEntry file : files) { + count += file.file().fileSize(); + } + + return count; + } +} diff --git a/paimon-core/src/test/java/org/apache/paimon/hash/BytesHashMapTest.java b/paimon-core/src/test/java/org/apache/paimon/hash/BytesHashMapTest.java new file mode 100644 index 000000000000..4020e3d489d1 --- /dev/null +++ b/paimon-core/src/test/java/org/apache/paimon/hash/BytesHashMapTest.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.hash; + +import org.apache.paimon.data.BinaryRow; +import org.apache.paimon.data.serializer.BinaryRowSerializer; +import org.apache.paimon.memory.MemorySegmentPool; +import org.apache.paimon.types.DataType; + +/** Test case for {@link BytesHashMap}. */ +public class BytesHashMapTest extends BytesHashMapTestBase { + + public BytesHashMapTest() { + super(new BinaryRowSerializer(KEY_TYPES.length)); + } + + @Override + public BytesHashMap createBytesHashMap( + MemorySegmentPool pool, DataType[] keyTypes, DataType[] valueTypes) { + return new BytesHashMap<>( + pool, new BinaryRowSerializer(keyTypes.length), valueTypes.length); + } + + @Override + public BinaryRow[] generateRandomKeys(int num) { + return getRandomizedInputs(num); + } +} diff --git a/paimon-core/src/test/java/org/apache/paimon/hash/BytesHashMapTestBase.java b/paimon-core/src/test/java/org/apache/paimon/hash/BytesHashMapTestBase.java new file mode 100644 index 000000000000..01bbd46e63aa --- /dev/null +++ b/paimon-core/src/test/java/org/apache/paimon/hash/BytesHashMapTestBase.java @@ -0,0 +1,373 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.hash; + +import org.apache.paimon.data.BinaryRow; +import org.apache.paimon.data.RandomAccessInputView; +import org.apache.paimon.data.serializer.BinaryRowSerializer; +import org.apache.paimon.data.serializer.PagedTypeSerializer; +import org.apache.paimon.memory.HeapMemorySegmentPool; +import org.apache.paimon.memory.MemorySegment; +import org.apache.paimon.memory.MemorySegmentPool; +import org.apache.paimon.types.BigIntType; +import org.apache.paimon.types.BooleanType; +import org.apache.paimon.types.DataType; +import org.apache.paimon.types.DoubleType; +import org.apache.paimon.types.FloatType; +import org.apache.paimon.types.IntType; +import org.apache.paimon.types.RowType; +import org.apache.paimon.types.SmallIntType; +import org.apache.paimon.types.VarCharType; +import org.apache.paimon.utils.KeyValueIterator; + +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Random; + +import static org.assertj.core.api.Assertions.assertThat; + +/** Base test class for both {@link BytesHashMap}. */ +abstract class BytesHashMapTestBase extends BytesMapTestBase { + + private static final int NUM_REWRITES = 10; + + static final DataType[] KEY_TYPES = + new DataType[] { + new IntType(), + VarCharType.STRING_TYPE, + new DoubleType(), + new BigIntType(), + new BooleanType(), + new FloatType(), + new SmallIntType() + }; + + static final DataType[] VALUE_TYPES = + new DataType[] { + new DoubleType(), + new BigIntType(), + new BooleanType(), + new FloatType(), + new SmallIntType() + }; + + protected final BinaryRow defaultValue; + protected final PagedTypeSerializer keySerializer; + protected final BinaryRowSerializer valueSerializer; + + public BytesHashMapTestBase(PagedTypeSerializer keySerializer) { + this.keySerializer = keySerializer; + this.valueSerializer = new BinaryRowSerializer(VALUE_TYPES.length); + this.defaultValue = valueSerializer.createInstance(); + int valueSize = defaultValue.getFixedLengthPartSize(); + this.defaultValue.pointTo(MemorySegment.wrap(new byte[valueSize]), 0, valueSize); + } + + /** Creates the specific BytesHashMap, either {@link BytesHashMap}. */ + public abstract BytesHashMap createBytesHashMap( + MemorySegmentPool memorySegmentPool, DataType[] keyTypes, DataType[] valueTypes); + + /** + * Generates {@code num} random keys, the types of key fields are defined in {@link #KEY_TYPES}. + */ + public abstract K[] generateRandomKeys(int num); + + // ------------------------------------------------------------------------------------------ + // Tests + // ------------------------------------------------------------------------------------------ + + @Test + void testHashSetMode() throws IOException { + final int numMemSegments = + needNumMemSegments( + NUM_ENTRIES, + rowLength(RowType.of(VALUE_TYPES)), + rowLength(RowType.of(KEY_TYPES)), + PAGE_SIZE); + int memorySize = numMemSegments * PAGE_SIZE; + MemorySegmentPool pool = new HeapMemorySegmentPool(memorySize, PAGE_SIZE); + + BytesHashMap table = createBytesHashMap(pool, KEY_TYPES, new DataType[] {}); + assertThat(table.isHashSetMode()).isTrue(); + + K[] keys = generateRandomKeys(NUM_ENTRIES); + verifyKeyInsert(keys, table); + verifyKeyPresent(keys, table); + table.free(); + } + + @Test + void testBuildAndRetrieve() throws Exception { + + final int numMemSegments = + needNumMemSegments( + NUM_ENTRIES, + rowLength(RowType.of(VALUE_TYPES)), + rowLength(RowType.of(KEY_TYPES)), + PAGE_SIZE); + int memorySize = numMemSegments * PAGE_SIZE; + MemorySegmentPool pool = new HeapMemorySegmentPool(memorySize, PAGE_SIZE); + + BytesHashMap table = createBytesHashMap(pool, KEY_TYPES, VALUE_TYPES); + + K[] keys = generateRandomKeys(NUM_ENTRIES); + List expected = new ArrayList<>(NUM_ENTRIES); + verifyInsert(keys, expected, table); + verifyRetrieve(table, keys, expected); + table.free(); + } + + @Test + void testBuildAndUpdate() throws Exception { + final int numMemSegments = + needNumMemSegments( + NUM_ENTRIES, + rowLength(RowType.of(VALUE_TYPES)), + rowLength(RowType.of(KEY_TYPES)), + PAGE_SIZE); + int memorySize = numMemSegments * PAGE_SIZE; + MemorySegmentPool pool = new HeapMemorySegmentPool(memorySize, PAGE_SIZE); + + BytesHashMap table = createBytesHashMap(pool, KEY_TYPES, VALUE_TYPES); + + K[] keys = generateRandomKeys(NUM_ENTRIES); + List expected = new ArrayList<>(NUM_ENTRIES); + verifyInsertAndUpdate(keys, expected, table); + verifyRetrieve(table, keys, expected); + table.free(); + } + + @Test + void testRest() throws Exception { + final int numMemSegments = + needNumMemSegments( + NUM_ENTRIES, + rowLength(RowType.of(VALUE_TYPES)), + rowLength(RowType.of(KEY_TYPES)), + PAGE_SIZE); + + int memorySize = numMemSegments * PAGE_SIZE; + + MemorySegmentPool pool = new HeapMemorySegmentPool(memorySize, PAGE_SIZE); + + BytesHashMap table = createBytesHashMap(pool, KEY_TYPES, VALUE_TYPES); + + final K[] keys = generateRandomKeys(NUM_ENTRIES); + List expected = new ArrayList<>(NUM_ENTRIES); + verifyInsertAndUpdate(keys, expected, table); + verifyRetrieve(table, keys, expected); + + table.reset(); + assertThat(table.getNumElements()).isEqualTo(0); + assertThat(table.getRecordAreaMemorySegments()).hasSize(1); + + expected.clear(); + verifyInsertAndUpdate(keys, expected, table); + verifyRetrieve(table, keys, expected); + table.free(); + } + + @Test + void testResetAndOutput() throws Exception { + final Random rnd = new Random(RANDOM_SEED); + final int reservedMemSegments = 64; + + int minMemorySize = reservedMemSegments * PAGE_SIZE; + MemorySegmentPool pool = new HeapMemorySegmentPool(minMemorySize, PAGE_SIZE); + + BytesHashMap table = createBytesHashMap(pool, KEY_TYPES, VALUE_TYPES); + + K[] keys = generateRandomKeys(NUM_ENTRIES); + List expected = new ArrayList<>(NUM_ENTRIES); + List actualValues = new ArrayList<>(NUM_ENTRIES); + List actualKeys = new ArrayList<>(NUM_ENTRIES); + for (int i = 0; i < NUM_ENTRIES; i++) { + K groupKey = keys[i]; + // look up and insert + BytesMap.LookupInfo lookupInfo = table.lookup(groupKey); + assertThat(lookupInfo.isFound()).isFalse(); + try { + BinaryRow entry = table.append(lookupInfo, defaultValue); + assertThat(entry).isNotNull(); + // mock multiple updates + for (int j = 0; j < NUM_REWRITES; j++) { + updateOutputBuffer(entry, rnd); + } + expected.add(entry.copy()); + } catch (Exception e) { + ArrayList segments = table.getRecordAreaMemorySegments(); + RandomAccessInputView inView = + new RandomAccessInputView(segments, segments.get(0).size()); + K reuseKey = keySerializer.createReuseInstance(); + BinaryRow reuseValue = valueSerializer.createInstance(); + for (int index = 0; index < table.getNumElements(); index++) { + reuseKey = keySerializer.mapFromPages(reuseKey, inView); + reuseValue = valueSerializer.mapFromPages(reuseValue, inView); + actualKeys.add(keySerializer.copy(reuseKey)); + actualValues.add(reuseValue.copy()); + } + table.reset(); + // retry + lookupInfo = table.lookup(groupKey); + BinaryRow entry = table.append(lookupInfo, defaultValue); + assertThat(entry).isNotNull(); + // mock multiple updates + for (int j = 0; j < NUM_REWRITES; j++) { + updateOutputBuffer(entry, rnd); + } + expected.add(entry.copy()); + } + } + KeyValueIterator iter = table.getEntryIterator(false); + while (iter.advanceNext()) { + actualKeys.add(keySerializer.copy(iter.getKey())); + actualValues.add(iter.getValue().copy()); + } + assertThat(expected).hasSize(NUM_ENTRIES); + assertThat(actualKeys).hasSize(NUM_ENTRIES); + assertThat(actualValues).hasSize(NUM_ENTRIES); + assertThat(actualValues).isEqualTo(expected); + table.free(); + } + + @Test + void testSingleKeyMultipleOps() throws Exception { + final int numMemSegments = + needNumMemSegments( + NUM_ENTRIES, + rowLength(RowType.of(VALUE_TYPES)), + rowLength(RowType.of(KEY_TYPES)), + PAGE_SIZE); + + int memorySize = numMemSegments * PAGE_SIZE; + + MemorySegmentPool pool = new HeapMemorySegmentPool(memorySize, PAGE_SIZE); + + BytesHashMap table = createBytesHashMap(pool, KEY_TYPES, VALUE_TYPES); + final K key = generateRandomKeys(1)[0]; + for (int i = 0; i < 3; i++) { + BytesMap.LookupInfo lookupInfo = table.lookup(key); + assertThat(lookupInfo.isFound()).isFalse(); + } + + for (int i = 0; i < 3; i++) { + BytesMap.LookupInfo lookupInfo = table.lookup(key); + BinaryRow entry = lookupInfo.getValue(); + if (i == 0) { + assertThat(lookupInfo.isFound()).isFalse(); + entry = table.append(lookupInfo, defaultValue); + } else { + assertThat(lookupInfo.isFound()).isTrue(); + } + assertThat(entry).isNotNull(); + } + table.free(); + } + + // ---------------------------------------------- + /** It will be codegened when in HashAggExec using rnd to mock update/initExprs resultTerm. */ + private void updateOutputBuffer(BinaryRow reuse, Random rnd) { + long longVal = rnd.nextLong(); + double doubleVal = rnd.nextDouble(); + boolean boolVal = longVal % 2 == 0; + reuse.setDouble(2, doubleVal); + reuse.setLong(3, longVal); + reuse.setBoolean(4, boolVal); + } + + // ----------------------- Utilities ----------------------- + + private void verifyRetrieve(BytesHashMap table, K[] keys, List expected) { + assertThat(table.getNumElements()).isEqualTo(NUM_ENTRIES); + for (int i = 0; i < NUM_ENTRIES; i++) { + K groupKey = keys[i]; + // look up and retrieve + BytesMap.LookupInfo lookupInfo = table.lookup(groupKey); + assertThat(lookupInfo.isFound()).isTrue(); + assertThat(lookupInfo.getValue()).isNotNull(); + assertThat(lookupInfo.getValue()).isEqualTo(expected.get(i)); + } + } + + private void verifyInsert(K[] keys, List inserted, BytesHashMap table) + throws IOException { + for (int i = 0; i < NUM_ENTRIES; i++) { + K groupKey = keys[i]; + // look up and insert + BytesMap.LookupInfo lookupInfo = table.lookup(groupKey); + assertThat(lookupInfo.isFound()).isFalse(); + BinaryRow entry = table.append(lookupInfo, defaultValue); + assertThat(entry).isNotNull(); + assertThat(defaultValue).isEqualTo(entry); + inserted.add(entry.copy()); + } + assertThat(table.getNumElements()).isEqualTo(NUM_ENTRIES); + } + + private void verifyInsertAndUpdate(K[] keys, List inserted, BytesHashMap table) + throws IOException { + final Random rnd = new Random(RANDOM_SEED); + for (int i = 0; i < NUM_ENTRIES; i++) { + K groupKey = keys[i]; + // look up and insert + BytesMap.LookupInfo lookupInfo = table.lookup(groupKey); + assertThat(lookupInfo.isFound()).isFalse(); + BinaryRow entry = table.append(lookupInfo, defaultValue); + assertThat(entry).isNotNull(); + // mock multiple updates + for (int j = 0; j < NUM_REWRITES; j++) { + updateOutputBuffer(entry, rnd); + } + inserted.add(entry.copy()); + } + assertThat(table.getNumElements()).isEqualTo(NUM_ENTRIES); + } + + private void verifyKeyPresent(K[] keys, BytesHashMap table) { + assertThat(table.getNumElements()).isEqualTo(NUM_ENTRIES); + BinaryRow present = new BinaryRow(0); + present.pointTo(MemorySegment.wrap(new byte[8]), 0, 8); + for (int i = 0; i < NUM_ENTRIES; i++) { + K groupKey = keys[i]; + // look up and retrieve + BytesMap.LookupInfo lookupInfo = table.lookup(groupKey); + assertThat(lookupInfo.isFound()).isTrue(); + assertThat(lookupInfo.getValue()).isNotNull(); + assertThat(lookupInfo.getValue()).isEqualTo(present); + } + } + + private void verifyKeyInsert(K[] keys, BytesHashMap table) throws IOException { + BinaryRow present = new BinaryRow(0); + present.pointTo(MemorySegment.wrap(new byte[8]), 0, 8); + for (int i = 0; i < NUM_ENTRIES; i++) { + K groupKey = keys[i]; + // look up and insert + BytesMap.LookupInfo lookupInfo = table.lookup(groupKey); + assertThat(lookupInfo.isFound()).isFalse(); + BinaryRow entry = table.append(lookupInfo, defaultValue); + assertThat(entry).isNotNull(); + assertThat(present).isEqualTo(entry); + } + assertThat(table.getNumElements()).isEqualTo(NUM_ENTRIES); + } +} diff --git a/paimon-core/src/test/java/org/apache/paimon/hash/BytesMapTestBase.java b/paimon-core/src/test/java/org/apache/paimon/hash/BytesMapTestBase.java new file mode 100644 index 000000000000..acde6d9a1689 --- /dev/null +++ b/paimon-core/src/test/java/org/apache/paimon/hash/BytesMapTestBase.java @@ -0,0 +1,118 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.hash; + +import org.apache.paimon.data.BinaryRow; +import org.apache.paimon.data.BinaryRowWriter; +import org.apache.paimon.data.BinaryString; +import org.apache.paimon.types.DataType; +import org.apache.paimon.types.RowType; + +import java.util.Random; + +/** Test case for {@link BytesMap}. */ +public class BytesMapTestBase { + + protected static final long RANDOM_SEED = 76518743207143L; + protected static final int PAGE_SIZE = 32 * 1024; + protected static final int NUM_ENTRIES = 10000; + + protected BinaryRow[] getRandomizedInputs(int num) { + final Random rnd = new Random(RANDOM_SEED); + return getRandomizedInputs(num, rnd, true); + } + + protected BinaryRow[] getRandomizedInputs(int num, Random rnd, boolean nullable) { + BinaryRow[] lists = new BinaryRow[num]; + for (int i = 0; i < num; i++) { + int intVal = rnd.nextInt(Integer.MAX_VALUE); + long longVal = -rnd.nextLong(); + boolean boolVal = longVal % 2 == 0; + String strVal = nullable && boolVal ? null : getString(intVal, intVal % 1024) + i; + Double doubleVal = rnd.nextDouble(); + Short shotVal = (short) intVal; + Float floatVal = nullable && boolVal ? null : rnd.nextFloat(); + lists[i] = createRow(intVal, strVal, doubleVal, longVal, boolVal, floatVal, shotVal); + } + return lists; + } + + protected BinaryRow createRow( + Integer f0, String f1, Double f2, Long f3, Boolean f4, Float f5, Short f6) { + + BinaryRow row = new BinaryRow(7); + BinaryRowWriter writer = new BinaryRowWriter(row); + + // int, string, double, long, boolean + if (f0 == null) { + writer.setNullAt(0); + } else { + writer.writeInt(0, f0); + } + if (f1 == null) { + writer.setNullAt(1); + } else { + writer.writeString(1, BinaryString.fromString(f1)); + } + if (f2 == null) { + writer.setNullAt(2); + } else { + writer.writeDouble(2, f2); + } + if (f3 == null) { + writer.setNullAt(3); + } else { + writer.writeLong(3, f3); + } + if (f4 == null) { + writer.setNullAt(4); + } else { + writer.writeBoolean(4, f4); + } + if (f5 == null) { + writer.setNullAt(5); + } else { + writer.writeFloat(5, f5); + } + if (f6 == null) { + writer.setNullAt(6); + } else { + writer.writeShort(6, f6); + } + writer.complete(); + return row; + } + + protected int needNumMemSegments(int numEntries, int valLen, int keyLen, int pageSize) { + return 2 * (valLen + keyLen + 1024 * 3 + 4 + 8 + 8) * numEntries / pageSize; + } + + protected int rowLength(RowType tpe) { + return BinaryRow.calculateFixPartSizeInBytes(tpe.getFieldCount()) + + BytesHashMap.getVariableLength(tpe.getFieldTypes().toArray(new DataType[0])); + } + + private String getString(int count, int length) { + StringBuilder builder = new StringBuilder(); + for (int i = 0; i < length; i++) { + builder.append(count); + } + return builder.toString(); + } +} diff --git a/paimon-core/src/test/java/org/apache/paimon/iceberg/IcebergCompatibilityTest.java b/paimon-core/src/test/java/org/apache/paimon/iceberg/IcebergCompatibilityTest.java index 141def4e224a..7258a1dd4170 100644 --- a/paimon-core/src/test/java/org/apache/paimon/iceberg/IcebergCompatibilityTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/iceberg/IcebergCompatibilityTest.java @@ -25,10 +25,15 @@ import org.apache.paimon.data.BinaryRowWriter; import org.apache.paimon.data.BinaryString; import org.apache.paimon.data.Decimal; +import org.apache.paimon.data.GenericArray; +import org.apache.paimon.data.GenericMap; import org.apache.paimon.data.GenericRow; +import org.apache.paimon.data.Timestamp; import org.apache.paimon.disk.IOManagerImpl; +import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.Path; import org.apache.paimon.fs.local.LocalFileIO; +import org.apache.paimon.iceberg.manifest.IcebergManifestFile; import org.apache.paimon.iceberg.manifest.IcebergManifestFileMeta; import org.apache.paimon.iceberg.manifest.IcebergManifestList; import org.apache.paimon.iceberg.metadata.IcebergMetadata; @@ -41,6 +46,7 @@ import org.apache.paimon.table.sink.CommitMessage; import org.apache.paimon.table.sink.TableCommitImpl; import org.apache.paimon.table.sink.TableWriteImpl; +import org.apache.paimon.types.DataField; import org.apache.paimon.types.DataType; import org.apache.paimon.types.DataTypeRoot; import org.apache.paimon.types.DataTypes; @@ -59,6 +65,7 @@ import java.math.BigDecimal; import java.time.LocalDate; +import java.time.LocalDateTime; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -75,6 +82,7 @@ import java.util.stream.Collectors; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; /** Tests for Iceberg compatibility. */ public class IcebergCompatibilityTest { @@ -199,7 +207,8 @@ public void testRetryCreateMetadata() throws Exception { commit.commit(2, commitMessages2); assertThat(table.latestSnapshotId()).hasValue(3L); - IcebergPathFactory pathFactory = new IcebergPathFactory(table.location()); + IcebergPathFactory pathFactory = + new IcebergPathFactory(new Path(table.location(), "metadata")); Path metadata3Path = pathFactory.toMetadataPath(3); assertThat(table.fileIO().exists(metadata3Path)).isTrue(); @@ -276,9 +285,10 @@ public void testIcebergSnapshotExpire() throws Exception { write.write(GenericRow.of(2, 20)); commit.commit(1, write.prepareCommit(false, 1)); assertThat(table.snapshotManager().latestSnapshotId()).isEqualTo(1L); + FileIO fileIO = table.fileIO(); IcebergMetadata metadata = IcebergMetadata.fromPath( - table.fileIO(), new Path(table.location(), "metadata/v1.metadata.json")); + fileIO, new Path(table.location(), "metadata/v1.metadata.json")); assertThat(metadata.snapshots()).hasSize(1); assertThat(metadata.currentSnapshotId()).isEqualTo(1); @@ -289,21 +299,47 @@ public void testIcebergSnapshotExpire() throws Exception { assertThat(table.snapshotManager().latestSnapshotId()).isEqualTo(3L); metadata = IcebergMetadata.fromPath( - table.fileIO(), new Path(table.location(), "metadata/v3.metadata.json")); + fileIO, new Path(table.location(), "metadata/v3.metadata.json")); assertThat(metadata.snapshots()).hasSize(3); assertThat(metadata.currentSnapshotId()).isEqualTo(3); // Number of snapshots will become 5 with the next commit, however only 3 Iceberg snapshots // are kept. So the first 2 Iceberg snapshots will be expired. - IcebergPathFactory pathFactory = new IcebergPathFactory(table.location()); + IcebergPathFactory pathFactory = + new IcebergPathFactory(new Path(table.location(), "metadata")); IcebergManifestList manifestList = IcebergManifestList.create(table, pathFactory); + assertThat(manifestList.compression()).isEqualTo("snappy"); + + IcebergManifestFile manifestFile = IcebergManifestFile.create(table, pathFactory); + assertThat(manifestFile.compression()).isEqualTo("snappy"); + Set usingManifests = new HashSet<>(); - for (IcebergManifestFileMeta fileMeta : - manifestList.read(new Path(metadata.currentSnapshot().manifestList()).getName())) { + String manifestListFile = new Path(metadata.currentSnapshot().manifestList()).getName(); + + assertThat(fileIO.readFileUtf8(new Path(pathFactory.metadataDirectory(), manifestListFile))) + .contains("snappy"); + + for (IcebergManifestFileMeta fileMeta : manifestList.read(manifestListFile)) { usingManifests.add(fileMeta.manifestPath()); + assertThat( + fileIO.readFileUtf8( + new Path( + pathFactory.metadataDirectory(), + fileMeta.manifestPath()))) + .contains("snappy"); } + IcebergManifestList legacyManifestList = + IcebergManifestList.create( + table.copy( + Collections.singletonMap( + IcebergOptions.MANIFEST_LEGACY_VERSION.key(), "true")), + pathFactory); + assertThatThrownBy(() -> legacyManifestList.read(manifestListFile)) + .rootCause() + .isInstanceOf(NullPointerException.class); + Set unusedFiles = new HashSet<>(); for (int i = 0; i < 2; i++) { unusedFiles.add(metadata.snapshots().get(i).manifestList()); @@ -324,7 +360,7 @@ public void testIcebergSnapshotExpire() throws Exception { assertThat(table.snapshotManager().latestSnapshotId()).isEqualTo(5L); metadata = IcebergMetadata.fromPath( - table.fileIO(), new Path(table.location(), "metadata/v5.metadata.json")); + fileIO, new Path(table.location(), "metadata/v5.metadata.json")); assertThat(metadata.snapshots()).hasSize(3); assertThat(metadata.currentSnapshotId()).isEqualTo(5); @@ -337,7 +373,7 @@ public void testIcebergSnapshotExpire() throws Exception { } for (String path : unusedFiles) { - assertThat(table.fileIO().exists(new Path(path))).isFalse(); + assertThat(fileIO.exists(new Path(path))).isFalse(); } // Test all existing Iceberg snapshots are valid. @@ -372,7 +408,8 @@ public void testAllTypeStatistics() throws Exception { DataTypes.STRING(), DataTypes.BINARY(20), DataTypes.VARBINARY(20), - DataTypes.DATE() + DataTypes.DATE(), + DataTypes.TIMESTAMP(6) }, new String[] { "v_int", @@ -385,7 +422,8 @@ public void testAllTypeStatistics() throws Exception { "v_varchar", "v_binary", "v_varbinary", - "v_date" + "v_date", + "v_timestamp" }); FileStoreTable table = createPaimonTable(rowType, Collections.emptyList(), Collections.emptyList(), -1); @@ -406,7 +444,8 @@ public void testAllTypeStatistics() throws Exception { BinaryString.fromString("cat"), "B_apple".getBytes(), "B_cat".getBytes(), - 100); + 100, + Timestamp.fromLocalDateTime(LocalDateTime.of(2024, 10, 10, 11, 22, 33))); write.write(lowerBounds); GenericRow upperBounds = GenericRow.of( @@ -420,7 +459,8 @@ public void testAllTypeStatistics() throws Exception { BinaryString.fromString("dog"), "B_banana".getBytes(), "B_dog".getBytes(), - 200); + 200, + Timestamp.fromLocalDateTime(LocalDateTime.of(2024, 10, 20, 11, 22, 33))); write.write(upperBounds); commit.commit(1, write.prepareCommit(false, 1)); @@ -448,6 +488,9 @@ public void testAllTypeStatistics() throws Exception { } else if (type.getTypeRoot() == DataTypeRoot.DECIMAL) { lower = new BigDecimal(lowerBounds.getField(i).toString()); upper = new BigDecimal(upperBounds.getField(i).toString()); + } else if (type.getTypeRoot() == DataTypeRoot.TIMESTAMP_WITHOUT_TIME_ZONE) { + lower = ((Timestamp) lowerBounds.getField(i)).toMicros(); + upper = ((Timestamp) upperBounds.getField(i)).toMicros(); } else { lower = lowerBounds.getField(i); upper = upperBounds.getField(i); @@ -458,6 +501,9 @@ public void testAllTypeStatistics() throws Exception { if (type.getTypeRoot() == DataTypeRoot.DATE) { expectedLower = LocalDate.ofEpochDay((int) lower).toString(); expectedUpper = LocalDate.ofEpochDay((int) upper).toString(); + } else if (type.getTypeRoot() == DataTypeRoot.TIMESTAMP_WITHOUT_TIME_ZONE) { + expectedLower = Timestamp.fromMicros((long) lower).toString(); + expectedUpper = Timestamp.fromMicros((long) upper).toString(); } assertThat( @@ -499,6 +545,56 @@ public void testAllTypeStatistics() throws Exception { } } + @Test + public void testNestedTypes() throws Exception { + RowType innerType = + RowType.of( + new DataField(2, "f1", DataTypes.STRING()), + new DataField(3, "f2", DataTypes.INT())); + RowType rowType = + RowType.of( + new DataField(0, "k", DataTypes.INT()), + new DataField( + 1, + "v", + DataTypes.MAP(DataTypes.INT(), DataTypes.ARRAY(innerType)))); + FileStoreTable table = + createPaimonTable(rowType, Collections.emptyList(), Collections.emptyList(), -1); + + String commitUser = UUID.randomUUID().toString(); + TableWriteImpl write = table.newWrite(commitUser); + TableCommitImpl commit = table.newCommit(commitUser); + + Map map1 = new HashMap<>(); + map1.put( + 10, + new GenericArray( + new GenericRow[] { + GenericRow.of(BinaryString.fromString("apple"), 100), + GenericRow.of(BinaryString.fromString("banana"), 101) + })); + write.write(GenericRow.of(1, new GenericMap(map1))); + + Map map2 = new HashMap<>(); + map2.put( + 20, + new GenericArray( + new GenericRow[] { + GenericRow.of(BinaryString.fromString("cherry"), 200), + GenericRow.of(BinaryString.fromString("pear"), 201) + })); + write.write(GenericRow.of(2, new GenericMap(map2))); + + commit.commit(1, write.prepareCommit(false, 1)); + write.close(); + commit.close(); + + assertThat(getIcebergResult()) + .containsExactlyInAnyOrder( + "Record(1, {10=[Record(apple, 100), Record(banana, 101)]})", + "Record(2, {20=[Record(cherry, 200), Record(pear, 201)]})"); + } + // ------------------------------------------------------------------------ // Random Tests // ------------------------------------------------------------------------ @@ -689,11 +785,12 @@ private FileStoreTable createPaimonTable( Options options = new Options(customOptions); options.set(CoreOptions.BUCKET, numBuckets); - options.set(CoreOptions.METADATA_ICEBERG_COMPATIBLE, true); + options.set( + IcebergOptions.METADATA_ICEBERG_STORAGE, IcebergOptions.StorageType.TABLE_LOCATION); options.set(CoreOptions.FILE_FORMAT, "avro"); options.set(CoreOptions.TARGET_FILE_SIZE, MemorySize.ofKibiBytes(32)); - options.set(AbstractIcebergCommitCallback.COMPACT_MIN_FILE_NUM, 4); - options.set(AbstractIcebergCommitCallback.COMPACT_MIN_FILE_NUM, 8); + options.set(IcebergOptions.COMPACT_MIN_FILE_NUM, 4); + options.set(IcebergOptions.COMPACT_MIN_FILE_NUM, 8); options.set(CoreOptions.MANIFEST_TARGET_FILE_SIZE, MemorySize.ofKibiBytes(8)); Schema schema = new Schema(rowType.getFields(), partitionKeys, primaryKeys, options.toMap(), ""); diff --git a/paimon-core/src/test/java/org/apache/paimon/index/IndexFileMetaSerializerTest.java b/paimon-core/src/test/java/org/apache/paimon/index/IndexFileMetaSerializerTest.java index 724d5b416359..a7e692d2e554 100644 --- a/paimon-core/src/test/java/org/apache/paimon/index/IndexFileMetaSerializerTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/index/IndexFileMetaSerializerTest.java @@ -21,7 +21,6 @@ import org.apache.paimon.deletionvectors.DeletionVectorsIndexFile; import org.apache.paimon.utils.ObjectSerializer; import org.apache.paimon.utils.ObjectSerializerTestBase; -import org.apache.paimon.utils.Pair; import java.util.LinkedHashMap; import java.util.Random; @@ -59,14 +58,20 @@ public static IndexFileMeta randomHashIndexFile() { public static IndexFileMeta randomDeletionVectorIndexFile() { Random rnd = new Random(); - LinkedHashMap> deletionVectorsRanges = new LinkedHashMap<>(); - deletionVectorsRanges.put("my_file_name1", Pair.of(rnd.nextInt(), rnd.nextInt())); - deletionVectorsRanges.put("my_file_name2", Pair.of(rnd.nextInt(), rnd.nextInt())); + LinkedHashMap deletionVectorMetas = new LinkedHashMap<>(); + deletionVectorMetas.put( + "my_file_name1", + new DeletionVectorMeta( + "my_file_name1", rnd.nextInt(), rnd.nextInt(), rnd.nextLong())); + deletionVectorMetas.put( + "my_file_name2", + new DeletionVectorMeta( + "my_file_name2", rnd.nextInt(), rnd.nextInt(), rnd.nextLong())); return new IndexFileMeta( DeletionVectorsIndexFile.DELETION_VECTORS_INDEX, "deletion_vectors_index_file_name" + rnd.nextLong(), rnd.nextInt(), rnd.nextInt(), - deletionVectorsRanges); + deletionVectorMetas); } } diff --git a/paimon-core/src/test/java/org/apache/paimon/io/DataFilePathFactoryTest.java b/paimon-core/src/test/java/org/apache/paimon/io/DataFilePathFactoryTest.java index 609566258961..d36966c55a0e 100644 --- a/paimon-core/src/test/java/org/apache/paimon/io/DataFilePathFactoryTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/io/DataFilePathFactoryTest.java @@ -38,7 +38,9 @@ public void testNoPartition() { new Path(tempDir + "/bucket-123"), CoreOptions.FILE_FORMAT.defaultValue().toString(), CoreOptions.DATA_FILE_PREFIX.defaultValue(), - CoreOptions.CHANGELOG_FILE_PREFIX.defaultValue()); + CoreOptions.CHANGELOG_FILE_PREFIX.defaultValue(), + CoreOptions.FILE_SUFFIX_INCLUDE_COMPRESSION.defaultValue(), + CoreOptions.FILE_COMPRESSION.defaultValue()); String uuid = pathFactory.uuid(); for (int i = 0; i < 20; i++) { @@ -64,7 +66,9 @@ public void testWithPartition() { new Path(tempDir + "/dt=20211224/bucket-123"), CoreOptions.FILE_FORMAT.defaultValue().toString(), CoreOptions.DATA_FILE_PREFIX.defaultValue(), - CoreOptions.CHANGELOG_FILE_PREFIX.defaultValue()); + CoreOptions.CHANGELOG_FILE_PREFIX.defaultValue(), + CoreOptions.FILE_SUFFIX_INCLUDE_COMPRESSION.defaultValue(), + CoreOptions.FILE_COMPRESSION.defaultValue()); String uuid = pathFactory.uuid(); for (int i = 0; i < 20; i++) { diff --git a/paimon-core/src/test/java/org/apache/paimon/io/DataFileTestDataGenerator.java b/paimon-core/src/test/java/org/apache/paimon/io/DataFileTestDataGenerator.java index 810cef860784..7de4214beaa9 100644 --- a/paimon-core/src/test/java/org/apache/paimon/io/DataFileTestDataGenerator.java +++ b/paimon-core/src/test/java/org/apache/paimon/io/DataFileTestDataGenerator.java @@ -156,15 +156,16 @@ private Data createDataFile(List kvs, int level, BinaryRow partition, kvs.size(), minKey, maxKey, - keyStatsSerializer.toBinary(keyStatsCollector.extract()), - valueStatsSerializer.toBinary(valueStatsCollector.extract()), + keyStatsSerializer.toBinaryAllMode(keyStatsCollector.extract()), + valueStatsSerializer.toBinaryAllMode(valueStatsCollector.extract()), minSequenceNumber, maxSequenceNumber, 0, level, kvs.stream().filter(kv -> kv.valueKind().isRetract()).count(), null, - FileSource.APPEND), + FileSource.APPEND, + null), kvs); } diff --git a/paimon-core/src/test/java/org/apache/paimon/io/DataFileTestUtils.java b/paimon-core/src/test/java/org/apache/paimon/io/DataFileTestUtils.java index 03b9babfaa39..48c8d44876ae 100644 --- a/paimon-core/src/test/java/org/apache/paimon/io/DataFileTestUtils.java +++ b/paimon-core/src/test/java/org/apache/paimon/io/DataFileTestUtils.java @@ -56,7 +56,8 @@ public static DataFileMeta newFile(long minSeq, long maxSeq) { Timestamp.fromEpochMillis(100), maxSeq - minSeq + 1, null, - FileSource.APPEND); + FileSource.APPEND, + null); } public static DataFileMeta newFile() { @@ -74,7 +75,8 @@ public static DataFileMeta newFile() { 0, 0L, null, - FileSource.APPEND); + FileSource.APPEND, + null); } public static DataFileMeta newFile( @@ -98,7 +100,8 @@ public static DataFileMeta newFile( level, deleteRowCount, null, - FileSource.APPEND); + FileSource.APPEND, + null); } public static BinaryRow row(int i) { diff --git a/paimon-core/src/test/java/org/apache/paimon/io/KeyValueFileReadWriteTest.java b/paimon-core/src/test/java/org/apache/paimon/io/KeyValueFileReadWriteTest.java index 3c89552310f6..e43cd898dbc2 100644 --- a/paimon-core/src/test/java/org/apache/paimon/io/KeyValueFileReadWriteTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/io/KeyValueFileReadWriteTest.java @@ -37,6 +37,7 @@ import org.apache.paimon.options.Options; import org.apache.paimon.reader.RecordReaderIterator; import org.apache.paimon.stats.StatsTestUtils; +import org.apache.paimon.table.SpecialFields; import org.apache.paimon.types.RowType; import org.apache.paimon.utils.CloseableIterator; import org.apache.paimon.utils.FailingFileIO; @@ -158,7 +159,7 @@ public void testReadKeyType() throws Exception { List actualMetas = writer.result(); // projection: (shopId, orderId) -> (orderId) - RowType readKeyType = KEY_TYPE.project("key_orderId"); + RowType readKeyType = KEY_TYPE.project(SpecialFields.KEY_FIELD_PREFIX + "orderId"); KeyValueFileReaderFactory readerFactory = createReaderFactory(tempDir.toString(), "avro", readKeyType, null); InternalRowSerializer projectedKeySerializer = new InternalRowSerializer(readKeyType); @@ -228,7 +229,11 @@ protected KeyValueFileWriterFactory createWriterFactory(String pathStr, String f CoreOptions.PARTITION_DEFAULT_NAME.defaultValue(), format, CoreOptions.DATA_FILE_PREFIX.defaultValue(), - CoreOptions.CHANGELOG_FILE_PREFIX.defaultValue()); + CoreOptions.CHANGELOG_FILE_PREFIX.defaultValue(), + CoreOptions.PARTITION_GENERATE_LEGCY_NAME.defaultValue(), + CoreOptions.FILE_SUFFIX_INCLUDE_COMPRESSION.defaultValue(), + CoreOptions.FILE_COMPRESSION.defaultValue(), + null); int suggestedFileSize = ThreadLocalRandom.current().nextInt(8192) + 1024; FileIO fileIO = FileIOFinder.find(path); Options options = new Options(); @@ -244,7 +249,11 @@ protected KeyValueFileWriterFactory createWriterFactory(String pathStr, String f CoreOptions.PARTITION_DEFAULT_NAME.defaultValue(), CoreOptions.FILE_FORMAT.defaultValue().toString(), CoreOptions.DATA_FILE_PREFIX.defaultValue(), - CoreOptions.CHANGELOG_FILE_PREFIX.defaultValue())); + CoreOptions.CHANGELOG_FILE_PREFIX.defaultValue(), + CoreOptions.PARTITION_GENERATE_LEGCY_NAME.defaultValue(), + CoreOptions.FILE_SUFFIX_INCLUDE_COMPRESSION.defaultValue(), + CoreOptions.FILE_COMPRESSION.defaultValue(), + null)); return KeyValueFileWriterFactory.builder( fileIO, diff --git a/paimon-core/src/test/java/org/apache/paimon/io/RollingFileWriterTest.java b/paimon-core/src/test/java/org/apache/paimon/io/RollingFileWriterTest.java index df9f2ac0ba36..9e1de71451a8 100644 --- a/paimon-core/src/test/java/org/apache/paimon/io/RollingFileWriterTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/io/RollingFileWriterTest.java @@ -35,6 +35,7 @@ import org.apache.paimon.utils.StatsCollectorFactories; import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; +import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; @@ -62,6 +63,10 @@ public class RollingFileWriterTest { private RollingFileWriter rollingFileWriter; public void initialize(String identifier) { + initialize(identifier, false); + } + + public void initialize(String identifier, boolean statsDenseStore) { FileFormat fileFormat = FileFormat.fromIdentifier(identifier, new Options()); rollingFileWriter = new RollingFileWriter<>( @@ -76,7 +81,10 @@ public void initialize(String identifier) { .toString(), CoreOptions.DATA_FILE_PREFIX.defaultValue(), CoreOptions.CHANGELOG_FILE_PREFIX - .defaultValue()) + .defaultValue(), + CoreOptions.FILE_SUFFIX_INCLUDE_COMPRESSION + .defaultValue(), + CoreOptions.FILE_COMPRESSION.defaultValue()) .newPath(), SCHEMA, fileFormat @@ -94,7 +102,8 @@ public void initialize(String identifier) { SCHEMA.getFieldNames()), new FileIndexOptions(), FileSource.APPEND, - true), + true, + statsDenseStore), TARGET_FILE_SIZE); } @@ -123,4 +132,16 @@ private void assertFileNum(int expected) { File[] files = dataDir.listFiles(); assertThat(files).isNotNull().hasSize(expected); } + + @Test + public void testStatsDenseStore() throws IOException { + initialize("parquet", true); + for (int i = 0; i < 1000; i++) { + rollingFileWriter.write(GenericRow.of(i)); + } + rollingFileWriter.close(); + DataFileMeta file = rollingFileWriter.result().get(0); + assertThat(file.valueStatsCols()).isNull(); + assertThat(file.valueStats().minValues().getFieldCount()).isEqualTo(SCHEMA.getFieldCount()); + } } diff --git a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestCommittableSerializerCompatibilityTest.java b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestCommittableSerializerCompatibilityTest.java index 744904c71084..fbc02b2d73f2 100644 --- a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestCommittableSerializerCompatibilityTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestCommittableSerializerCompatibilityTest.java @@ -19,6 +19,7 @@ package org.apache.paimon.manifest; import org.apache.paimon.data.Timestamp; +import org.apache.paimon.index.DeletionVectorMeta; import org.apache.paimon.index.IndexFileMeta; import org.apache.paimon.io.CompactIncrement; import org.apache.paimon.io.DataFileMeta; @@ -27,7 +28,6 @@ import org.apache.paimon.stats.SimpleStats; import org.apache.paimon.table.sink.CommitMessageImpl; import org.apache.paimon.utils.IOUtils; -import org.apache.paimon.utils.Pair; import org.junit.jupiter.api.Test; @@ -74,14 +74,15 @@ public void testProduction() throws IOException { Timestamp.fromLocalDateTime(LocalDateTime.parse("2022-03-02T20:20:12")), 11L, new byte[] {1, 2, 4}, - FileSource.COMPACT); + FileSource.COMPACT, + Arrays.asList("field1", "field2", "field3")); List dataFiles = Collections.singletonList(dataFile); - LinkedHashMap> dvRanges = new LinkedHashMap<>(); - dvRanges.put("dv_key1", Pair.of(1, 2)); - dvRanges.put("dv_key2", Pair.of(3, 4)); + LinkedHashMap dvMetas = new LinkedHashMap<>(); + dvMetas.put("dv_key1", new DeletionVectorMeta("dv_key1", 1, 2, 3L)); + dvMetas.put("dv_key2", new DeletionVectorMeta("dv_key2", 3, 4, 5L)); IndexFileMeta indexFile = - new IndexFileMeta("my_index_type", "my_index_file", 1024 * 100, 1002, dvRanges); + new IndexFileMeta("my_index_type", "my_index_file", 1024 * 100, 1002, dvMetas); List indexFiles = Collections.singletonList(indexFile); CommitMessageImpl commitMessage = @@ -105,6 +106,146 @@ public void testProduction() throws IOException { assertThat(deserialized).isEqualTo(manifestCommittable); } + @Test + public void testCompatibilityToVersion4() throws IOException { + SimpleStats keyStats = + new SimpleStats( + singleColumn("min_key"), + singleColumn("max_key"), + fromLongArray(new Long[] {0L})); + SimpleStats valueStats = + new SimpleStats( + singleColumn("min_value"), + singleColumn("max_value"), + fromLongArray(new Long[] {0L})); + DataFileMeta dataFile = + new DataFileMeta( + "my_file", + 1024 * 1024, + 1024, + singleColumn("min_key"), + singleColumn("max_key"), + keyStats, + valueStats, + 15, + 200, + 5, + 3, + Arrays.asList("extra1", "extra2"), + Timestamp.fromLocalDateTime(LocalDateTime.parse("2022-03-02T20:20:12")), + 11L, + new byte[] {1, 2, 4}, + FileSource.COMPACT, + Arrays.asList("field1", "field2", "field3")); + List dataFiles = Collections.singletonList(dataFile); + + LinkedHashMap dvMetas = new LinkedHashMap<>(); + dvMetas.put("dv_key1", new DeletionVectorMeta("dv_key1", 1, 2, null)); + dvMetas.put("dv_key2", new DeletionVectorMeta("dv_key2", 3, 4, null)); + IndexFileMeta indexFile = + new IndexFileMeta("my_index_type", "my_index_file", 1024 * 100, 1002, dvMetas); + List indexFiles = Collections.singletonList(indexFile); + + CommitMessageImpl commitMessage = + new CommitMessageImpl( + singleColumn("my_partition"), + 11, + new DataIncrement(dataFiles, dataFiles, dataFiles), + new CompactIncrement(dataFiles, dataFiles, dataFiles), + new IndexIncrement(indexFiles)); + + ManifestCommittable manifestCommittable = + new ManifestCommittable( + 5, + 202020L, + Collections.singletonMap(5, 555L), + Collections.singletonList(commitMessage)); + + ManifestCommittableSerializer serializer = new ManifestCommittableSerializer(); + byte[] bytes = serializer.serialize(manifestCommittable); + ManifestCommittable deserialized = serializer.deserialize(3, bytes); + assertThat(deserialized).isEqualTo(manifestCommittable); + + byte[] v2Bytes = + IOUtils.readFully( + ManifestCommittableSerializerCompatibilityTest.class + .getClassLoader() + .getResourceAsStream("compatibility/manifest-committable-v4"), + true); + deserialized = serializer.deserialize(2, v2Bytes); + assertThat(deserialized).isEqualTo(manifestCommittable); + } + + @Test + public void testCompatibilityToVersion3() throws IOException { + SimpleStats keyStats = + new SimpleStats( + singleColumn("min_key"), + singleColumn("max_key"), + fromLongArray(new Long[] {0L})); + SimpleStats valueStats = + new SimpleStats( + singleColumn("min_value"), + singleColumn("max_value"), + fromLongArray(new Long[] {0L})); + DataFileMeta dataFile = + new DataFileMeta( + "my_file", + 1024 * 1024, + 1024, + singleColumn("min_key"), + singleColumn("max_key"), + keyStats, + valueStats, + 15, + 200, + 5, + 3, + Arrays.asList("extra1", "extra2"), + Timestamp.fromLocalDateTime(LocalDateTime.parse("2022-03-02T20:20:12")), + 11L, + new byte[] {1, 2, 4}, + FileSource.COMPACT, + null); + List dataFiles = Collections.singletonList(dataFile); + + LinkedHashMap dvMetas = new LinkedHashMap<>(); + dvMetas.put("dv_key1", new DeletionVectorMeta("dv_key1", 1, 2, null)); + dvMetas.put("dv_key2", new DeletionVectorMeta("dv_key2", 3, 4, null)); + IndexFileMeta indexFile = + new IndexFileMeta("my_index_type", "my_index_file", 1024 * 100, 1002, dvMetas); + List indexFiles = Collections.singletonList(indexFile); + + CommitMessageImpl commitMessage = + new CommitMessageImpl( + singleColumn("my_partition"), + 11, + new DataIncrement(dataFiles, dataFiles, dataFiles), + new CompactIncrement(dataFiles, dataFiles, dataFiles), + new IndexIncrement(indexFiles)); + + ManifestCommittable manifestCommittable = + new ManifestCommittable( + 5, + 202020L, + Collections.singletonMap(5, 555L), + Collections.singletonList(commitMessage)); + + ManifestCommittableSerializer serializer = new ManifestCommittableSerializer(); + byte[] bytes = serializer.serialize(manifestCommittable); + ManifestCommittable deserialized = serializer.deserialize(3, bytes); + assertThat(deserialized).isEqualTo(manifestCommittable); + + byte[] v2Bytes = + IOUtils.readFully( + ManifestCommittableSerializerCompatibilityTest.class + .getClassLoader() + .getResourceAsStream("compatibility/manifest-committable-v3"), + true); + deserialized = serializer.deserialize(2, v2Bytes); + assertThat(deserialized).isEqualTo(manifestCommittable); + } + @Test public void testCompatibilityToVersion2() throws IOException { SimpleStats keyStats = @@ -134,14 +275,15 @@ public void testCompatibilityToVersion2() throws IOException { Timestamp.fromLocalDateTime(LocalDateTime.parse("2022-03-02T20:20:12")), 11L, new byte[] {1, 2, 4}, + null, null); List dataFiles = Collections.singletonList(dataFile); - LinkedHashMap> dvRanges = new LinkedHashMap<>(); - dvRanges.put("dv_key1", Pair.of(1, 2)); - dvRanges.put("dv_key2", Pair.of(3, 4)); + LinkedHashMap dvMetas = new LinkedHashMap<>(); + dvMetas.put("dv_key1", new DeletionVectorMeta("dv_key1", 1, 2, null)); + dvMetas.put("dv_key2", new DeletionVectorMeta("dv_key2", 3, 4, null)); IndexFileMeta indexFile = - new IndexFileMeta("my_index_type", "my_index_file", 1024 * 100, 1002, dvRanges); + new IndexFileMeta("my_index_type", "my_index_file", 1024 * 100, 1002, dvMetas); List indexFiles = Collections.singletonList(indexFile); CommitMessageImpl commitMessage = @@ -203,6 +345,7 @@ public void testCompatibilityToVersion2PaimonV07() throws IOException { Timestamp.fromLocalDateTime(LocalDateTime.parse("2022-03-02T20:20:12")), null, null, + null, null); List dataFiles = Collections.singletonList(dataFile); diff --git a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestCommittableSerializerTest.java b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestCommittableSerializerTest.java index 099c38003391..8de8309bc8fb 100644 --- a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestCommittableSerializerTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestCommittableSerializerTest.java @@ -83,7 +83,7 @@ private static void addFileCommittables( if (!committable.logOffsets().containsKey(bucket)) { int offset = ID.incrementAndGet(); - committable.addLogOffset(bucket, offset); + committable.addLogOffset(bucket, offset, false); assertThat(committable.logOffsets().get(bucket)).isEqualTo(offset); } } @@ -117,6 +117,7 @@ public static DataFileMeta newFile(int name, int level) { level, 0L, null, - FileSource.APPEND); + FileSource.APPEND, + null); } } diff --git a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileMetaTest.java b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileMetaTest.java index c2a7f821dda0..1be5993fb0d0 100644 --- a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileMetaTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileMetaTest.java @@ -18,9 +18,11 @@ package org.apache.paimon.manifest; +import org.apache.paimon.data.BinaryRow; import org.apache.paimon.fs.Path; import org.apache.paimon.fs.local.LocalFileIO; import org.apache.paimon.operation.ManifestFileMerger; +import org.apache.paimon.partition.PartitionPredicate; import org.apache.paimon.types.IntType; import org.apache.paimon.types.RowType; import org.apache.paimon.utils.FailingFileIO; @@ -39,8 +41,12 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.LinkedList; import java.util.List; import java.util.Optional; +import java.util.Set; import java.util.TreeSet; import java.util.UUID; import java.util.concurrent.ThreadLocalRandom; @@ -300,13 +306,13 @@ public void testTriggerFullCompaction() throws Exception { // case5:the sizes of some manifest files are greater than suggestedMetaSize, while the // sizes of other manifest files is less than. All manifest files have no delete files input.clear(); - input.addAll(Arrays.asList(manifest1, manifest2, manifest3)); + input.addAll(Arrays.asList(manifest1, manifest2, manifest3, manifest4)); List newMetas5 = new ArrayList<>(); List fullCompacted5 = ManifestFileMerger.tryFullCompaction( input, newMetas5, manifestFile, 1800, 100, getPartitionType(), null) .get(); - assertThat(fullCompacted5.size()).isEqualTo(2); + assertThat(fullCompacted5.size()).isEqualTo(3); assertThat(newMetas5.size()).isEqualTo(1); // trigger full compaction @@ -446,6 +452,135 @@ public void testIdentifierAfterFullCompaction() throws Exception { containSameIdentifyEntryFile(fullCompacted, entryIdentifierExpected); } + @RepeatedTest(10) + public void testRandomFullCompaction() throws Exception { + List input = new ArrayList<>(); + Set manifestEntrySet = new HashSet<>(); + Set deleteManifestEntrySet = new HashSet<>(); + int inputSize = ThreadLocalRandom.current().nextInt(100) + 1; + int totalEntryNums = 0; + for (int i = 0; i < inputSize; i++) { + int entryNums = ThreadLocalRandom.current().nextInt(100) + 1; + input.add( + generateRandomData( + entryNums, totalEntryNums, manifestEntrySet, deleteManifestEntrySet)); + totalEntryNums += entryNums; + } + int suggerstSize = ThreadLocalRandom.current().nextInt(3000) + 1; + int sizeTrigger = ThreadLocalRandom.current().nextInt(40000) + 1; + List newMetas = new ArrayList<>(); + Optional> fullCompacted = + ManifestFileMerger.tryFullCompaction( + input, + newMetas, + manifestFile, + suggerstSize, + sizeTrigger, + getPartitionType(), + null); + + // *****verify result***** + List mustMergedFiles = + input.stream() + .filter( + manifest -> + manifest.fileSize() < suggerstSize + || manifest.numDeletedFiles() > 0) + .collect(Collectors.toList()); + long mustMergeSize = + mustMergedFiles.stream().map(ManifestFileMeta::fileSize).reduce(0L, Long::sum); + // manifest files which might be merged + List toBeMerged = new LinkedList<>(input); + if (getPartitionType().getFieldCount() > 0) { + Set deletePartitions = + deleteManifestEntrySet.stream() + .map(entry -> entry.partition) + .collect(Collectors.toSet()); + PartitionPredicate predicate = + PartitionPredicate.fromMultiple(getPartitionType(), deletePartitions); + if (predicate != null) { + Iterator iterator = toBeMerged.iterator(); + while (iterator.hasNext()) { + ManifestFileMeta file = iterator.next(); + if (file.fileSize() < suggerstSize || file.numDeletedFiles() > 0) { + continue; + } + if (!predicate.test( + file.numAddedFiles() + file.numDeletedFiles(), + file.partitionStats().minValues(), + file.partitionStats().maxValues(), + file.partitionStats().nullCounts())) { + iterator.remove(); + } + } + } + } + // manifest files which were not written after full compaction + List notMergedFiles = + input.stream() + .filter( + manifest -> + manifest.fileSize() >= suggerstSize + && manifest.numDeletedFiles() == 0) + .filter( + manifest -> + manifestFile.read(manifest.fileName(), manifest.fileSize()) + .stream() + .map(ManifestEntry::identifier) + .noneMatch(deleteManifestEntrySet::contains)) + .collect(Collectors.toList()); + + if (mustMergeSize < sizeTrigger) { + assertThat(fullCompacted).isEmpty(); + assertThat(newMetas).isEmpty(); + } else if (toBeMerged.size() <= 1) { + assertThat(fullCompacted).isEmpty(); + assertThat(newMetas).isEmpty(); + } else { + assertThat(fullCompacted.get().size()).isEqualTo(notMergedFiles.size() + 1); + assertThat(newMetas).size().isEqualTo(1); + } + } + + private ManifestFileMeta generateRandomData( + int entryNums, + int totalEntryNums, + Set manifestEntrySet, + Set deleteManifestEntrySet) { + List entries = new ArrayList<>(); + for (int i = 0; i < entryNums; i++) { + // 70% add, 30% delete + boolean isAdd = ThreadLocalRandom.current().nextInt(10) < 7; + if (manifestEntrySet.isEmpty() || isAdd) { + String fileName = String.format("file-%d", totalEntryNums + i); + Integer partition = ThreadLocalRandom.current().nextInt(10); + int level = ThreadLocalRandom.current().nextInt(6); + List extraFiles = Lists.newArrayList(String.format("index-%s", fileName)); + byte[] embeddedIndex = new byte[] {1, 2, 3}; + entries.add(makeEntry(true, fileName, partition, level, extraFiles, embeddedIndex)); + } else { + FileEntry.Identifier identifier = + manifestEntrySet.stream() + .skip(ThreadLocalRandom.current().nextInt(manifestEntrySet.size())) + .findFirst() + .get(); + entries.add( + makeEntry( + false, + identifier.fileName, + identifier.partition.getInt(0), + identifier.level, + identifier.extraFiles, + new byte[] {1, 2, 3})); + manifestEntrySet.remove(identifier); + deleteManifestEntrySet.add(identifier); + } + } + manifestEntrySet.addAll( + entries.stream().map(ManifestEntry::identifier).collect(Collectors.toSet())); + return makeManifest(entries.toArray(new ManifestEntry[0])); + } + private void createData( int numLastBits, List input, List expected) { // suggested size 500 and suggested count 3 diff --git a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileMetaTestBase.java b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileMetaTestBase.java index a40d0538fdfa..52d82e76be2a 100644 --- a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileMetaTestBase.java +++ b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileMetaTestBase.java @@ -94,7 +94,8 @@ protected ManifestEntry makeEntry( Timestamp.fromEpochMillis(200000), 0L, // not used embeddedIndex, // not used - FileSource.APPEND)); + FileSource.APPEND, + null)); } protected ManifestFileMeta makeManifest(ManifestEntry... entries) { @@ -145,7 +146,11 @@ protected ManifestFile createManifestFile(String pathStr) { "default", CoreOptions.FILE_FORMAT.defaultValue(), CoreOptions.DATA_FILE_PREFIX.defaultValue(), - CoreOptions.CHANGELOG_FILE_PREFIX.defaultValue()), + CoreOptions.CHANGELOG_FILE_PREFIX.defaultValue(), + CoreOptions.PARTITION_GENERATE_LEGCY_NAME.defaultValue(), + CoreOptions.FILE_SUFFIX_INCLUDE_COMPRESSION.defaultValue(), + CoreOptions.FILE_COMPRESSION.defaultValue(), + null), Long.MAX_VALUE, null) .create(); @@ -272,6 +277,7 @@ public static ManifestEntry makeEntry( 0, // not used 0L, null, - FileSource.APPEND)); + FileSource.APPEND, + null)); } } diff --git a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileTest.java b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileTest.java index a11f3e485053..089e11656a99 100644 --- a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileTest.java @@ -102,7 +102,11 @@ private ManifestFile createManifestFile(String pathStr) { "default", CoreOptions.FILE_FORMAT.defaultValue().toString(), CoreOptions.DATA_FILE_PREFIX.defaultValue(), - CoreOptions.CHANGELOG_FILE_PREFIX.defaultValue()); + CoreOptions.CHANGELOG_FILE_PREFIX.defaultValue(), + CoreOptions.PARTITION_GENERATE_LEGCY_NAME.defaultValue(), + CoreOptions.FILE_SUFFIX_INCLUDE_COMPRESSION.defaultValue(), + CoreOptions.FILE_COMPRESSION.defaultValue(), + null); int suggestedFileSize = ThreadLocalRandom.current().nextInt(8192) + 1024; FileIO fileIO = FileIOFinder.find(path); return new ManifestFile.Factory( diff --git a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestListTest.java b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestListTest.java index faf1004e1619..5bf01f32cb07 100644 --- a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestListTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestListTest.java @@ -106,7 +106,11 @@ private ManifestList createManifestList(String pathStr) { "default", CoreOptions.FILE_FORMAT.defaultValue().toString(), CoreOptions.DATA_FILE_PREFIX.defaultValue(), - CoreOptions.CHANGELOG_FILE_PREFIX.defaultValue()); + CoreOptions.CHANGELOG_FILE_PREFIX.defaultValue(), + CoreOptions.PARTITION_GENERATE_LEGCY_NAME.defaultValue(), + CoreOptions.FILE_SUFFIX_INCLUDE_COMPRESSION.defaultValue(), + CoreOptions.FILE_COMPRESSION.defaultValue(), + null); return new ManifestList.Factory(FileIOFinder.find(path), avro, "zstd", pathFactory, null) .create(); } diff --git a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestTestDataGenerator.java b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestTestDataGenerator.java index 5294436aefc9..0283eda853e2 100644 --- a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestTestDataGenerator.java +++ b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestTestDataGenerator.java @@ -113,7 +113,7 @@ public ManifestFileMeta createManifestFileMeta(List entries) { entries.size() * 100L, numAddedFiles, numDeletedFiles, - serializer.toBinary(collector.extract()), + serializer.toBinaryAllMode(collector.extract()), 0); } diff --git a/paimon-core/src/test/java/org/apache/paimon/mergetree/ContainsLevelsTest.java b/paimon-core/src/test/java/org/apache/paimon/mergetree/ContainsLevelsTest.java index 0ab636c33aa3..be49311427a0 100644 --- a/paimon-core/src/test/java/org/apache/paimon/mergetree/ContainsLevelsTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/mergetree/ContainsLevelsTest.java @@ -41,6 +41,7 @@ import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.schema.TableSchema; import org.apache.paimon.table.SchemaEvolutionTableTestBase; +import org.apache.paimon.table.SpecialFields; import org.apache.paimon.types.DataField; import org.apache.paimon.types.DataTypes; import org.apache.paimon.types.RowKind; @@ -77,7 +78,12 @@ public class ContainsLevelsTest { private final Comparator comparator = Comparator.comparingInt(o -> o.getInt(0)); - private final RowType keyType = DataTypes.ROW(DataTypes.FIELD(0, "_key", DataTypes.INT())); + private final RowType keyType = + DataTypes.ROW( + DataTypes.FIELD( + SpecialFields.KEY_FIELD_ID_START, + SpecialFields.KEY_FIELD_PREFIX + "key", + DataTypes.INT())); private final RowType rowType = DataTypes.ROW( DataTypes.FIELD(0, "key", DataTypes.INT()), diff --git a/paimon-core/src/test/java/org/apache/paimon/mergetree/LevelsTest.java b/paimon-core/src/test/java/org/apache/paimon/mergetree/LevelsTest.java index a86aa445b2a6..d804c9790282 100644 --- a/paimon-core/src/test/java/org/apache/paimon/mergetree/LevelsTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/mergetree/LevelsTest.java @@ -83,6 +83,7 @@ public static DataFileMeta newFile(int level) { level, 0L, null, - FileSource.APPEND); + FileSource.APPEND, + null); } } diff --git a/paimon-core/src/test/java/org/apache/paimon/mergetree/LookupLevelsTest.java b/paimon-core/src/test/java/org/apache/paimon/mergetree/LookupLevelsTest.java index 2dce81ce56b4..a678534042eb 100644 --- a/paimon-core/src/test/java/org/apache/paimon/mergetree/LookupLevelsTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/mergetree/LookupLevelsTest.java @@ -41,6 +41,7 @@ import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.schema.TableSchema; import org.apache.paimon.table.SchemaEvolutionTableTestBase; +import org.apache.paimon.table.SpecialFields; import org.apache.paimon.types.DataField; import org.apache.paimon.types.DataTypes; import org.apache.paimon.types.RowKind; @@ -79,7 +80,9 @@ public class LookupLevelsTest { private final Comparator comparator = Comparator.comparingInt(o -> o.getInt(0)); - private final RowType keyType = DataTypes.ROW(DataTypes.FIELD(0, "_key", DataTypes.INT())); + private final RowType keyType = + DataTypes.ROW( + DataTypes.FIELD(SpecialFields.KEY_FIELD_ID_START, "_KEY_key", DataTypes.INT())); private final RowType rowType = DataTypes.ROW( DataTypes.FIELD(0, "key", DataTypes.INT()), diff --git a/paimon-core/src/test/java/org/apache/paimon/mergetree/compact/IntervalPartitionTest.java b/paimon-core/src/test/java/org/apache/paimon/mergetree/compact/IntervalPartitionTest.java index 05fce451a40e..bdee5c5f7507 100644 --- a/paimon-core/src/test/java/org/apache/paimon/mergetree/compact/IntervalPartitionTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/mergetree/compact/IntervalPartitionTest.java @@ -183,7 +183,8 @@ private DataFileMeta makeInterval(int left, int right) { Timestamp.fromEpochMillis(100000), 0L, null, - FileSource.APPEND); + FileSource.APPEND, + null); } private List> toMultiset(List> sections) { diff --git a/paimon-core/src/test/java/org/apache/paimon/mergetree/compact/LookupChangelogMergeFunctionWrapperTest.java b/paimon-core/src/test/java/org/apache/paimon/mergetree/compact/LookupChangelogMergeFunctionWrapperTest.java index c8344c44d5bb..28cb4c099aa5 100644 --- a/paimon-core/src/test/java/org/apache/paimon/mergetree/compact/LookupChangelogMergeFunctionWrapperTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/mergetree/compact/LookupChangelogMergeFunctionWrapperTest.java @@ -26,8 +26,8 @@ import org.apache.paimon.lookup.LookupStrategy; import org.apache.paimon.mergetree.compact.aggregate.AggregateMergeFunction; import org.apache.paimon.mergetree.compact.aggregate.FieldAggregator; -import org.apache.paimon.mergetree.compact.aggregate.FieldLastValueAgg; -import org.apache.paimon.mergetree.compact.aggregate.FieldSumAgg; +import org.apache.paimon.mergetree.compact.aggregate.factory.FieldLastValueAggFactory; +import org.apache.paimon.mergetree.compact.aggregate.factory.FieldSumAggFactory; import org.apache.paimon.types.DataField; import org.apache.paimon.types.DataType; import org.apache.paimon.types.DataTypes; @@ -293,7 +293,8 @@ public void testSum(boolean changelogRowDeduplicate) { row -> row.isNullAt(0) ? null : row.getInt(0) }, new FieldAggregator[] { - new FieldSumAgg(DataTypes.INT()) + new FieldSumAggFactory() + .create(DataTypes.INT(), null, null) }), RowType.of(DataTypes.INT()), RowType.of(DataTypes.INT())), @@ -381,7 +382,8 @@ public void testMergeHighLevelOrder() { row -> row.isNullAt(0) ? null : row.getInt(0) }, new FieldAggregator[] { - new FieldLastValueAgg(DataTypes.INT()) + new FieldLastValueAggFactory() + .create(DataTypes.INT(), null, null) }), RowType.of(DataTypes.INT()), RowType.of(DataTypes.INT())), diff --git a/paimon-core/src/test/java/org/apache/paimon/mergetree/compact/PartialUpdateMergeFunctionTest.java b/paimon-core/src/test/java/org/apache/paimon/mergetree/compact/PartialUpdateMergeFunctionTest.java index a6e1b5f90fe2..93f634944e6e 100644 --- a/paimon-core/src/test/java/org/apache/paimon/mergetree/compact/PartialUpdateMergeFunctionTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/mergetree/compact/PartialUpdateMergeFunctionTest.java @@ -95,6 +95,42 @@ public void testSequenceGroup() { validate(func, 1, null, null, 6, null, null, 6); } + @Test + public void testSequenceGroupPartialDelete() { + Options options = new Options(); + options.set("fields.f3.sequence-group", "f1,f2"); + options.set("fields.f6.sequence-group", "f4,f5"); + options.set("partial-update.remove-record-on-sequence-group", "f6"); + RowType rowType = + RowType.of( + DataTypes.INT(), + DataTypes.INT(), + DataTypes.INT(), + DataTypes.INT(), + DataTypes.INT(), + DataTypes.INT(), + DataTypes.INT()); + MergeFunction func = + PartialUpdateMergeFunction.factory(options, rowType, ImmutableList.of("f0")) + .create(); + func.reset(); + add(func, 1, 1, 1, 1, 1, 1, 1); + add(func, 1, 2, 2, 2, 2, 2, null); + validate(func, 1, 2, 2, 2, 1, 1, 1); + add(func, 1, 3, 3, 1, 3, 3, 3); + validate(func, 1, 2, 2, 2, 3, 3, 3); + + // delete + add(func, RowKind.DELETE, 1, 1, 1, 3, 1, 1, null); + validate(func, 1, null, null, 3, 3, 3, 3); + add(func, RowKind.DELETE, 1, 1, 1, 3, 1, 1, 4); + validate(func, null, null, null, null, null, null, null); + add(func, 1, 4, 4, 4, 5, 5, 5); + validate(func, 1, 4, 4, 4, 5, 5, 5); + add(func, RowKind.DELETE, 1, 1, 1, 6, 1, 1, 6); + validate(func, null, null, null, null, null, null, null); + } + @Test public void testMultiSequenceFields() { Options options = new Options(); diff --git a/paimon-core/src/test/java/org/apache/paimon/mergetree/compact/UniversalCompactionTest.java b/paimon-core/src/test/java/org/apache/paimon/mergetree/compact/UniversalCompactionTest.java index 25d263a93102..793aea7598e9 100644 --- a/paimon-core/src/test/java/org/apache/paimon/mergetree/compact/UniversalCompactionTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/mergetree/compact/UniversalCompactionTest.java @@ -359,6 +359,6 @@ private LevelSortedRun level(int level, long size) { static DataFileMeta file(long size) { return new DataFileMeta( - "", size, 1, null, null, null, null, 0, 0, 0, 0, 0L, null, FileSource.APPEND); + "", size, 1, null, null, null, null, 0, 0, 0, 0, 0L, null, FileSource.APPEND, null); } } diff --git a/paimon-core/src/test/java/org/apache/paimon/mergetree/compact/aggregate/FieldAggregatorTest.java b/paimon-core/src/test/java/org/apache/paimon/mergetree/compact/aggregate/FieldAggregatorTest.java index 2d9a03b74d62..d32098b80f00 100644 --- a/paimon-core/src/test/java/org/apache/paimon/mergetree/compact/aggregate/FieldAggregatorTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/mergetree/compact/aggregate/FieldAggregatorTest.java @@ -27,6 +27,25 @@ import org.apache.paimon.data.InternalArray; import org.apache.paimon.data.InternalMap; import org.apache.paimon.data.InternalRow; +import org.apache.paimon.mergetree.compact.aggregate.factory.FieldAggregatorFactory; +import org.apache.paimon.mergetree.compact.aggregate.factory.FieldBoolAndAggFactory; +import org.apache.paimon.mergetree.compact.aggregate.factory.FieldBoolOrAggFactory; +import org.apache.paimon.mergetree.compact.aggregate.factory.FieldCollectAggFactory; +import org.apache.paimon.mergetree.compact.aggregate.factory.FieldFirstNonNullValueAggFactory; +import org.apache.paimon.mergetree.compact.aggregate.factory.FieldFirstValueAggFactory; +import org.apache.paimon.mergetree.compact.aggregate.factory.FieldHllSketchAggFactory; +import org.apache.paimon.mergetree.compact.aggregate.factory.FieldLastNonNullValueAggFactory; +import org.apache.paimon.mergetree.compact.aggregate.factory.FieldLastValueAggFactory; +import org.apache.paimon.mergetree.compact.aggregate.factory.FieldListaggAggFactory; +import org.apache.paimon.mergetree.compact.aggregate.factory.FieldMaxAggFactory; +import org.apache.paimon.mergetree.compact.aggregate.factory.FieldMergeMapAggFactory; +import org.apache.paimon.mergetree.compact.aggregate.factory.FieldMinAggFactory; +import org.apache.paimon.mergetree.compact.aggregate.factory.FieldNestedUpdateAggFactory; +import org.apache.paimon.mergetree.compact.aggregate.factory.FieldProductAggFactory; +import org.apache.paimon.mergetree.compact.aggregate.factory.FieldRoaringBitmap32AggFactory; +import org.apache.paimon.mergetree.compact.aggregate.factory.FieldRoaringBitmap64AggFactory; +import org.apache.paimon.mergetree.compact.aggregate.factory.FieldSumAggFactory; +import org.apache.paimon.mergetree.compact.aggregate.factory.FieldThetaSketchAggFactory; import org.apache.paimon.types.ArrayType; import org.apache.paimon.types.BigIntType; import org.apache.paimon.types.BooleanType; @@ -67,14 +86,16 @@ public class FieldAggregatorTest { @Test public void testFieldBoolAndAgg() { - FieldBoolAndAgg fieldBoolAndAgg = new FieldBoolAndAgg(new BooleanType()); + FieldBoolAndAgg fieldBoolAndAgg = + new FieldBoolAndAggFactory().create(new BooleanType(), null, null); assertThat(fieldBoolAndAgg.agg(false, true)).isEqualTo(false); assertThat(fieldBoolAndAgg.agg(true, true)).isEqualTo(true); } @Test public void testFieldBoolOrAgg() { - FieldBoolOrAgg fieldBoolOrAgg = new FieldBoolOrAgg(new BooleanType()); + FieldBoolOrAgg fieldBoolOrAgg = + new FieldBoolOrAggFactory().create(new BooleanType(), null, null); assertThat(fieldBoolOrAgg.agg(false, true)).isEqualTo(true); assertThat(fieldBoolOrAgg.agg(false, false)).isEqualTo(false); } @@ -82,7 +103,7 @@ public void testFieldBoolOrAgg() { @Test public void testFieldLastNonNullValueAgg() { FieldLastNonNullValueAgg fieldLastNonNullValueAgg = - new FieldLastNonNullValueAgg(new IntType()); + new FieldLastNonNullValueAggFactory().create(new IntType(), null, null); Integer accumulator = null; Integer inputField = 1; assertThat(fieldLastNonNullValueAgg.agg(accumulator, inputField)).isEqualTo(1); @@ -94,7 +115,8 @@ public void testFieldLastNonNullValueAgg() { @Test public void testFieldLastValueAgg() { - FieldLastValueAgg fieldLastValueAgg = new FieldLastValueAgg(new IntType()); + FieldLastValueAgg fieldLastValueAgg = + new FieldLastValueAggFactory().create(new IntType(), null, null); Integer accumulator = null; Integer inputField = 1; assertThat(fieldLastValueAgg.agg(accumulator, inputField)).isEqualTo(1); @@ -106,7 +128,8 @@ public void testFieldLastValueAgg() { @Test public void testFieldFirstValueAgg() { - FieldFirstValueAgg fieldFirstValueAgg = new FieldFirstValueAgg(new IntType()); + FieldFirstValueAgg fieldFirstValueAgg = + new FieldFirstValueAggFactory().create(new IntType(), null, null); assertThat(fieldFirstValueAgg.agg(null, 1)).isEqualTo(1); assertThat(fieldFirstValueAgg.agg(1, 2)).isEqualTo(1); @@ -117,7 +140,7 @@ public void testFieldFirstValueAgg() { @Test public void testFieldFirstNonNullValueAgg() { FieldFirstNonNullValueAgg fieldFirstNonNullValueAgg = - new FieldFirstNonNullValueAgg(new IntType()); + new FieldFirstNonNullValueAggFactory().create(new IntType(), null, null); assertThat(fieldFirstNonNullValueAgg.agg(null, null)).isNull(); assertThat(fieldFirstNonNullValueAgg.agg(null, 1)).isEqualTo(1); assertThat(fieldFirstNonNullValueAgg.agg(1, 2)).isEqualTo(1); @@ -129,8 +152,8 @@ public void testFieldFirstNonNullValueAgg() { @Test public void testFieldListAggWithDefaultDelimiter() { FieldListaggAgg fieldListaggAgg = - new FieldListaggAgg( - new VarCharType(), new CoreOptions(new HashMap<>()), "fieldName"); + new FieldListaggAggFactory() + .create(new VarCharType(), new CoreOptions(new HashMap<>()), "fieldName"); BinaryString accumulator = BinaryString.fromString("user1"); BinaryString inputField = BinaryString.fromString("user2"); assertThat(fieldListaggAgg.agg(accumulator, inputField).toString()) @@ -140,11 +163,13 @@ public void testFieldListAggWithDefaultDelimiter() { @Test public void testFieldListAggWithCustomDelimiter() { FieldListaggAgg fieldListaggAgg = - new FieldListaggAgg( - new VarCharType(), - CoreOptions.fromMap( - ImmutableMap.of("fields.fieldName.list-agg-delimiter", "-")), - "fieldName"); + new FieldListaggAggFactory() + .create( + new VarCharType(), + CoreOptions.fromMap( + ImmutableMap.of( + "fields.fieldName.list-agg-delimiter", "-")), + "fieldName"); BinaryString accumulator = BinaryString.fromString("user1"); BinaryString inputField = BinaryString.fromString("user2"); assertThat(fieldListaggAgg.agg(accumulator, inputField).toString()) @@ -153,7 +178,7 @@ public void testFieldListAggWithCustomDelimiter() { @Test public void testFieldMaxAgg() { - FieldMaxAgg fieldMaxAgg = new FieldMaxAgg(new IntType()); + FieldMaxAgg fieldMaxAgg = new FieldMaxAggFactory().create(new IntType(), null, null); Integer accumulator = 1; Integer inputField = 10; assertThat(fieldMaxAgg.agg(accumulator, inputField)).isEqualTo(10); @@ -161,7 +186,7 @@ public void testFieldMaxAgg() { @Test public void testFieldMinAgg() { - FieldMinAgg fieldMinAgg = new FieldMinAgg(new IntType()); + FieldMinAgg fieldMinAgg = new FieldMinAggFactory().create(new IntType(), null, null); Integer accumulator = 1; Integer inputField = 10; assertThat(fieldMinAgg.agg(accumulator, inputField)).isEqualTo(1); @@ -169,7 +194,7 @@ public void testFieldMinAgg() { @Test public void testFieldSumIntAgg() { - FieldSumAgg fieldSumAgg = new FieldSumAgg(new IntType()); + FieldSumAgg fieldSumAgg = new FieldSumAggFactory().create(new IntType(), null, null); assertThat(fieldSumAgg.agg(null, 10)).isEqualTo(10); assertThat(fieldSumAgg.agg(1, 10)).isEqualTo(11); assertThat(fieldSumAgg.retract(10, 5)).isEqualTo(5); @@ -178,7 +203,8 @@ public void testFieldSumIntAgg() { @Test public void testFieldProductIntAgg() { - FieldProductAgg fieldProductAgg = new FieldProductAgg(new IntType()); + FieldProductAgg fieldProductAgg = + new FieldProductAggFactory().create(new IntType(), null, null); assertThat(fieldProductAgg.agg(null, 10)).isEqualTo(10); assertThat(fieldProductAgg.agg(1, 10)).isEqualTo(10); assertThat(fieldProductAgg.retract(10, 5)).isEqualTo(2); @@ -187,7 +213,7 @@ public void testFieldProductIntAgg() { @Test public void testFieldSumByteAgg() { - FieldSumAgg fieldSumAgg = new FieldSumAgg(new TinyIntType()); + FieldSumAgg fieldSumAgg = new FieldSumAggFactory().create(new TinyIntType(), null, null); assertThat(fieldSumAgg.agg(null, (byte) 10)).isEqualTo((byte) 10); assertThat(fieldSumAgg.agg((byte) 1, (byte) 10)).isEqualTo((byte) 11); assertThat(fieldSumAgg.retract((byte) 10, (byte) 5)).isEqualTo((byte) 5); @@ -196,7 +222,8 @@ public void testFieldSumByteAgg() { @Test public void testFieldProductByteAgg() { - FieldProductAgg fieldProductAgg = new FieldProductAgg(new TinyIntType()); + FieldProductAgg fieldProductAgg = + new FieldProductAggFactory().create(new TinyIntType(), null, null); assertThat(fieldProductAgg.agg(null, (byte) 10)).isEqualTo((byte) 10); assertThat(fieldProductAgg.agg((byte) 1, (byte) 10)).isEqualTo((byte) 10); assertThat(fieldProductAgg.retract((byte) 10, (byte) 5)).isEqualTo((byte) 2); @@ -205,7 +232,8 @@ public void testFieldProductByteAgg() { @Test public void testFieldProductShortAgg() { - FieldProductAgg fieldProductAgg = new FieldProductAgg(new SmallIntType()); + FieldProductAgg fieldProductAgg = + new FieldProductAggFactory().create(new SmallIntType(), null, null); assertThat(fieldProductAgg.agg(null, (short) 10)).isEqualTo((short) 10); assertThat(fieldProductAgg.agg((short) 1, (short) 10)).isEqualTo((short) 10); assertThat(fieldProductAgg.retract((short) 10, (short) 5)).isEqualTo((short) 2); @@ -214,7 +242,7 @@ public void testFieldProductShortAgg() { @Test public void testFieldSumShortAgg() { - FieldSumAgg fieldSumAgg = new FieldSumAgg(new SmallIntType()); + FieldSumAgg fieldSumAgg = new FieldSumAggFactory().create(new SmallIntType(), null, null); assertThat(fieldSumAgg.agg(null, (short) 10)).isEqualTo((short) 10); assertThat(fieldSumAgg.agg((short) 1, (short) 10)).isEqualTo((short) 11); assertThat(fieldSumAgg.retract((short) 10, (short) 5)).isEqualTo((short) 5); @@ -223,7 +251,7 @@ public void testFieldSumShortAgg() { @Test public void testFieldSumLongAgg() { - FieldSumAgg fieldSumAgg = new FieldSumAgg(new BigIntType()); + FieldSumAgg fieldSumAgg = new FieldSumAggFactory().create(new BigIntType(), null, null); assertThat(fieldSumAgg.agg(null, 10L)).isEqualTo(10L); assertThat(fieldSumAgg.agg(1L, 10L)).isEqualTo(11L); assertThat(fieldSumAgg.retract(10L, 5L)).isEqualTo(5L); @@ -232,7 +260,8 @@ public void testFieldSumLongAgg() { @Test public void testFieldProductLongAgg() { - FieldProductAgg fieldProductAgg = new FieldProductAgg(new BigIntType()); + FieldProductAgg fieldProductAgg = + new FieldProductAggFactory().create(new BigIntType(), null, null); assertThat(fieldProductAgg.agg(null, 10L)).isEqualTo(10L); assertThat(fieldProductAgg.agg(1L, 10L)).isEqualTo(10L); assertThat(fieldProductAgg.retract(10L, 5L)).isEqualTo(2L); @@ -241,7 +270,8 @@ public void testFieldProductLongAgg() { @Test public void testFieldProductFloatAgg() { - FieldProductAgg fieldProductAgg = new FieldProductAgg(new FloatType()); + FieldProductAgg fieldProductAgg = + new FieldProductAggFactory().create(new FloatType(), null, null); assertThat(fieldProductAgg.agg(null, (float) 10)).isEqualTo((float) 10); assertThat(fieldProductAgg.agg((float) 1, (float) 10)).isEqualTo((float) 10); assertThat(fieldProductAgg.retract((float) 10, (float) 5)).isEqualTo((float) 2); @@ -250,7 +280,7 @@ public void testFieldProductFloatAgg() { @Test public void testFieldSumFloatAgg() { - FieldSumAgg fieldSumAgg = new FieldSumAgg(new FloatType()); + FieldSumAgg fieldSumAgg = new FieldSumAggFactory().create(new FloatType(), null, null); assertThat(fieldSumAgg.agg(null, (float) 10)).isEqualTo((float) 10); assertThat(fieldSumAgg.agg((float) 1, (float) 10)).isEqualTo((float) 11); assertThat(fieldSumAgg.retract((float) 10, (float) 5)).isEqualTo((float) 5); @@ -259,7 +289,8 @@ public void testFieldSumFloatAgg() { @Test public void testFieldProductDoubleAgg() { - FieldProductAgg fieldProductAgg = new FieldProductAgg(new DoubleType()); + FieldProductAgg fieldProductAgg = + new FieldProductAggFactory().create(new DoubleType(), null, null); assertThat(fieldProductAgg.agg(null, (double) 10)).isEqualTo((double) 10); assertThat(fieldProductAgg.agg((double) 1, (double) 10)).isEqualTo((double) 10); assertThat(fieldProductAgg.retract((double) 10, (double) 5)).isEqualTo((double) 2); @@ -268,7 +299,7 @@ public void testFieldProductDoubleAgg() { @Test public void testFieldSumDoubleAgg() { - FieldSumAgg fieldSumAgg = new FieldSumAgg(new DoubleType()); + FieldSumAgg fieldSumAgg = new FieldSumAggFactory().create(new DoubleType(), null, null); assertThat(fieldSumAgg.agg(null, (double) 10)).isEqualTo((double) 10); assertThat(fieldSumAgg.agg((double) 1, (double) 10)).isEqualTo((double) 11); assertThat(fieldSumAgg.retract((double) 10, (double) 5)).isEqualTo((double) 5); @@ -277,7 +308,8 @@ public void testFieldSumDoubleAgg() { @Test public void testFieldProductDecimalAgg() { - FieldProductAgg fieldProductAgg = new FieldProductAgg(new DecimalType()); + FieldProductAgg fieldProductAgg = + new FieldProductAggFactory().create(new DecimalType(), null, null); assertThat(fieldProductAgg.agg(null, toDecimal(10))).isEqualTo(toDecimal(10)); assertThat(fieldProductAgg.agg(toDecimal(1), toDecimal(10))).isEqualTo(toDecimal(10)); assertThat(fieldProductAgg.retract(toDecimal(10), toDecimal(5))).isEqualTo(toDecimal(2)); @@ -286,7 +318,7 @@ public void testFieldProductDecimalAgg() { @Test public void testFieldSumDecimalAgg() { - FieldSumAgg fieldSumAgg = new FieldSumAgg(new DecimalType()); + FieldSumAgg fieldSumAgg = new FieldSumAggFactory().create(new DecimalType(), null, null); assertThat(fieldSumAgg.agg(null, toDecimal(10))).isEqualTo(toDecimal(10)); assertThat(fieldSumAgg.agg(toDecimal(1), toDecimal(10))).isEqualTo(toDecimal(11)); assertThat(fieldSumAgg.retract(toDecimal(10), toDecimal(5))).isEqualTo(toDecimal(5)); @@ -307,6 +339,7 @@ public void testFieldNestedUpdateAgg() { DataTypes.FIELD(2, "v", DataTypes.STRING())); FieldNestedUpdateAgg agg = new FieldNestedUpdateAgg( + FieldNestedUpdateAggFactory.NAME, DataTypes.ARRAY( DataTypes.ROW( DataTypes.FIELD(0, "k0", DataTypes.INT()), @@ -347,7 +380,10 @@ public void testFieldNestedAppendAgg() { DataTypes.FIELD(1, "k1", DataTypes.INT()), DataTypes.FIELD(2, "v", DataTypes.STRING())); FieldNestedUpdateAgg agg = - new FieldNestedUpdateAgg(DataTypes.ARRAY(elementRowType), Collections.emptyList()); + new FieldNestedUpdateAgg( + FieldNestedUpdateAggFactory.NAME, + DataTypes.ARRAY(elementRowType), + Collections.emptyList()); InternalArray accumulator = null; InternalArray.ElementGetter elementGetter = @@ -385,7 +421,13 @@ private InternalRow row(int k0, int k1, String v) { @Test public void testFieldCollectAggWithDistinct() { - FieldCollectAgg agg = new FieldCollectAgg(DataTypes.ARRAY(DataTypes.INT()), true); + FieldCollectAgg agg = + new FieldCollectAggFactory() + .create( + DataTypes.ARRAY(DataTypes.INT()), + CoreOptions.fromMap( + ImmutableMap.of("fields.fieldName.distinct", "true")), + "fieldName"); InternalArray result; InternalArray.ElementGetter elementGetter = @@ -407,7 +449,13 @@ public void testFieldCollectAggWithDistinct() { @Test public void testFiledCollectAggWithRowType() { RowType rowType = RowType.of(DataTypes.INT(), DataTypes.STRING()); - FieldCollectAgg agg = new FieldCollectAgg(DataTypes.ARRAY(rowType), true); + FieldCollectAgg agg = + new FieldCollectAggFactory() + .create( + DataTypes.ARRAY(rowType), + CoreOptions.fromMap( + ImmutableMap.of("fields.fieldName.distinct", "true")), + "fieldName"); InternalArray result; InternalArray.ElementGetter elementGetter = InternalArray.createElementGetter(rowType); @@ -438,7 +486,13 @@ public void testFiledCollectAggWithRowType() { @Test public void testFiledCollectAggWithArrayType() { ArrayType arrayType = new ArrayType(DataTypes.INT()); - FieldCollectAgg agg = new FieldCollectAgg(DataTypes.ARRAY(arrayType), true); + FieldCollectAgg agg = + new FieldCollectAggFactory() + .create( + DataTypes.ARRAY(arrayType), + CoreOptions.fromMap( + ImmutableMap.of("fields.fieldName.distinct", "true")), + "fieldName"); InternalArray result; InternalArray.ElementGetter elementGetter = InternalArray.createElementGetter(arrayType); @@ -469,7 +523,13 @@ public void testFiledCollectAggWithArrayType() { @Test public void testFiledCollectAggWithMapType() { MapType mapType = new MapType(DataTypes.INT(), DataTypes.STRING()); - FieldCollectAgg agg = new FieldCollectAgg(DataTypes.ARRAY(mapType), true); + FieldCollectAgg agg = + new FieldCollectAggFactory() + .create( + DataTypes.ARRAY(mapType), + CoreOptions.fromMap( + ImmutableMap.of("fields.fieldName.distinct", "true")), + "fieldName"); InternalArray result; InternalArray.ElementGetter elementGetter = InternalArray.createElementGetter(mapType); @@ -497,7 +557,13 @@ public void testFiledCollectAggWithMapType() { @Test public void testFieldCollectAggWithoutDistinct() { - FieldCollectAgg agg = new FieldCollectAgg(DataTypes.ARRAY(DataTypes.INT()), false); + FieldCollectAgg agg = + new FieldCollectAggFactory() + .create( + DataTypes.ARRAY(DataTypes.INT()), + CoreOptions.fromMap( + ImmutableMap.of("fields.fieldName.distinct", "false")), + "fieldName"); InternalArray result; InternalArray.ElementGetter elementGetter = @@ -522,7 +588,13 @@ public void testFieldCollectAggRetractWithDistinct() { InternalArray.ElementGetter elementGetter; // primitive type - agg = new FieldCollectAgg(DataTypes.ARRAY(DataTypes.INT()), true); + agg = + new FieldCollectAggFactory() + .create( + DataTypes.ARRAY(DataTypes.INT()), + CoreOptions.fromMap( + ImmutableMap.of("fields.fieldName.distinct", "true")), + "fieldName"); elementGetter = InternalArray.createElementGetter(DataTypes.INT()); InternalArray result = (InternalArray) @@ -533,7 +605,13 @@ public void testFieldCollectAggRetractWithDistinct() { // row type RowType rowType = RowType.of(DataTypes.INT(), DataTypes.STRING()); - agg = new FieldCollectAgg(DataTypes.ARRAY(rowType), true); + agg = + new FieldCollectAggFactory() + .create( + DataTypes.ARRAY(rowType), + CoreOptions.fromMap( + ImmutableMap.of("fields.fieldName.distinct", "true")), + "fieldName"); elementGetter = InternalArray.createElementGetter(rowType); Object[] accElements = @@ -556,7 +634,13 @@ public void testFieldCollectAggRetractWithDistinct() { // array type ArrayType arrayType = new ArrayType(DataTypes.INT()); - agg = new FieldCollectAgg(DataTypes.ARRAY(arrayType), true); + agg = + new FieldCollectAggFactory() + .create( + DataTypes.ARRAY(arrayType), + CoreOptions.fromMap( + ImmutableMap.of("fields.fieldName.distinct", "true")), + "fieldName"); elementGetter = InternalArray.createElementGetter(arrayType); accElements = @@ -578,7 +662,13 @@ public void testFieldCollectAggRetractWithDistinct() { // map type MapType mapType = new MapType(DataTypes.INT(), DataTypes.STRING()); - agg = new FieldCollectAgg(DataTypes.ARRAY(mapType), true); + agg = + new FieldCollectAggFactory() + .create( + DataTypes.ARRAY(mapType), + CoreOptions.fromMap( + ImmutableMap.of("fields.fieldName.distinct", "true")), + "fieldName"); elementGetter = InternalArray.createElementGetter(mapType); accElements = @@ -604,7 +694,13 @@ public void testFieldCollectAggRetractWithoutDistinct() { InternalArray.ElementGetter elementGetter; // primitive type - agg = new FieldCollectAgg(DataTypes.ARRAY(DataTypes.INT()), true); + agg = + new FieldCollectAggFactory() + .create( + DataTypes.ARRAY(DataTypes.INT()), + CoreOptions.fromMap( + ImmutableMap.of("fields.fieldName.distinct", "true")), + "fieldName"); elementGetter = InternalArray.createElementGetter(DataTypes.INT()); InternalArray result = (InternalArray) @@ -615,7 +711,13 @@ public void testFieldCollectAggRetractWithoutDistinct() { // row type RowType rowType = RowType.of(DataTypes.INT(), DataTypes.STRING()); - agg = new FieldCollectAgg(DataTypes.ARRAY(rowType), true); + agg = + new FieldCollectAggFactory() + .create( + DataTypes.ARRAY(rowType), + CoreOptions.fromMap( + ImmutableMap.of("fields.fieldName.distinct", "true")), + "fieldName"); elementGetter = InternalArray.createElementGetter(rowType); Object[] accElements = @@ -641,7 +743,13 @@ public void testFieldCollectAggRetractWithoutDistinct() { // array type ArrayType arrayType = new ArrayType(DataTypes.INT()); - agg = new FieldCollectAgg(DataTypes.ARRAY(arrayType), true); + agg = + new FieldCollectAggFactory() + .create( + DataTypes.ARRAY(arrayType), + CoreOptions.fromMap( + ImmutableMap.of("fields.fieldName.distinct", "true")), + "fieldName"); elementGetter = InternalArray.createElementGetter(arrayType); accElements = @@ -666,7 +774,13 @@ public void testFieldCollectAggRetractWithoutDistinct() { // map type MapType mapType = new MapType(DataTypes.INT(), DataTypes.STRING()); - agg = new FieldCollectAgg(DataTypes.ARRAY(mapType), true); + agg = + new FieldCollectAggFactory() + .create( + DataTypes.ARRAY(mapType), + CoreOptions.fromMap( + ImmutableMap.of("fields.fieldName.distinct", "true")), + "fieldName"); elementGetter = InternalArray.createElementGetter(mapType); accElements = @@ -691,7 +805,8 @@ public void testFieldCollectAggRetractWithoutDistinct() { @Test public void testFieldMergeMapAgg() { FieldMergeMapAgg agg = - new FieldMergeMapAgg(DataTypes.MAP(DataTypes.INT(), DataTypes.STRING())); + new FieldMergeMapAggFactory() + .create(DataTypes.MAP(DataTypes.INT(), DataTypes.STRING()), null, null); assertThat(agg.agg(null, null)).isNull(); @@ -710,7 +825,8 @@ public void testFieldMergeMapAgg() { @Test public void testFieldMergeMapAggRetract() { FieldMergeMapAgg agg = - new FieldMergeMapAgg(DataTypes.MAP(DataTypes.INT(), DataTypes.STRING())); + new FieldMergeMapAggFactory() + .create(DataTypes.MAP(DataTypes.INT(), DataTypes.STRING()), null, null); Object result = agg.retract( new GenericMap(toMap(1, "A", 2, "B", 3, "C")), @@ -720,7 +836,8 @@ public void testFieldMergeMapAggRetract() { @Test public void testFieldThetaSketchAgg() { - FieldThetaSketchAgg agg = new FieldThetaSketchAgg(DataTypes.VARBINARY(20)); + FieldThetaSketchAgg agg = + new FieldThetaSketchAggFactory().create(DataTypes.VARBINARY(20), null, null); byte[] inputVal = sketchOf(1); byte[] acc1 = sketchOf(2, 3); @@ -743,7 +860,8 @@ public void testFieldThetaSketchAgg() { @Test public void testFieldHllSketchAgg() { - FieldHllSketchAgg agg = new FieldHllSketchAgg(DataTypes.VARBINARY(20)); + FieldHllSketchAgg agg = + new FieldHllSketchAggFactory().create(DataTypes.VARBINARY(20), null, null); byte[] inputVal = HllSketchUtil.sketchOf(1); byte[] acc1 = HllSketchUtil.sketchOf(2, 3); @@ -766,7 +884,8 @@ public void testFieldHllSketchAgg() { @Test public void testFieldRoaringBitmap32Agg() { - FieldRoaringBitmap32Agg agg = new FieldRoaringBitmap32Agg(DataTypes.VARBINARY(20)); + FieldRoaringBitmap32Agg agg = + new FieldRoaringBitmap32AggFactory().create(DataTypes.VARBINARY(20), null, null); byte[] inputVal = RoaringBitmap32.bitmapOf(1).serialize(); byte[] acc1 = RoaringBitmap32.bitmapOf(2, 3).serialize(); @@ -789,7 +908,8 @@ public void testFieldRoaringBitmap32Agg() { @Test public void testFieldRoaringBitmap64Agg() throws IOException { - FieldRoaringBitmap64Agg agg = new FieldRoaringBitmap64Agg(DataTypes.VARBINARY(20)); + FieldRoaringBitmap64Agg agg = + new FieldRoaringBitmap64AggFactory().create(DataTypes.VARBINARY(20), null, null); byte[] inputVal = RoaringBitmap64.bitmapOf(1L).serialize(); byte[] acc1 = RoaringBitmap64.bitmapOf(2L, 3L).serialize(); @@ -810,6 +930,21 @@ public void testFieldRoaringBitmap64Agg() throws IOException { assertThat(result4).isEqualTo(acc2); } + @Test + public void testCustomAgg() throws IOException { + FieldAggregator fieldAggregator = + FieldAggregatorFactory.create( + DataTypes.STRING(), + "custom", + false, + false, + CoreOptions.fromMap(new HashMap<>()), + "custom"); + + Object agg = fieldAggregator.agg("test", "test"); + assertThat(agg).isEqualTo("test"); + } + private Map toMap(Object... kvs) { Map result = new HashMap<>(); for (int i = 0; i < kvs.length; i += 2) { diff --git a/paimon-core/src/test/java/org/apache/paimon/mergetree/compact/aggregate/TestCustomAgg.java b/paimon-core/src/test/java/org/apache/paimon/mergetree/compact/aggregate/TestCustomAgg.java new file mode 100644 index 000000000000..3550ebe27715 --- /dev/null +++ b/paimon-core/src/test/java/org/apache/paimon/mergetree/compact/aggregate/TestCustomAgg.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.mergetree.compact.aggregate; + +import org.apache.paimon.types.DataType; + +/** Custom FieldAggregator for Test. */ +public class TestCustomAgg extends FieldAggregator { + + public TestCustomAgg(String name, DataType dataType) { + super(name, dataType); + } + + @Override + public Object agg(Object accumulator, Object inputField) { + return inputField; + } +} diff --git a/paimon-core/src/test/java/org/apache/paimon/mergetree/compact/aggregate/TestCustomAggFactory.java b/paimon-core/src/test/java/org/apache/paimon/mergetree/compact/aggregate/TestCustomAggFactory.java new file mode 100644 index 000000000000..7e7715f6d91b --- /dev/null +++ b/paimon-core/src/test/java/org/apache/paimon/mergetree/compact/aggregate/TestCustomAggFactory.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.mergetree.compact.aggregate; + +import org.apache.paimon.CoreOptions; +import org.apache.paimon.mergetree.compact.aggregate.factory.FieldAggregatorFactory; +import org.apache.paimon.types.DataType; + +/** FieldAggregatorFactory for #{@link TestCustomAgg} test. */ +public class TestCustomAggFactory implements FieldAggregatorFactory { + + public static final String NAME = "custom"; + + @Override + public FieldAggregator create(DataType fieldType, CoreOptions options, String field) { + return new TestCustomAgg(identifier(), fieldType); + } + + @Override + public String identifier() { + return NAME; + } +} diff --git a/paimon-core/src/test/java/org/apache/paimon/operation/ExpireSnapshotsTest.java b/paimon-core/src/test/java/org/apache/paimon/operation/ExpireSnapshotsTest.java index 16a2ab56ba03..9dc98343734b 100644 --- a/paimon-core/src/test/java/org/apache/paimon/operation/ExpireSnapshotsTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/operation/ExpireSnapshotsTest.java @@ -29,6 +29,7 @@ import org.apache.paimon.fs.Path; import org.apache.paimon.fs.local.LocalFileIO; import org.apache.paimon.io.DataFileMeta; +import org.apache.paimon.manifest.ExpireFileEntry; import org.apache.paimon.manifest.FileKind; import org.apache.paimon.manifest.FileSource; import org.apache.paimon.manifest.ManifestEntry; @@ -44,6 +45,7 @@ import org.assertj.core.api.Assertions; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.RepeatedTest; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -211,12 +213,15 @@ public void testExpireExtraFiles() throws IOException { Timestamp.now(), 0L, null, - FileSource.APPEND); + FileSource.APPEND, + null); ManifestEntry add = new ManifestEntry(FileKind.ADD, partition, 0, 1, dataFile); ManifestEntry delete = new ManifestEntry(FileKind.DELETE, partition, 0, 1, dataFile); // expire - expire.snapshotDeletion().cleanUnusedDataFile(Arrays.asList(add, delete)); + expire.snapshotDeletion() + .cleanUnusedDataFile( + Arrays.asList(ExpireFileEntry.from(add), ExpireFileEntry.from(delete))); // check assertThat(fileIO.exists(myDataFile)).isFalse(); @@ -450,7 +455,7 @@ public void testExpireWithUpgradedFile() throws Exception { store.assertCleaned(); } - @Test + @RepeatedTest(5) public void testChangelogOutLivedSnapshot() throws Exception { List allData = new ArrayList<>(); List snapshotPositions = new ArrayList<>(); diff --git a/paimon-core/src/test/java/org/apache/paimon/operation/FileDeletionTest.java b/paimon-core/src/test/java/org/apache/paimon/operation/FileDeletionTest.java index 6ae74c669f94..3a5ee93daa37 100644 --- a/paimon-core/src/test/java/org/apache/paimon/operation/FileDeletionTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/operation/FileDeletionTest.java @@ -789,6 +789,7 @@ private void cleanBucket(TestFileStore store, BinaryRow partition, int bucket) { // commit try (FileStoreCommitImpl commit = store.newCommit()) { commit.tryCommitOnce( + null, delete, Collections.emptyList(), Collections.emptyList(), diff --git a/paimon-core/src/test/java/org/apache/paimon/operation/FileStoreCommitTest.java b/paimon-core/src/test/java/org/apache/paimon/operation/FileStoreCommitTest.java index aeccafb85186..9e4ba30eb878 100644 --- a/paimon-core/src/test/java/org/apache/paimon/operation/FileStoreCommitTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/operation/FileStoreCommitTest.java @@ -31,8 +31,12 @@ import org.apache.paimon.fs.local.LocalFileIO; import org.apache.paimon.index.IndexFileHandler; import org.apache.paimon.index.IndexFileMeta; +import org.apache.paimon.manifest.FileKind; import org.apache.paimon.manifest.IndexManifestEntry; import org.apache.paimon.manifest.ManifestCommittable; +import org.apache.paimon.manifest.ManifestEntry; +import org.apache.paimon.manifest.ManifestFile; +import org.apache.paimon.manifest.ManifestFileMeta; import org.apache.paimon.mergetree.compact.DeduplicateMergeFunction; import org.apache.paimon.predicate.PredicateBuilder; import org.apache.paimon.schema.Schema; @@ -79,6 +83,7 @@ import static org.apache.paimon.index.HashIndexFile.HASH_INDEX; import static org.apache.paimon.partition.PartitionPredicate.createPartitionPredicate; +import static org.apache.paimon.stats.SimpleStats.EMPTY_STATS; import static org.apache.paimon.testutils.assertj.PaimonAssertions.anyCauseMatches; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -238,6 +243,8 @@ protected void testRandomConcurrentNoConflict( testRandomConcurrent( dataPerThread, + // overwrite cannot produce changelog + // so only enable it when changelog producer is none changelogProducer == CoreOptions.ChangelogProducer.NONE, failing, changelogProducer); @@ -908,17 +915,126 @@ public void testDVIndexFiles() throws Exception { assertThat(dvs.get("f2").isDeleted(3)).isTrue(); } + @Test + public void testManifestCompact() throws Exception { + TestFileStore store = createStore(false); + + List keyValues = generateDataList(1); + BinaryRow partition = gen.getPartition(keyValues.get(0)); + // commit 1 + Snapshot snapshot1 = + store.commitData(keyValues, s -> partition, kv -> 0, Collections.emptyMap()).get(0); + // commit 2 + Snapshot snapshot2 = + store.overwriteData(keyValues, s -> partition, kv -> 0, Collections.emptyMap()) + .get(0); + // commit 3 + Snapshot snapshot3 = + store.overwriteData(keyValues, s -> partition, kv -> 0, Collections.emptyMap()) + .get(0); + + long deleteNum = + store.manifestListFactory().create().readDataManifests(snapshot3).stream() + .mapToLong(ManifestFileMeta::numDeletedFiles) + .sum(); + assertThat(deleteNum).isGreaterThan(0); + store.newCommit().compactManifest(); + Snapshot latest = store.snapshotManager().latestSnapshot(); + assertThat( + store.manifestListFactory().create().readDataManifests(latest).stream() + .mapToLong(ManifestFileMeta::numDeletedFiles) + .sum()) + .isEqualTo(0); + } + + @Test + public void testDropStatsForOverwrite() throws Exception { + TestFileStore store = createStore(false); + store.options().toConfiguration().set(CoreOptions.MANIFEST_DELETE_FILE_DROP_STATS, true); + + List keyValues = generateDataList(1); + BinaryRow partition = gen.getPartition(keyValues.get(0)); + // commit 1 + Snapshot snapshot1 = + store.commitData(keyValues, s -> partition, kv -> 0, Collections.emptyMap()).get(0); + // overwrite commit 2 + Snapshot snapshot2 = + store.overwriteData(keyValues, s -> partition, kv -> 0, Collections.emptyMap()) + .get(0); + ManifestFile manifestFile = store.manifestFileFactory().create(); + List entries = + store.manifestListFactory().create().readDataManifests(snapshot2).stream() + .flatMap(meta -> manifestFile.read(meta.fileName()).stream()) + .collect(Collectors.toList()); + for (ManifestEntry manifestEntry : entries) { + if (manifestEntry.kind() == FileKind.DELETE) { + assertThat(manifestEntry.file().valueStats()).isEqualTo(EMPTY_STATS); + } + } + } + + @Test + public void testManifestCompactFull() throws Exception { + // Disable full compaction by options. + TestFileStore store = + createStore( + false, + Collections.singletonMap( + CoreOptions.MANIFEST_FULL_COMPACTION_FILE_SIZE.key(), + String.valueOf(Long.MAX_VALUE))); + + List keyValues = generateDataList(1); + BinaryRow partition = gen.getPartition(keyValues.get(0)); + // commit 1 + Snapshot snapshot = + store.commitData(keyValues, s -> partition, kv -> 0, Collections.emptyMap()).get(0); + + for (int i = 0; i < 100; i++) { + snapshot = + store.overwriteData(keyValues, s -> partition, kv -> 0, Collections.emptyMap()) + .get(0); + } + + long deleteNum = + store.manifestListFactory().create().readDataManifests(snapshot).stream() + .mapToLong(ManifestFileMeta::numDeletedFiles) + .sum(); + assertThat(deleteNum).isGreaterThan(0); + store.newCommit().compactManifest(); + Snapshot latest = store.snapshotManager().latestSnapshot(); + assertThat( + store.manifestListFactory().create().readDataManifests(latest).stream() + .mapToLong(ManifestFileMeta::numDeletedFiles) + .sum()) + .isEqualTo(0); + } + + private TestFileStore createStore(boolean failing, Map options) + throws Exception { + return createStore(failing, 1, CoreOptions.ChangelogProducer.NONE, options); + } + private TestFileStore createStore(boolean failing) throws Exception { return createStore(failing, 1); } private TestFileStore createStore(boolean failing, int numBucket) throws Exception { - return createStore(failing, numBucket, CoreOptions.ChangelogProducer.NONE); + return createStore( + failing, numBucket, CoreOptions.ChangelogProducer.NONE, Collections.emptyMap()); } private TestFileStore createStore( boolean failing, int numBucket, CoreOptions.ChangelogProducer changelogProducer) throws Exception { + return createStore(failing, numBucket, changelogProducer, Collections.emptyMap()); + } + + private TestFileStore createStore( + boolean failing, + int numBucket, + CoreOptions.ChangelogProducer changelogProducer, + Map options) + throws Exception { String root = failing ? FailingFileIO.getFailingPath(failingName, tempDir.toString()) @@ -932,7 +1048,7 @@ private TestFileStore createStore( TestKeyValueGenerator.DEFAULT_PART_TYPE.getFieldNames(), TestKeyValueGenerator.getPrimaryKeys( TestKeyValueGenerator.GeneratorMode.MULTI_PARTITIONED), - Collections.emptyMap(), + options, null)); return new TestFileStore.Builder( "avro", diff --git a/paimon-core/src/test/java/org/apache/paimon/operation/KeyValueFileStoreScanTest.java b/paimon-core/src/test/java/org/apache/paimon/operation/KeyValueFileStoreScanTest.java index ce17450538b1..4f3d5c1c24dd 100644 --- a/paimon-core/src/test/java/org/apache/paimon/operation/KeyValueFileStoreScanTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/operation/KeyValueFileStoreScanTest.java @@ -26,8 +26,6 @@ import org.apache.paimon.fs.Path; import org.apache.paimon.fs.local.LocalFileIO; import org.apache.paimon.manifest.ManifestEntry; -import org.apache.paimon.manifest.ManifestFileMeta; -import org.apache.paimon.manifest.ManifestList; import org.apache.paimon.mergetree.compact.DeduplicateMergeFunction; import org.apache.paimon.predicate.PredicateBuilder; import org.apache.paimon.schema.Schema; @@ -50,6 +48,7 @@ import java.util.concurrent.ThreadLocalRandom; import java.util.stream.Collectors; +import static org.apache.paimon.stats.SimpleStats.EMPTY_STATS; import static org.assertj.core.api.Assertions.assertThat; /** Tests for {@link KeyValueFileStoreScan}. */ @@ -252,26 +251,24 @@ public void testWithSnapshot() throws Exception { } @Test - public void testWithManifestList() throws Exception { + public void testDropStatsInPlan() throws Exception { ThreadLocalRandom random = ThreadLocalRandom.current(); - int numCommits = random.nextInt(10) + 1; - for (int i = 0; i < numCommits; i++) { - List data = generateData(random.nextInt(100) + 1); - writeData(data); - } - - ManifestList manifestList = store.manifestListFactory().create(); - long wantedSnapshotId = random.nextLong(snapshotManager.latestSnapshotId()) + 1; - Snapshot wantedSnapshot = snapshotManager.snapshot(wantedSnapshotId); - List wantedManifests = manifestList.readDataManifests(wantedSnapshot); + List data = generateData(100, 0, (long) Math.abs(random.nextInt(1000))); + writeData(data, 0); + data = generateData(100, 1, (long) Math.abs(random.nextInt(1000)) + 1000); + writeData(data, 0); + data = generateData(100, 2, (long) Math.abs(random.nextInt(1000)) + 2000); + writeData(data, 0); + data = generateData(100, 3, (long) Math.abs(random.nextInt(1000)) + 3000); + Snapshot snapshot = writeData(data, 0); - FileStoreScan scan = store.newScan(); - scan.withManifestList(wantedManifests); + KeyValueFileStoreScan scan = store.newScan(); + scan.withSnapshot(snapshot.id()).dropStats(); + List files = scan.plan().files(); - List expectedKvs = store.readKvsFromSnapshot(wantedSnapshotId); - gen.sort(expectedKvs); - Map expected = store.toKvMap(expectedKvs); - runTestExactMatch(scan, null, expected); + for (ManifestEntry manifestEntry : files) { + assertThat(manifestEntry.file().valueStats()).isEqualTo(EMPTY_STATS); + } } private void runTestExactMatch( diff --git a/paimon-core/src/test/java/org/apache/paimon/operation/LocalOrphanFilesCleanTest.java b/paimon-core/src/test/java/org/apache/paimon/operation/LocalOrphanFilesCleanTest.java index fdc68b34abb4..5139dd44957d 100644 --- a/paimon-core/src/test/java/org/apache/paimon/operation/LocalOrphanFilesCleanTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/operation/LocalOrphanFilesCleanTest.java @@ -165,22 +165,20 @@ public void testNormallyRemoving() throws Throwable { // randomly delete tags List deleteTags = Collections.emptyList(); - if (!allTags.isEmpty()) { - deleteTags = randomlyPick(allTags); - for (String tagName : deleteTags) { - table.deleteTag(tagName); - } + deleteTags = randomlyPick(allTags); + for (String tagName : deleteTags) { + table.deleteTag(tagName); } // first check, nothing will be deleted because the default olderThan interval is 1 day LocalOrphanFilesClean orphanFilesClean = new LocalOrphanFilesClean(table); - assertThat(orphanFilesClean.clean().size()).isEqualTo(0); + assertThat(orphanFilesClean.clean().getDeletedFilesPath().size()).isEqualTo(0); // second check orphanFilesClean = new LocalOrphanFilesClean( table, System.currentTimeMillis() + TimeUnit.SECONDS.toMillis(2)); - List deleted = orphanFilesClean.clean(); + List deleted = orphanFilesClean.clean().getDeletedFilesPath(); try { validate(deleted, snapshotData, new HashMap<>()); } catch (Throwable t) { @@ -363,13 +361,13 @@ public void testCleanOrphanFilesWithChangelogDecoupled(String changelogProducer) // first check, nothing will be deleted because the default olderThan interval is 1 day LocalOrphanFilesClean orphanFilesClean = new LocalOrphanFilesClean(table); - assertThat(orphanFilesClean.clean().size()).isEqualTo(0); + assertThat(orphanFilesClean.clean().getDeletedFilesPath().size()).isEqualTo(0); // second check orphanFilesClean = new LocalOrphanFilesClean( table, System.currentTimeMillis() + TimeUnit.SECONDS.toMillis(2)); - List deleted = orphanFilesClean.clean(); + List deleted = orphanFilesClean.clean().getDeletedFilesPath(); validate(deleted, snapshotData, changelogData); } @@ -399,7 +397,7 @@ public void testAbnormallyRemoving() throws Exception { LocalOrphanFilesClean orphanFilesClean = new LocalOrphanFilesClean( table, System.currentTimeMillis() + TimeUnit.SECONDS.toMillis(2)); - assertThat(orphanFilesClean.clean().size()).isGreaterThan(0); + assertThat(orphanFilesClean.clean().getDeletedFilesPath().size()).isGreaterThan(0); } private void writeData( diff --git a/paimon-core/src/test/java/org/apache/paimon/operation/MergeFileSplitReadTest.java b/paimon-core/src/test/java/org/apache/paimon/operation/MergeFileSplitReadTest.java index 46b64422fd9b..59f848a296cf 100644 --- a/paimon-core/src/test/java/org/apache/paimon/operation/MergeFileSplitReadTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/operation/MergeFileSplitReadTest.java @@ -37,6 +37,7 @@ import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.schema.TableSchema; +import org.apache.paimon.table.SpecialFields; import org.apache.paimon.table.source.DataSplit; import org.apache.paimon.types.BigIntType; import org.apache.paimon.types.DataField; @@ -284,7 +285,12 @@ private TestFileStore createStore( ? Collections.emptyList() : Stream.concat( keyType.getFieldNames().stream() - .map(field -> field.replace("key_", "")), + .map( + field -> + field.replace( + SpecialFields + .KEY_FIELD_PREFIX, + "")), partitionType.getFieldNames().stream()) .collect(Collectors.toList()), Collections.emptyMap(), diff --git a/paimon-core/src/test/java/org/apache/paimon/operation/metrics/CompactionMetricsTest.java b/paimon-core/src/test/java/org/apache/paimon/operation/metrics/CompactionMetricsTest.java index 2ce1cc4eabd2..222b99769d06 100644 --- a/paimon-core/src/test/java/org/apache/paimon/operation/metrics/CompactionMetricsTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/operation/metrics/CompactionMetricsTest.java @@ -19,7 +19,9 @@ package org.apache.paimon.operation.metrics; import org.apache.paimon.data.BinaryRow; +import org.apache.paimon.metrics.Counter; import org.apache.paimon.metrics.Gauge; +import org.apache.paimon.metrics.Metric; import org.apache.paimon.metrics.MetricRegistryImpl; import org.junit.jupiter.api.Test; @@ -36,6 +38,8 @@ public void testReportMetrics() { assertThat(getMetric(metrics, CompactionMetrics.AVG_LEVEL0_FILE_COUNT)).isEqualTo(-1.0); assertThat(getMetric(metrics, CompactionMetrics.COMPACTION_THREAD_BUSY)).isEqualTo(0.0); assertThat(getMetric(metrics, CompactionMetrics.AVG_COMPACTION_TIME)).isEqualTo(0.0); + assertThat(getMetric(metrics, CompactionMetrics.COMPACTION_COMPLETED_COUNT)).isEqualTo(0L); + assertThat(getMetric(metrics, CompactionMetrics.COMPACTION_QUEUED_COUNT)).isEqualTo(0L); CompactionMetrics.Reporter[] reporters = new CompactionMetrics.Reporter[3]; for (int i = 0; i < reporters.length; i++) { reporters[i] = metrics.createReporter(BinaryRow.EMPTY_ROW, i); @@ -44,6 +48,8 @@ public void testReportMetrics() { assertThat(getMetric(metrics, CompactionMetrics.MAX_LEVEL0_FILE_COUNT)).isEqualTo(0L); assertThat(getMetric(metrics, CompactionMetrics.AVG_LEVEL0_FILE_COUNT)).isEqualTo(0.0); assertThat(getMetric(metrics, CompactionMetrics.COMPACTION_THREAD_BUSY)).isEqualTo(0.0); + assertThat(getMetric(metrics, CompactionMetrics.COMPACTION_COMPLETED_COUNT)).isEqualTo(0L); + assertThat(getMetric(metrics, CompactionMetrics.COMPACTION_QUEUED_COUNT)).isEqualTo(0L); reporters[0].reportLevel0FileCount(5); reporters[1].reportLevel0FileCount(3); @@ -60,9 +66,28 @@ public void testReportMetrics() { reporters[0].reportCompactionTime(270000); assertThat(getMetric(metrics, CompactionMetrics.AVG_COMPACTION_TIME)) .isEqualTo(273333.3333333333); + + // enqueue compaction request + reporters[0].increaseCompactionsQueuedCount(); + reporters[1].increaseCompactionsQueuedCount(); + reporters[2].increaseCompactionsQueuedCount(); + assertThat(getMetric(metrics, CompactionMetrics.COMPACTION_COMPLETED_COUNT)).isEqualTo(0L); + assertThat(getMetric(metrics, CompactionMetrics.COMPACTION_QUEUED_COUNT)).isEqualTo(3L); + + // completed compactions and remove them from queue + reporters[0].increaseCompactionsCompletedCount(); + reporters[0].decreaseCompactionsQueuedCount(); + reporters[1].decreaseCompactionsQueuedCount(); + assertThat(getMetric(metrics, CompactionMetrics.COMPACTION_COMPLETED_COUNT)).isEqualTo(1L); + assertThat(getMetric(metrics, CompactionMetrics.COMPACTION_QUEUED_COUNT)).isEqualTo(1L); } private Object getMetric(CompactionMetrics metrics, String metricName) { - return ((Gauge) metrics.getMetricGroup().getMetrics().get(metricName)).getValue(); + Metric metric = metrics.getMetricGroup().getMetrics().get(metricName); + if (metric instanceof Gauge) { + return ((Gauge) metric).getValue(); + } else { + return ((Counter) metric).getCount(); + } } } diff --git a/paimon-core/src/test/java/org/apache/paimon/operation/metrics/ScanMetricsTest.java b/paimon-core/src/test/java/org/apache/paimon/operation/metrics/ScanMetricsTest.java index 2b9d0e0cb728..a0427d95cab1 100644 --- a/paimon-core/src/test/java/org/apache/paimon/operation/metrics/ScanMetricsTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/operation/metrics/ScanMetricsTest.java @@ -48,9 +48,7 @@ public void testGenericMetricsRegistration() { ScanMetrics.SCAN_DURATION, ScanMetrics.LAST_SCANNED_MANIFESTS, ScanMetrics.LAST_SCAN_SKIPPED_TABLE_FILES, - ScanMetrics.LAST_SCAN_RESULTED_TABLE_FILES, - ScanMetrics.LAST_SKIPPED_BY_PARTITION_AND_STATS, - ScanMetrics.LAST_SKIPPED_BY_WHOLE_BUCKET_FILES_FILTER); + ScanMetrics.LAST_SCAN_RESULTED_TABLE_FILES); } /** Tests that the metrics are updated properly. */ @@ -66,14 +64,6 @@ public void testMetricsAreUpdated() { (Histogram) registeredGenericMetrics.get(ScanMetrics.SCAN_DURATION); Gauge lastScannedManifests = (Gauge) registeredGenericMetrics.get(ScanMetrics.LAST_SCANNED_MANIFESTS); - Gauge lastSkippedByPartitionAndStats = - (Gauge) - registeredGenericMetrics.get( - ScanMetrics.LAST_SKIPPED_BY_PARTITION_AND_STATS); - Gauge lastSkippedByWholeBucketFilesFilter = - (Gauge) - registeredGenericMetrics.get( - ScanMetrics.LAST_SKIPPED_BY_WHOLE_BUCKET_FILES_FILTER); Gauge lastScanSkippedTableFiles = (Gauge) registeredGenericMetrics.get(ScanMetrics.LAST_SCAN_SKIPPED_TABLE_FILES); @@ -85,8 +75,6 @@ public void testMetricsAreUpdated() { assertThat(scanDuration.getCount()).isEqualTo(0); assertThat(scanDuration.getStatistics().size()).isEqualTo(0); assertThat(lastScannedManifests.getValue()).isEqualTo(0); - assertThat(lastSkippedByPartitionAndStats.getValue()).isEqualTo(0); - assertThat(lastSkippedByWholeBucketFilesFilter.getValue()).isEqualTo(0); assertThat(lastScanSkippedTableFiles.getValue()).isEqualTo(0); assertThat(lastScanResultedTableFiles.getValue()).isEqualTo(0); @@ -104,9 +92,7 @@ public void testMetricsAreUpdated() { assertThat(scanDuration.getStatistics().getMax()).isEqualTo(200); assertThat(scanDuration.getStatistics().getStdDev()).isEqualTo(0); assertThat(lastScannedManifests.getValue()).isEqualTo(20); - assertThat(lastSkippedByPartitionAndStats.getValue()).isEqualTo(25); - assertThat(lastSkippedByWholeBucketFilesFilter.getValue()).isEqualTo(32); - assertThat(lastScanSkippedTableFiles.getValue()).isEqualTo(57); + assertThat(lastScanSkippedTableFiles.getValue()).isEqualTo(25); assertThat(lastScanResultedTableFiles.getValue()).isEqualTo(10); // report again @@ -123,19 +109,17 @@ public void testMetricsAreUpdated() { assertThat(scanDuration.getStatistics().getMax()).isEqualTo(500); assertThat(scanDuration.getStatistics().getStdDev()).isCloseTo(212.132, offset(0.001)); assertThat(lastScannedManifests.getValue()).isEqualTo(22); - assertThat(lastSkippedByPartitionAndStats.getValue()).isEqualTo(30); - assertThat(lastSkippedByWholeBucketFilesFilter.getValue()).isEqualTo(33); - assertThat(lastScanSkippedTableFiles.getValue()).isEqualTo(63); + assertThat(lastScanSkippedTableFiles.getValue()).isEqualTo(30); assertThat(lastScanResultedTableFiles.getValue()).isEqualTo(8); } private void reportOnce(ScanMetrics scanMetrics) { - ScanStats scanStats = new ScanStats(200, 20, 25, 32, 10); + ScanStats scanStats = new ScanStats(200, 20, 25, 10); scanMetrics.reportScan(scanStats); } private void reportAgain(ScanMetrics scanMetrics) { - ScanStats scanStats = new ScanStats(500, 22, 30, 33, 8); + ScanStats scanStats = new ScanStats(500, 22, 30, 8); scanMetrics.reportScan(scanStats); } diff --git a/paimon-core/src/test/java/org/apache/paimon/privilege/PrivilegedCatalogTest.java b/paimon-core/src/test/java/org/apache/paimon/privilege/PrivilegedCatalogTest.java new file mode 100644 index 000000000000..019707108730 --- /dev/null +++ b/paimon-core/src/test/java/org/apache/paimon/privilege/PrivilegedCatalogTest.java @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.privilege; + +import org.apache.paimon.catalog.Catalog; +import org.apache.paimon.catalog.FileSystemCatalogTest; +import org.apache.paimon.catalog.Identifier; +import org.apache.paimon.table.FileStoreTable; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.function.Executable; + +import java.util.OptionalLong; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.junit.jupiter.api.Assertions.assertThrows; + +/** Tests for {@link PrivilegedCatalog}. */ +public class PrivilegedCatalogTest extends FileSystemCatalogTest { + private static final String PASSWORD_ROOT = "123456"; + private static final String USERNAME_TEST_USER = "test_user"; + private static final String PASSWORD_TEST_USER = "test_password"; + + @BeforeEach + public void setUp() throws Exception { + super.setUp(); + getPrivilegeManager("anonymous", "anonymous").initializePrivilege(PASSWORD_ROOT); + catalog = new PrivilegedCatalog(catalog, getPrivilegeManager("root", PASSWORD_ROOT)); + } + + @Override + @Test + public void testGetTable() throws Exception { + super.testGetTable(); + Identifier identifier = Identifier.create("test_db", "test_table"); + + PrivilegedCatalog rootCatalog = ((PrivilegedCatalog) catalog); + rootCatalog.createPrivilegedUser(USERNAME_TEST_USER, PASSWORD_TEST_USER); + Catalog userCatalog = + new PrivilegedCatalog( + rootCatalog.wrapped(), + getPrivilegeManager(USERNAME_TEST_USER, PASSWORD_TEST_USER)); + FileStoreTable dataTable = (FileStoreTable) userCatalog.getTable(identifier); + + assertNoPrivilege(dataTable::snapshotManager); + assertNoPrivilege(dataTable::latestSnapshotId); + assertNoPrivilege(() -> dataTable.snapshot(0)); + + rootCatalog.grantPrivilegeOnTable(USERNAME_TEST_USER, identifier, PrivilegeType.SELECT); + userCatalog = + new PrivilegedCatalog( + rootCatalog.wrapped(), + getPrivilegeManager(USERNAME_TEST_USER, PASSWORD_TEST_USER)); + FileStoreTable dataTable2 = (FileStoreTable) userCatalog.getTable(identifier); + + assertThat(dataTable2.snapshotManager().latestSnapshotId()).isNull(); + assertThat(dataTable2.latestSnapshotId()).isEqualTo(OptionalLong.empty()); + assertThatThrownBy(() -> dataTable2.snapshot(0)).isNotNull(); + } + + private FileBasedPrivilegeManager getPrivilegeManager(String user, String password) { + return new FileBasedPrivilegeManager(warehouse, fileIO, user, password); + } + + private void assertNoPrivilege(Executable executable) { + Exception e = assertThrows(Exception.class, executable); + if (e.getCause() != null) { + assertThat(e).hasRootCauseInstanceOf(NoPrivilegeException.class); + } else { + assertThat(e).isInstanceOf(NoPrivilegeException.class); + } + } +} diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/DefaultErrorHandlerTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/DefaultErrorHandlerTest.java new file mode 100644 index 000000000000..340e38f6a7f8 --- /dev/null +++ b/paimon-core/src/test/java/org/apache/paimon/rest/DefaultErrorHandlerTest.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest; + +import org.apache.paimon.rest.exceptions.AlreadyExistsException; +import org.apache.paimon.rest.exceptions.BadRequestException; +import org.apache.paimon.rest.exceptions.ForbiddenException; +import org.apache.paimon.rest.exceptions.NoSuchResourceException; +import org.apache.paimon.rest.exceptions.NotAuthorizedException; +import org.apache.paimon.rest.exceptions.RESTException; +import org.apache.paimon.rest.exceptions.ServiceFailureException; +import org.apache.paimon.rest.exceptions.ServiceUnavailableException; +import org.apache.paimon.rest.responses.ErrorResponse; + +import org.junit.Before; +import org.junit.Test; + +import java.io.IOException; +import java.util.ArrayList; + +import static org.junit.Assert.assertThrows; + +/** Test for {@link DefaultErrorHandler}. */ +public class DefaultErrorHandlerTest { + private ErrorHandler defaultErrorHandler; + + @Before + public void setUp() throws IOException { + defaultErrorHandler = DefaultErrorHandler.getInstance(); + } + + @Test + public void testHandleErrorResponse() { + assertThrows( + BadRequestException.class, + () -> defaultErrorHandler.accept(generateErrorResponse(400))); + assertThrows( + NotAuthorizedException.class, + () -> defaultErrorHandler.accept(generateErrorResponse(401))); + assertThrows( + ForbiddenException.class, + () -> defaultErrorHandler.accept(generateErrorResponse(403))); + assertThrows( + NoSuchResourceException.class, + () -> defaultErrorHandler.accept(generateErrorResponse(404))); + assertThrows( + RESTException.class, () -> defaultErrorHandler.accept(generateErrorResponse(405))); + assertThrows( + RESTException.class, () -> defaultErrorHandler.accept(generateErrorResponse(406))); + assertThrows( + AlreadyExistsException.class, + () -> defaultErrorHandler.accept(generateErrorResponse(409))); + assertThrows( + ServiceFailureException.class, + () -> defaultErrorHandler.accept(generateErrorResponse(500))); + assertThrows( + UnsupportedOperationException.class, + () -> defaultErrorHandler.accept(generateErrorResponse(501))); + assertThrows( + RESTException.class, () -> defaultErrorHandler.accept(generateErrorResponse(502))); + assertThrows( + ServiceUnavailableException.class, + () -> defaultErrorHandler.accept(generateErrorResponse(503))); + } + + private ErrorResponse generateErrorResponse(int code) { + return new ErrorResponse("message", code, new ArrayList()); + } +} diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/HttpClientTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/HttpClientTest.java new file mode 100644 index 000000000000..a3b06b8ce3a9 --- /dev/null +++ b/paimon-core/src/test/java/org/apache/paimon/rest/HttpClientTest.java @@ -0,0 +1,145 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest; + +import org.apache.paimon.rest.auth.BearTokenCredentialsProvider; +import org.apache.paimon.rest.auth.CredentialsProvider; + +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.databind.ObjectMapper; + +import okhttp3.mockwebserver.MockResponse; +import okhttp3.mockwebserver.MockWebServer; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.io.IOException; +import java.time.Duration; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; + +import static org.junit.Assert.assertEquals; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +/** Test for {@link HttpClient}. */ +public class HttpClientTest { + + private MockWebServer mockWebServer; + private HttpClient httpClient; + private ObjectMapper objectMapper = RESTObjectMapper.create(); + private ErrorHandler errorHandler; + private MockRESTData mockResponseData; + private String mockResponseDataStr; + private Map headers; + private static final String MOCK_PATH = "/v1/api/mock"; + private static final String TOKEN = "token"; + + @Before + public void setUp() throws IOException { + mockWebServer = new MockWebServer(); + mockWebServer.start(); + String baseUrl = mockWebServer.url("").toString(); + errorHandler = mock(ErrorHandler.class); + HttpClientOptions httpClientOptions = + new HttpClientOptions( + baseUrl, + Optional.of(Duration.ofSeconds(3)), + Optional.of(Duration.ofSeconds(3)), + objectMapper, + 1, + errorHandler); + mockResponseData = new MockRESTData(MOCK_PATH); + mockResponseDataStr = objectMapper.writeValueAsString(mockResponseData); + httpClient = new HttpClient(httpClientOptions); + CredentialsProvider credentialsProvider = new BearTokenCredentialsProvider(TOKEN); + headers = credentialsProvider.authHeader(); + } + + @After + public void tearDown() throws IOException { + mockWebServer.shutdown(); + } + + @Test + public void testGetSuccess() { + mockHttpCallWithCode(mockResponseDataStr, 200); + MockRESTData response = httpClient.get(MOCK_PATH, MockRESTData.class, headers); + verify(errorHandler, times(0)).accept(any()); + assertEquals(mockResponseData.data(), response.data()); + } + + @Test + public void testGetFail() { + mockHttpCallWithCode(mockResponseDataStr, 400); + httpClient.get(MOCK_PATH, MockRESTData.class, headers); + verify(errorHandler, times(1)).accept(any()); + } + + @Test + public void testPostSuccess() { + mockHttpCallWithCode(mockResponseDataStr, 200); + MockRESTData response = + httpClient.post(MOCK_PATH, mockResponseData, MockRESTData.class, headers); + verify(errorHandler, times(0)).accept(any()); + assertEquals(mockResponseData.data(), response.data()); + } + + @Test + public void testPostFail() { + mockHttpCallWithCode(mockResponseDataStr, 400); + httpClient.post(MOCK_PATH, mockResponseData, MockRESTData.class, headers); + verify(errorHandler, times(1)).accept(any()); + } + + @Test + public void testDeleteSuccess() { + mockHttpCallWithCode(mockResponseDataStr, 200); + MockRESTData response = httpClient.delete(MOCK_PATH, headers); + verify(errorHandler, times(0)).accept(any()); + } + + @Test + public void testDeleteFail() { + mockHttpCallWithCode(mockResponseDataStr, 400); + httpClient.delete(MOCK_PATH, headers); + verify(errorHandler, times(1)).accept(any()); + } + + private Map headers(String token) { + Map header = new HashMap<>(); + header.put("Authorization", "Bearer " + token); + return header; + } + + private void mockHttpCallWithCode(String body, Integer code) { + MockResponse mockResponseObj = generateMockResponse(body, code); + mockWebServer.enqueue(mockResponseObj); + } + + private MockResponse generateMockResponse(String data, Integer code) { + return new MockResponse() + .setResponseCode(code) + .setBody(data) + .addHeader("Content-Type", "application/json"); + } +} diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTData.java b/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTData.java new file mode 100644 index 000000000000..9b7f1003e76f --- /dev/null +++ b/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTData.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest; + +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonGetter; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty; + +import java.beans.ConstructorProperties; + +/** Mock REST data. */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class MockRESTData implements RESTRequest, RESTResponse { + private static final String FIELD_DATA = "data"; + + @JsonProperty(FIELD_DATA) + private String data; + + @ConstructorProperties({FIELD_DATA}) + public MockRESTData(String data) { + this.data = data; + } + + @JsonGetter(FIELD_DATA) + public String data() { + return data; + } +} diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTMessage.java b/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTMessage.java new file mode 100644 index 000000000000..a605e5e77c2a --- /dev/null +++ b/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTMessage.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest; + +import org.apache.paimon.rest.requests.CreateDatabaseRequest; +import org.apache.paimon.rest.responses.CreateDatabaseResponse; +import org.apache.paimon.rest.responses.DatabaseName; +import org.apache.paimon.rest.responses.ErrorResponse; +import org.apache.paimon.rest.responses.GetDatabaseResponse; +import org.apache.paimon.rest.responses.ListDatabasesResponse; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.apache.paimon.rest.RESTCatalogInternalOptions.DATABASE_COMMENT; + +/** Mock REST message. */ +public class MockRESTMessage { + + public static String databaseName() { + return "database"; + } + + public static CreateDatabaseRequest createDatabaseRequest(String name) { + boolean ignoreIfExists = true; + Map options = new HashMap<>(); + options.put("a", "b"); + return new CreateDatabaseRequest(name, ignoreIfExists, options); + } + + public static CreateDatabaseResponse createDatabaseResponse(String name) { + Map options = new HashMap<>(); + options.put("a", "b"); + return new CreateDatabaseResponse(name, options); + } + + public static GetDatabaseResponse getDatabaseResponse(String name) { + Map options = new HashMap<>(); + options.put("a", "b"); + options.put(DATABASE_COMMENT.key(), "comment"); + return new GetDatabaseResponse(name, options); + } + + public static ListDatabasesResponse listDatabasesResponse(String name) { + DatabaseName databaseName = new DatabaseName(name); + List databaseNameList = new ArrayList<>(); + databaseNameList.add(databaseName); + return new ListDatabasesResponse(databaseNameList); + } + + public static ErrorResponse noSuchResourceExceptionErrorResponse() { + return new ErrorResponse("message", 404, new ArrayList<>()); + } +} diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java new file mode 100644 index 000000000000..0fff81afdcde --- /dev/null +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java @@ -0,0 +1,196 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest; + +import org.apache.paimon.catalog.Catalog; +import org.apache.paimon.catalog.Database; +import org.apache.paimon.options.CatalogOptions; +import org.apache.paimon.options.Options; +import org.apache.paimon.rest.responses.CreateDatabaseResponse; +import org.apache.paimon.rest.responses.ErrorResponse; +import org.apache.paimon.rest.responses.GetDatabaseResponse; +import org.apache.paimon.rest.responses.ListDatabasesResponse; + +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.core.JsonProcessingException; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.databind.ObjectMapper; + +import okhttp3.mockwebserver.MockResponse; +import okhttp3.mockwebserver.MockWebServer; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** Test for REST Catalog. */ +public class RESTCatalogTest { + + private final ObjectMapper mapper = RESTObjectMapper.create(); + private MockWebServer mockWebServer; + private RESTCatalog restCatalog; + private RESTCatalog mockRestCatalog; + + @Before + public void setUp() throws IOException { + mockWebServer = new MockWebServer(); + mockWebServer.start(); + String baseUrl = mockWebServer.url("").toString(); + Options options = new Options(); + options.set(RESTCatalogOptions.URI, baseUrl); + String initToken = "init_token"; + options.set(RESTCatalogOptions.TOKEN, initToken); + options.set(RESTCatalogOptions.THREAD_POOL_SIZE, 1); + String mockResponse = + String.format( + "{\"defaults\": {\"%s\": \"%s\"}}", + RESTCatalogInternalOptions.PREFIX.key(), "prefix"); + mockResponse(mockResponse, 200); + restCatalog = new RESTCatalog(options); + mockRestCatalog = spy(restCatalog); + } + + @After + public void tearDown() throws IOException { + mockWebServer.shutdown(); + } + + @Test + public void testInitFailWhenDefineWarehouse() { + Options options = new Options(); + options.set(CatalogOptions.WAREHOUSE, "/a/b/c"); + assertThrows(IllegalArgumentException.class, () -> new RESTCatalog(options)); + } + + @Test + public void testGetConfig() { + String key = "a"; + String value = "b"; + String mockResponse = String.format("{\"defaults\": {\"%s\": \"%s\"}}", key, value); + mockResponse(mockResponse, 200); + Map header = new HashMap<>(); + Map response = restCatalog.fetchOptionsFromServer(header, new HashMap<>()); + assertEquals(value, response.get(key)); + } + + @Test + public void testListDatabases() throws JsonProcessingException { + String name = MockRESTMessage.databaseName(); + ListDatabasesResponse response = MockRESTMessage.listDatabasesResponse(name); + mockResponse(mapper.writeValueAsString(response), 200); + List result = restCatalog.listDatabases(); + assertEquals(response.getDatabases().size(), result.size()); + assertEquals(name, result.get(0)); + } + + @Test + public void testCreateDatabase() throws Exception { + String name = MockRESTMessage.databaseName(); + CreateDatabaseResponse response = MockRESTMessage.createDatabaseResponse(name); + mockResponse(mapper.writeValueAsString(response), 200); + assertDoesNotThrow(() -> restCatalog.createDatabase(name, false, response.getOptions())); + } + + @Test + public void testGetDatabase() throws Exception { + String name = MockRESTMessage.databaseName(); + GetDatabaseResponse response = MockRESTMessage.getDatabaseResponse(name); + mockResponse(mapper.writeValueAsString(response), 200); + Database result = restCatalog.getDatabase(name); + assertEquals(name, result.name()); + assertEquals(response.getOptions().size(), result.options().size()); + assertEquals(response.comment().get(), result.comment().get()); + } + + @Test + public void testDropDatabase() throws Exception { + String name = MockRESTMessage.databaseName(); + mockResponse("", 200); + assertDoesNotThrow(() -> mockRestCatalog.dropDatabase(name, false, true)); + verify(mockRestCatalog, times(1)).dropDatabase(eq(name), eq(false), eq(true)); + verify(mockRestCatalog, times(0)).listTables(eq(name)); + } + + @Test + public void testDropDatabaseWhenNoExistAndIgnoreIfNotExistsIsFalse() throws Exception { + String name = MockRESTMessage.databaseName(); + ErrorResponse response = MockRESTMessage.noSuchResourceExceptionErrorResponse(); + mockResponse(mapper.writeValueAsString(response), 404); + assertThrows( + Catalog.DatabaseNotExistException.class, + () -> mockRestCatalog.dropDatabase(name, false, true)); + } + + @Test + public void testDropDatabaseWhenNoExistAndIgnoreIfNotExistsIsTrue() throws Exception { + String name = MockRESTMessage.databaseName(); + ErrorResponse response = MockRESTMessage.noSuchResourceExceptionErrorResponse(); + mockResponse(mapper.writeValueAsString(response), 404); + assertDoesNotThrow(() -> mockRestCatalog.dropDatabase(name, true, true)); + verify(mockRestCatalog, times(1)).dropDatabase(eq(name), eq(true), eq(true)); + verify(mockRestCatalog, times(0)).listTables(eq(name)); + } + + @Test + public void testDropDatabaseWhenCascadeIsFalseAndNoTables() throws Exception { + String name = MockRESTMessage.databaseName(); + boolean cascade = false; + mockResponse("", 200); + when(mockRestCatalog.listTables(name)).thenReturn(new ArrayList<>()); + assertDoesNotThrow(() -> mockRestCatalog.dropDatabase(name, false, cascade)); + verify(mockRestCatalog, times(1)).dropDatabase(eq(name), eq(false), eq(cascade)); + verify(mockRestCatalog, times(1)).listTables(eq(name)); + } + + @Test + public void testDropDatabaseWhenCascadeIsFalseAndTablesExist() throws Exception { + String name = MockRESTMessage.databaseName(); + boolean cascade = false; + mockResponse("", 200); + List tables = new ArrayList<>(); + tables.add("t1"); + when(mockRestCatalog.listTables(name)).thenReturn(tables); + assertThrows( + Catalog.DatabaseNotEmptyException.class, + () -> mockRestCatalog.dropDatabase(name, false, cascade)); + verify(mockRestCatalog, times(1)).dropDatabase(eq(name), eq(false), eq(cascade)); + verify(mockRestCatalog, times(1)).listTables(eq(name)); + } + + private void mockResponse(String mockResponse, int httpCode) { + MockResponse mockResponseObj = + new MockResponse() + .setResponseCode(httpCode) + .setBody(mockResponse) + .addHeader("Content-Type", "application/json"); + mockWebServer.enqueue(mockResponseObj); + } +} diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTObjectMapperTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTObjectMapperTest.java new file mode 100644 index 000000000000..7fee81ef1024 --- /dev/null +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTObjectMapperTest.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest; + +import org.apache.paimon.rest.requests.CreateDatabaseRequest; +import org.apache.paimon.rest.responses.ConfigResponse; +import org.apache.paimon.rest.responses.CreateDatabaseResponse; +import org.apache.paimon.rest.responses.ErrorResponse; +import org.apache.paimon.rest.responses.GetDatabaseResponse; +import org.apache.paimon.rest.responses.ListDatabasesResponse; + +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.databind.ObjectMapper; + +import org.junit.Test; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Map; + +import static org.junit.Assert.assertEquals; + +/** Test for {@link RESTObjectMapper}. */ +public class RESTObjectMapperTest { + private ObjectMapper mapper = RESTObjectMapper.create(); + + @Test + public void configResponseParseTest() throws Exception { + String confKey = "a"; + Map conf = new HashMap<>(); + conf.put(confKey, "b"); + ConfigResponse response = new ConfigResponse(conf, conf); + String responseStr = mapper.writeValueAsString(response); + ConfigResponse parseData = mapper.readValue(responseStr, ConfigResponse.class); + assertEquals(conf.get(confKey), parseData.getDefaults().get(confKey)); + } + + @Test + public void errorResponseParseTest() throws Exception { + String message = "message"; + Integer code = 400; + ErrorResponse response = new ErrorResponse(message, code, new ArrayList()); + String responseStr = mapper.writeValueAsString(response); + ErrorResponse parseData = mapper.readValue(responseStr, ErrorResponse.class); + assertEquals(message, parseData.getMessage()); + assertEquals(code, parseData.getCode()); + } + + @Test + public void createDatabaseRequestParseTest() throws Exception { + String name = MockRESTMessage.databaseName(); + CreateDatabaseRequest request = MockRESTMessage.createDatabaseRequest(name); + String requestStr = mapper.writeValueAsString(request); + CreateDatabaseRequest parseData = mapper.readValue(requestStr, CreateDatabaseRequest.class); + assertEquals(request.getName(), parseData.getName()); + assertEquals(request.getIgnoreIfExists(), parseData.getIgnoreIfExists()); + assertEquals(request.getOptions().size(), parseData.getOptions().size()); + } + + @Test + public void createDatabaseResponseParseTest() throws Exception { + String name = MockRESTMessage.databaseName(); + CreateDatabaseResponse response = MockRESTMessage.createDatabaseResponse(name); + String responseStr = mapper.writeValueAsString(response); + CreateDatabaseResponse parseData = + mapper.readValue(responseStr, CreateDatabaseResponse.class); + assertEquals(name, parseData.getName()); + assertEquals(response.getOptions().size(), parseData.getOptions().size()); + } + + @Test + public void getDatabaseResponseParseTest() throws Exception { + String name = MockRESTMessage.databaseName(); + GetDatabaseResponse response = MockRESTMessage.getDatabaseResponse(name); + String responseStr = mapper.writeValueAsString(response); + GetDatabaseResponse parseData = mapper.readValue(responseStr, GetDatabaseResponse.class); + assertEquals(name, parseData.getName()); + assertEquals(response.getOptions().size(), parseData.getOptions().size()); + assertEquals(response.comment().get(), parseData.comment().get()); + } + + @Test + public void listDatabaseResponseParseTest() throws Exception { + String name = MockRESTMessage.databaseName(); + ListDatabasesResponse response = MockRESTMessage.listDatabasesResponse(name); + String responseStr = mapper.writeValueAsString(response); + ListDatabasesResponse parseData = + mapper.readValue(responseStr, ListDatabasesResponse.class); + assertEquals(response.getDatabases().size(), parseData.getDatabases().size()); + assertEquals(name, parseData.getDatabases().get(0).getName()); + } +} diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/auth/AuthSessionTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/auth/AuthSessionTest.java new file mode 100644 index 000000000000..1f4a48fd5e8c --- /dev/null +++ b/paimon-core/src/test/java/org/apache/paimon/rest/auth/AuthSessionTest.java @@ -0,0 +1,148 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest.auth; + +import org.apache.paimon.utils.Pair; +import org.apache.paimon.utils.ThreadPoolUtils; + +import org.apache.commons.io.FileUtils; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.mockito.Mockito; + +import java.io.File; +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.ScheduledExecutorService; + +import static org.apache.paimon.rest.auth.AuthSession.MAX_REFRESH_WINDOW_MILLIS; +import static org.apache.paimon.rest.auth.AuthSession.MIN_REFRESH_WAIT_MILLIS; +import static org.apache.paimon.rest.auth.AuthSession.TOKEN_REFRESH_NUM_RETRIES; +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** Test for {@link AuthSession}. */ +public class AuthSessionTest { + + @Rule public TemporaryFolder folder = new TemporaryFolder(); + + @Test + public void testRefreshBearTokenFileCredentialsProvider() + throws IOException, InterruptedException { + String fileName = "token"; + Pair tokenFile2Token = generateTokenAndWriteToFile(fileName); + String token = tokenFile2Token.getRight(); + File tokenFile = tokenFile2Token.getLeft(); + Map initialHeaders = new HashMap<>(); + long expiresInMillis = 1000L; + CredentialsProvider credentialsProvider = + new BearTokenFileCredentialsProvider(tokenFile.getPath(), expiresInMillis); + ScheduledExecutorService executor = + ThreadPoolUtils.createScheduledThreadPool(1, "refresh-token"); + AuthSession session = + AuthSession.fromRefreshCredentialsProvider( + executor, initialHeaders, credentialsProvider); + Map header = session.getHeaders(); + assertEquals(header.get("Authorization"), "Bearer " + token); + tokenFile.delete(); + tokenFile2Token = generateTokenAndWriteToFile(fileName); + token = tokenFile2Token.getRight(); + Thread.sleep(expiresInMillis + 500L); + header = session.getHeaders(); + assertEquals(header.get("Authorization"), "Bearer " + token); + } + + @Test + public void testRefreshCredentialsProviderIsSoonExpire() + throws IOException, InterruptedException { + String fileName = "token"; + Pair tokenFile2Token = generateTokenAndWriteToFile(fileName); + String token = tokenFile2Token.getRight(); + File tokenFile = tokenFile2Token.getLeft(); + Map initialHeaders = new HashMap<>(); + long expiresInMillis = 1000L; + CredentialsProvider credentialsProvider = + new BearTokenFileCredentialsProvider(tokenFile.getPath(), expiresInMillis); + AuthSession session = + AuthSession.fromRefreshCredentialsProvider( + null, initialHeaders, credentialsProvider); + Map header = session.getHeaders(); + assertEquals(header.get("Authorization"), "Bearer " + token); + tokenFile.delete(); + tokenFile2Token = generateTokenAndWriteToFile(fileName); + token = tokenFile2Token.getRight(); + tokenFile = tokenFile2Token.getLeft(); + FileUtils.writeStringToFile(tokenFile, token); + Thread.sleep( + (long) (expiresInMillis * (1 - BearTokenFileCredentialsProvider.EXPIRED_FACTOR)) + + 10L); + header = session.getHeaders(); + assertEquals(header.get("Authorization"), "Bearer " + token); + } + + @Test + public void testRetryWhenRefreshFail() throws Exception { + Map initialHeaders = new HashMap<>(); + CredentialsProvider credentialsProvider = + Mockito.mock(BearTokenFileCredentialsProvider.class); + long expiresAtMillis = System.currentTimeMillis() - 1000L; + when(credentialsProvider.expiresAtMillis()).thenReturn(Optional.of(expiresAtMillis)); + when(credentialsProvider.expiresInMills()).thenReturn(Optional.of(50L)); + when(credentialsProvider.supportRefresh()).thenReturn(true); + when(credentialsProvider.keepRefreshed()).thenReturn(true); + when(credentialsProvider.refresh()).thenReturn(false); + AuthSession session = + AuthSession.fromRefreshCredentialsProvider( + null, initialHeaders, credentialsProvider); + AuthSession.scheduleTokenRefresh( + ThreadPoolUtils.createScheduledThreadPool(1, "refresh-token"), + session, + expiresAtMillis); + Thread.sleep(10_000L); + verify(credentialsProvider, Mockito.times(TOKEN_REFRESH_NUM_RETRIES + 1)).refresh(); + } + + @Test + public void testGetTimeToWaitByExpiresInMills() { + long expiresInMillis = -100L; + long timeToWait = AuthSession.getTimeToWaitByExpiresInMills(expiresInMillis); + assertEquals(MIN_REFRESH_WAIT_MILLIS, timeToWait); + expiresInMillis = (long) (MAX_REFRESH_WINDOW_MILLIS * 0.5); + timeToWait = AuthSession.getTimeToWaitByExpiresInMills(expiresInMillis); + assertEquals(MIN_REFRESH_WAIT_MILLIS, timeToWait); + expiresInMillis = MAX_REFRESH_WINDOW_MILLIS; + timeToWait = AuthSession.getTimeToWaitByExpiresInMills(expiresInMillis); + assertEquals(timeToWait, MIN_REFRESH_WAIT_MILLIS); + expiresInMillis = MAX_REFRESH_WINDOW_MILLIS * 2L; + timeToWait = AuthSession.getTimeToWaitByExpiresInMills(expiresInMillis); + assertEquals(timeToWait, MAX_REFRESH_WINDOW_MILLIS); + } + + private Pair generateTokenAndWriteToFile(String fileName) throws IOException { + File tokenFile = folder.newFile(fileName); + String token = UUID.randomUUID().toString(); + FileUtils.writeStringToFile(tokenFile, token); + return Pair.of(tokenFile, token); + } +} diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/auth/CredentialsProviderFactoryTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/auth/CredentialsProviderFactoryTest.java new file mode 100644 index 000000000000..e62a65a79aed --- /dev/null +++ b/paimon-core/src/test/java/org/apache/paimon/rest/auth/CredentialsProviderFactoryTest.java @@ -0,0 +1,131 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest.auth; + +import org.apache.paimon.options.Options; +import org.apache.paimon.rest.RESTCatalogOptions; + +import org.apache.commons.io.FileUtils; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.File; +import java.time.Duration; +import java.util.UUID; + +import static org.apache.paimon.rest.RESTCatalogInternalOptions.CREDENTIALS_PROVIDER; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; + +/** Test for {@link CredentialsProviderFactory}. */ +public class CredentialsProviderFactoryTest { + + @Rule public TemporaryFolder folder = new TemporaryFolder(); + + @Test + public void testCreateBearTokenCredentialsProviderSuccess() { + Options options = new Options(); + String token = UUID.randomUUID().toString(); + options.set(RESTCatalogOptions.TOKEN, token); + BearTokenCredentialsProvider credentialsProvider = + (BearTokenCredentialsProvider) + CredentialsProviderFactory.createCredentialsProvider( + options, this.getClass().getClassLoader()); + assertEquals(token, credentialsProvider.token()); + } + + @Test + public void testCreateBearTokenCredentialsProviderFail() { + Options options = new Options(); + assertThrows( + IllegalArgumentException.class, + () -> + CredentialsProviderFactory.createCredentialsProvider( + options, this.getClass().getClassLoader())); + } + + @Test + public void testCreateBearTokenFileCredentialsProviderSuccess() throws Exception { + Options options = new Options(); + String fileName = "token"; + File tokenFile = folder.newFile(fileName); + String token = UUID.randomUUID().toString(); + FileUtils.writeStringToFile(tokenFile, token); + options.set(RESTCatalogOptions.TOKEN_PROVIDER_PATH, tokenFile.getPath()); + BearTokenFileCredentialsProvider credentialsProvider = + (BearTokenFileCredentialsProvider) + CredentialsProviderFactory.createCredentialsProvider( + options, this.getClass().getClassLoader()); + assertEquals(token, credentialsProvider.token()); + } + + @Test + public void testCreateBearTokenFileCredentialsProviderFail() throws Exception { + Options options = new Options(); + options.set(CREDENTIALS_PROVIDER, CredentialsProviderType.BEAR_TOKEN_FILE.name()); + assertThrows( + IllegalArgumentException.class, + () -> + CredentialsProviderFactory.createCredentialsProvider( + options, this.getClass().getClassLoader())); + } + + @Test + public void testCreateRefreshBearTokenFileCredentialsProviderSuccess() throws Exception { + Options options = new Options(); + String fileName = "token"; + File tokenFile = folder.newFile(fileName); + String token = UUID.randomUUID().toString(); + FileUtils.writeStringToFile(tokenFile, token); + options.set(RESTCatalogOptions.TOKEN_PROVIDER_PATH, tokenFile.getPath()); + options.set(RESTCatalogOptions.TOKEN_EXPIRATION_TIME, Duration.ofSeconds(10L)); + BearTokenFileCredentialsProvider credentialsProvider = + (BearTokenFileCredentialsProvider) + CredentialsProviderFactory.createCredentialsProvider( + options, this.getClass().getClassLoader()); + assertEquals(token, credentialsProvider.token()); + } + + @Test + public void getCredentialsProviderTypeByConfWhenDefineTokenPath() { + Options options = new Options(); + options.set(RESTCatalogOptions.TOKEN_PROVIDER_PATH, "/a/b/c"); + assertEquals( + CredentialsProviderType.BEAR_TOKEN_FILE, + CredentialsProviderFactory.getCredentialsProviderTypeByConf(options)); + } + + @Test + public void getCredentialsProviderTypeByConfWhenConfNotDefined() { + Options options = new Options(); + assertEquals( + CredentialsProviderType.BEAR_TOKEN, + CredentialsProviderFactory.getCredentialsProviderTypeByConf(options)); + } + + @Test + public void getCredentialsProviderTypeByConfWhenDefineProviderType() { + Options options = new Options(); + options.set(CREDENTIALS_PROVIDER, CredentialsProviderType.BEAR_TOKEN_FILE.name()); + assertEquals( + CredentialsProviderType.BEAR_TOKEN_FILE, + CredentialsProviderFactory.getCredentialsProviderTypeByConf(options)); + } +} diff --git a/paimon-core/src/test/java/org/apache/paimon/schema/SchemaManagerTest.java b/paimon-core/src/test/java/org/apache/paimon/schema/SchemaManagerTest.java index 4bd965268f00..c8b102b3584d 100644 --- a/paimon-core/src/test/java/org/apache/paimon/schema/SchemaManagerTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/schema/SchemaManagerTest.java @@ -31,6 +31,7 @@ import org.apache.paimon.table.FileStoreTableFactory; import org.apache.paimon.table.sink.TableCommitImpl; import org.apache.paimon.table.sink.TableWriteImpl; +import org.apache.paimon.types.ArrayType; import org.apache.paimon.types.BigIntType; import org.apache.paimon.types.DataField; import org.apache.paimon.types.DataTypes; @@ -65,6 +66,7 @@ import static org.apache.paimon.utils.FailingFileIO.retryArtificialException; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatCode; import static org.assertj.core.api.Assertions.assertThatThrownBy; /** Test for {@link SchemaManager}. */ @@ -527,4 +529,235 @@ public void testAlterImmutableOptionsOnEmptyTable() throws Exception { .isInstanceOf(UnsupportedOperationException.class) .hasMessage("Change 'merge-engine' is not supported yet."); } + + @Test + public void testAddAndDropNestedColumns() throws Exception { + RowType innerType = + RowType.of( + new DataField(4, "f1", DataTypes.INT()), + new DataField(5, "f2", DataTypes.BIGINT())); + RowType middleType = + RowType.of( + new DataField(2, "f1", DataTypes.STRING()), + new DataField(3, "f2", innerType)); + RowType outerType = + RowType.of( + new DataField(0, "k", DataTypes.INT()), new DataField(1, "v", middleType)); + + Schema schema = + new Schema( + outerType.getFields(), + Collections.singletonList("k"), + Collections.emptyList(), + new HashMap<>(), + ""); + SchemaManager manager = new SchemaManager(LocalFileIO.create(), path); + manager.createTable(schema); + + SchemaChange addColumn = + SchemaChange.addColumn( + new String[] {"v", "f2", "f3"}, + DataTypes.STRING(), + "", + SchemaChange.Move.after("f3", "f1")); + manager.commitChanges(addColumn); + + innerType = + RowType.of( + new DataField(4, "f1", DataTypes.INT()), + new DataField(6, "f3", DataTypes.STRING(), ""), + new DataField(5, "f2", DataTypes.BIGINT())); + middleType = + RowType.of( + new DataField(2, "f1", DataTypes.STRING()), + new DataField(3, "f2", innerType)); + outerType = + RowType.of( + new DataField(0, "k", DataTypes.INT()), new DataField(1, "v", middleType)); + assertThat(manager.latest().get().logicalRowType()).isEqualTo(outerType); + + assertThatCode(() -> manager.commitChanges(addColumn)) + .hasMessageContaining("Column v.f2.f3 already exists"); + SchemaChange middleColumnNotExistAddColumn = + SchemaChange.addColumn( + new String[] {"v", "invalid", "f4"}, DataTypes.STRING(), "", null); + assertThatCode(() -> manager.commitChanges(middleColumnNotExistAddColumn)) + .hasMessageContaining("Column v.invalid does not exist"); + + SchemaChange dropColumn = SchemaChange.dropColumn(new String[] {"v", "f2", "f1"}); + manager.commitChanges(dropColumn); + + innerType = + RowType.of( + new DataField(6, "f3", DataTypes.STRING(), ""), + new DataField(5, "f2", DataTypes.BIGINT())); + middleType = + RowType.of( + new DataField(2, "f1", DataTypes.STRING()), + new DataField(3, "f2", innerType)); + outerType = + RowType.of( + new DataField(0, "k", DataTypes.INT()), new DataField(1, "v", middleType)); + assertThat(manager.latest().get().logicalRowType()).isEqualTo(outerType); + + assertThatCode(() -> manager.commitChanges(dropColumn)) + .hasMessageContaining("Column v.f2.f1 does not exist"); + SchemaChange middleColumnNotExistDropColumn = + SchemaChange.dropColumn(new String[] {"v", "invalid", "f2"}); + assertThatCode(() -> manager.commitChanges(middleColumnNotExistDropColumn)) + .hasMessageContaining("Column v.invalid does not exist"); + } + + @Test + public void testRenameNestedColumns() throws Exception { + RowType innerType = + RowType.of( + new DataField(4, "f1", DataTypes.INT()), + new DataField(5, "f2", DataTypes.BIGINT())); + RowType middleType = + RowType.of( + new DataField(2, "f1", DataTypes.STRING()), + new DataField(3, "f2", innerType)); + RowType outerType = + RowType.of( + new DataField(0, "k", DataTypes.INT()), new DataField(1, "v", middleType)); + + Schema schema = + new Schema( + outerType.getFields(), + Collections.singletonList("k"), + Collections.emptyList(), + new HashMap<>(), + ""); + SchemaManager manager = new SchemaManager(LocalFileIO.create(), path); + manager.createTable(schema); + + SchemaChange renameColumn = + SchemaChange.renameColumn(new String[] {"v", "f2", "f1"}, "f100"); + manager.commitChanges(renameColumn); + + innerType = + RowType.of( + new DataField(4, "f100", DataTypes.INT()), + new DataField(5, "f2", DataTypes.BIGINT())); + middleType = + RowType.of( + new DataField(2, "f1", DataTypes.STRING()), + new DataField(3, "f2", innerType)); + outerType = + RowType.of( + new DataField(0, "k", DataTypes.INT()), new DataField(1, "v", middleType)); + assertThat(manager.latest().get().logicalRowType()).isEqualTo(outerType); + + SchemaChange middleColumnNotExistRenameColumn = + SchemaChange.renameColumn(new String[] {"v", "invalid", "f2"}, "f200"); + assertThatCode(() -> manager.commitChanges(middleColumnNotExistRenameColumn)) + .hasMessageContaining("Column v.invalid does not exist"); + + SchemaChange lastColumnNotExistRenameColumn = + SchemaChange.renameColumn(new String[] {"v", "f2", "invalid"}, "new_invalid"); + assertThatCode(() -> manager.commitChanges(lastColumnNotExistRenameColumn)) + .hasMessageContaining("Column v.f2.invalid does not exist"); + + SchemaChange newNameAlreadyExistRenameColumn = + SchemaChange.renameColumn(new String[] {"v", "f2", "f2"}, "f100"); + assertThatCode(() -> manager.commitChanges(newNameAlreadyExistRenameColumn)) + .hasMessageContaining("Column v.f2.f100 already exists"); + } + + @Test + public void testUpdateNestedColumnType() throws Exception { + RowType innerType = + RowType.of( + new DataField(4, "f1", DataTypes.INT()), + new DataField(5, "f2", DataTypes.BIGINT())); + RowType middleType = + RowType.of( + new DataField(2, "f1", DataTypes.STRING()), + new DataField(3, "f2", innerType)); + RowType outerType = + RowType.of( + new DataField(0, "k", DataTypes.INT()), new DataField(1, "v", middleType)); + + Schema schema = + new Schema( + outerType.getFields(), + Collections.singletonList("k"), + Collections.emptyList(), + new HashMap<>(), + ""); + SchemaManager manager = new SchemaManager(LocalFileIO.create(), path); + manager.createTable(schema); + + SchemaChange updateColumnType = + SchemaChange.updateColumnType( + new String[] {"v", "f2", "f1"}, DataTypes.BIGINT(), false); + manager.commitChanges(updateColumnType); + + innerType = + RowType.of( + new DataField(4, "f1", DataTypes.BIGINT()), + new DataField(5, "f2", DataTypes.BIGINT())); + middleType = + RowType.of( + new DataField(2, "f1", DataTypes.STRING()), + new DataField(3, "f2", innerType)); + outerType = + RowType.of( + new DataField(0, "k", DataTypes.INT()), new DataField(1, "v", middleType)); + assertThat(manager.latest().get().logicalRowType()).isEqualTo(outerType); + + SchemaChange middleColumnNotExistUpdateColumnType = + SchemaChange.updateColumnType( + new String[] {"v", "invalid", "f1"}, DataTypes.BIGINT(), false); + assertThatCode(() -> manager.commitChanges(middleColumnNotExistUpdateColumnType)) + .hasMessageContaining("Column v.invalid does not exist"); + } + + @Test + public void testUpdateRowTypeInArrayAndMap() throws Exception { + RowType innerType = + RowType.of( + new DataField(2, "f1", DataTypes.INT()), + new DataField(3, "f2", DataTypes.BIGINT())); + RowType outerType = + RowType.of( + new DataField(0, "k", DataTypes.INT()), + new DataField( + 1, "v", new ArrayType(new MapType(DataTypes.INT(), innerType)))); + + Schema schema = + new Schema( + outerType.getFields(), + Collections.singletonList("k"), + Collections.emptyList(), + new HashMap<>(), + ""); + SchemaManager manager = new SchemaManager(LocalFileIO.create(), path); + manager.createTable(schema); + + SchemaChange addColumn = + SchemaChange.addColumn( + new String[] {"v", "element", "value", "f3"}, + DataTypes.STRING(), + null, + SchemaChange.Move.first("f3")); + SchemaChange dropColumn = + SchemaChange.dropColumn(new String[] {"v", "element", "value", "f2"}); + SchemaChange updateColumnType = + SchemaChange.updateColumnType( + new String[] {"v", "element", "value", "f1"}, DataTypes.BIGINT(), false); + manager.commitChanges(addColumn, dropColumn, updateColumnType); + + innerType = + RowType.of( + new DataField(4, "f3", DataTypes.STRING()), + new DataField(2, "f1", DataTypes.BIGINT())); + outerType = + RowType.of( + new DataField(0, "k", DataTypes.INT()), + new DataField( + 1, "v", new ArrayType(new MapType(DataTypes.INT(), innerType)))); + assertThat(manager.latest().get().logicalRowType()).isEqualTo(outerType); + } } diff --git a/paimon-core/src/test/java/org/apache/paimon/schema/SchemaMergingUtilsTest.java b/paimon-core/src/test/java/org/apache/paimon/schema/SchemaMergingUtilsTest.java index 8ad40852721a..bfc6dd7aa36b 100644 --- a/paimon-core/src/test/java/org/apache/paimon/schema/SchemaMergingUtilsTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/schema/SchemaMergingUtilsTest.java @@ -44,6 +44,7 @@ import org.assertj.core.util.Lists; import org.junit.jupiter.api.Test; +import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.concurrent.atomic.AtomicInteger; @@ -92,6 +93,27 @@ public void testMergeTableSchemas() { assertThat(fields.get(4).type() instanceof RowType).isTrue(); } + @Test + public void testMergeTableSchemaNotChanges() { + // Init the table schema + DataField a = new DataField(0, "a", new IntType()); + DataField b = new DataField(1, "b", new DoubleType()); + TableSchema current = + new TableSchema( + 0, + Lists.newArrayList(a, b), + 3, + new ArrayList<>(), + Lists.newArrayList("a"), + new HashMap<>(), + ""); + + // fake the RowType of data with different field sequences + RowType t = new RowType(Lists.newArrayList(b, a)); + TableSchema merged = SchemaMergingUtils.mergeSchemas(current, t, false); + assertThat(merged.id()).isEqualTo(0); + } + @Test public void testMergeSchemas() { // This will test both `mergeSchemas` and `merge` methods. diff --git a/paimon-core/src/test/java/org/apache/paimon/schema/TableSchemaTest.java b/paimon-core/src/test/java/org/apache/paimon/schema/TableSchemaTest.java index cf09d1ce40ff..d7f2e660a05a 100644 --- a/paimon-core/src/test/java/org/apache/paimon/schema/TableSchemaTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/schema/TableSchemaTest.java @@ -154,7 +154,7 @@ public void testSequenceField() { options.put(MERGE_ENGINE.key(), CoreOptions.MergeEngine.FIRST_ROW.toString()); assertThatThrownBy(() -> validateTableSchema(schema)) .hasMessageContaining( - "Do not support use sequence field on FIRST_MERGE merge engine."); + "Do not support use sequence field on FIRST_ROW merge engine."); options.put(FIELDS_PREFIX + ".f3." + AGG_FUNCTION, "max"); assertThatThrownBy(() -> validateTableSchema(schema)) diff --git a/paimon-core/src/test/java/org/apache/paimon/stats/SimpleStatsConverterTest.java b/paimon-core/src/test/java/org/apache/paimon/stats/SimpleStatsEvolutionTest.java similarity index 91% rename from paimon-core/src/test/java/org/apache/paimon/stats/SimpleStatsConverterTest.java rename to paimon-core/src/test/java/org/apache/paimon/stats/SimpleStatsEvolutionTest.java index a88276c503d2..af4fb78b24c7 100644 --- a/paimon-core/src/test/java/org/apache/paimon/stats/SimpleStatsConverterTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/stats/SimpleStatsEvolutionTest.java @@ -36,8 +36,8 @@ import static org.apache.paimon.io.DataFileTestUtils.row; import static org.assertj.core.api.Assertions.assertThat; -/** Tests for {@link SimpleStatsConverter}. */ -public class SimpleStatsConverterTest { +/** Tests for {@link SimpleStatsEvolution}. */ +public class SimpleStatsEvolutionTest { @Test public void testFromBinary() { @@ -73,8 +73,8 @@ public void testFromBinary() { SchemaEvolutionUtil.createIndexCastMapping( tableSchema.fields(), dataSchema.fields()); int[] indexMapping = indexCastMapping.getIndexMapping(); - SimpleStatsConverter serializer = - new SimpleStatsConverter( + SimpleStatsEvolution evolution = + new SimpleStatsEvolution( tableSchema.logicalRowType(), indexMapping, indexCastMapping.getCastMapping()); @@ -83,10 +83,10 @@ public void testFromBinary() { Long[] nullCounts = new Long[] {1L, 0L, 10L, 100L}; SimpleStats stats = new SimpleStats(minRowData, maxRowData, BinaryArray.fromLongArray(nullCounts)); - - InternalRow min = serializer.evolution(stats.minValues()); - InternalRow max = serializer.evolution(stats.maxValues()); - InternalArray nulls = serializer.evolution(stats.nullCounts(), 1000L); + SimpleStatsEvolution.Result result = evolution.evolution(stats, 1000L, null); + InternalRow min = result.minValues(); + InternalRow max = result.maxValues(); + InternalArray nulls = result.nullCounts(); checkFieldStats(min, max, nulls, 0, 2, 99, 0L); checkFieldStats(min, max, nulls, 1, 4, 97, 100L); diff --git a/paimon-core/src/test/java/org/apache/paimon/stats/StatsTableTest.java b/paimon-core/src/test/java/org/apache/paimon/stats/StatsTableTest.java index 494b2e28e459..ce8cfc9228ad 100644 --- a/paimon-core/src/test/java/org/apache/paimon/stats/StatsTableTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/stats/StatsTableTest.java @@ -33,20 +33,25 @@ import org.apache.paimon.table.TableTestBase; import org.apache.paimon.types.DataTypes; -import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import static org.apache.paimon.CoreOptions.METADATA_STATS_DENSE_STORE; import static org.apache.paimon.CoreOptions.METADATA_STATS_MODE; import static org.assertj.core.api.Assertions.assertThat; /** Test for table stats mode. */ public class StatsTableTest extends TableTestBase { - @Test - public void testPartitionStats() throws Exception { + @ParameterizedTest + @ValueSource(booleans = {true, false}) + public void testPartitionStatsNotDense(boolean thinMode) throws Exception { Identifier identifier = identifier("T"); Options options = new Options(); options.set(METADATA_STATS_MODE, "NONE"); + options.set(METADATA_STATS_DENSE_STORE, false); options.set(CoreOptions.BUCKET, 1); + options.set(CoreOptions.DATA_FILE_THIN_MODE, thinMode); Schema schema = Schema.newBuilder() .column("pt", DataTypes.INT()) @@ -84,10 +89,65 @@ public void testPartitionStats() throws Exception { manifestFile.read(manifest.fileName(), manifest.fileSize()).get(0).file(); SimpleStats recordStats = file.valueStats(); assertThat(recordStats.minValues().isNullAt(0)).isTrue(); - assertThat(recordStats.minValues().isNullAt(1)).isTrue(); + assertThat(recordStats.minValues().isNullAt(1)).isEqualTo(!thinMode); assertThat(recordStats.minValues().isNullAt(2)).isTrue(); assertThat(recordStats.maxValues().isNullAt(0)).isTrue(); - assertThat(recordStats.maxValues().isNullAt(1)).isTrue(); + assertThat(recordStats.maxValues().isNullAt(1)).isEqualTo(!thinMode); assertThat(recordStats.maxValues().isNullAt(2)).isTrue(); + + SimpleStats keyStats = file.keyStats(); + assertThat(keyStats.minValues().isNullAt(0)).isFalse(); + assertThat(keyStats.maxValues().isNullAt(0)).isFalse(); + } + + @ParameterizedTest + @ValueSource(booleans = {true, false}) + public void testPartitionStatsDenseMode(boolean thinMode) throws Exception { + Identifier identifier = identifier("T"); + Options options = new Options(); + options.set(METADATA_STATS_MODE, "NONE"); + options.set(CoreOptions.BUCKET, 1); + options.set(CoreOptions.DATA_FILE_THIN_MODE, thinMode); + Schema schema = + Schema.newBuilder() + .column("pt", DataTypes.INT()) + .column("pk", DataTypes.INT()) + .column("col1", DataTypes.INT()) + .partitionKeys("pt") + .primaryKey("pk", "pt") + .options(options.toMap()) + .build(); + catalog.createTable(identifier, schema, true); + Table table = catalog.getTable(identifier); + + write( + table, + GenericRow.of(1, 1, 1), + GenericRow.of(1, 2, 1), + GenericRow.of(1, 3, 1), + GenericRow.of(2, 1, 1)); + + FileStoreTable storeTable = (FileStoreTable) table; + FileStore store = storeTable.store(); + String manifestListFile = storeTable.snapshotManager().latestSnapshot().deltaManifestList(); + + ManifestList manifestList = store.manifestListFactory().create(); + ManifestFileMeta manifest = manifestList.read(manifestListFile).get(0); + + // should have partition stats + SimpleStats partitionStats = manifest.partitionStats(); + assertThat(partitionStats.minValues().getInt(0)).isEqualTo(1); + assertThat(partitionStats.maxValues().getInt(0)).isEqualTo(2); + + // should not have record stats because of NONE mode + ManifestFile manifestFile = store.manifestFileFactory().create(); + DataFileMeta file = + manifestFile.read(manifest.fileName(), manifest.fileSize()).get(0).file(); + SimpleStats recordStats = file.valueStats(); + int count = thinMode ? 1 : 0; + assertThat(file.valueStatsCols().size()).isEqualTo(count); + assertThat(recordStats.minValues().getFieldCount()).isEqualTo(count); + assertThat(recordStats.maxValues().getFieldCount()).isEqualTo(count); + assertThat(recordStats.nullCounts().size()).isEqualTo(count); } } diff --git a/paimon-core/src/test/java/org/apache/paimon/stats/StatsTestUtils.java b/paimon-core/src/test/java/org/apache/paimon/stats/StatsTestUtils.java index 0b20815795ec..20c1dc8bcd69 100644 --- a/paimon-core/src/test/java/org/apache/paimon/stats/StatsTestUtils.java +++ b/paimon-core/src/test/java/org/apache/paimon/stats/StatsTestUtils.java @@ -94,11 +94,12 @@ public static SimpleStats newEmptySimpleStats(int fieldCount) { for (int i = 0; i < fieldCount; i++) { array[i] = new SimpleColStats(null, null, 0L); } - return statsConverter.toBinary(array); + return statsConverter.toBinaryAllMode(array); } public static SimpleStats newSimpleStats(int min, int max) { SimpleStatsConverter statsConverter = new SimpleStatsConverter(RowType.of(new IntType())); - return statsConverter.toBinary(new SimpleColStats[] {new SimpleColStats(min, max, 0L)}); + return statsConverter.toBinaryAllMode( + new SimpleColStats[] {new SimpleColStats(min, max, 0L)}); } } diff --git a/paimon-core/src/test/java/org/apache/paimon/table/AppendOnlyFileDataTableTest.java b/paimon-core/src/test/java/org/apache/paimon/table/AppendOnlyFileDataTableTest.java index 93854e766198..9ce3db0b1ada 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/AppendOnlyFileDataTableTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/AppendOnlyFileDataTableTest.java @@ -33,7 +33,7 @@ protected FileStoreTable createFileStoreTable(Map tableSchema return new AppendOnlyFileStoreTable( FileIOFinder.find(tablePath), tablePath, schemaManager.latest().get()) { @Override - protected SchemaManager schemaManager() { + public SchemaManager schemaManager() { return schemaManager; } }; diff --git a/paimon-core/src/test/java/org/apache/paimon/table/AppendOnlyFileStoreTableTest.java b/paimon-core/src/test/java/org/apache/paimon/table/AppendOnlyFileStoreTableTest.java index 102477fe4848..01d4e89af95d 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/AppendOnlyFileStoreTableTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/AppendOnlyFileStoreTableTest.java @@ -26,7 +26,9 @@ import org.apache.paimon.data.InternalRow; import org.apache.paimon.data.serializer.InternalRowSerializer; import org.apache.paimon.fileindex.FileIndexOptions; +import org.apache.paimon.fileindex.bitmap.BitmapFileIndexFactory; import org.apache.paimon.fileindex.bloomfilter.BloomFilterFileIndexFactory; +import org.apache.paimon.fileindex.bsi.BitSliceIndexBitmapFileIndexFactory; import org.apache.paimon.fs.FileIOFinder; import org.apache.paimon.fs.Path; import org.apache.paimon.fs.local.LocalFileIO; @@ -58,6 +60,8 @@ import org.assertj.core.api.Assertions; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; import java.util.ArrayList; import java.util.Arrays; @@ -74,7 +78,9 @@ import static org.apache.paimon.CoreOptions.BUCKET; import static org.apache.paimon.CoreOptions.BUCKET_KEY; +import static org.apache.paimon.CoreOptions.DATA_FILE_PATH_DIRECTORY; import static org.apache.paimon.CoreOptions.FILE_INDEX_IN_MANIFEST_THRESHOLD; +import static org.apache.paimon.CoreOptions.METADATA_STATS_MODE; import static org.apache.paimon.io.DataFileTestUtils.row; import static org.apache.paimon.table.sink.KeyAndBucketExtractor.bucket; import static org.apache.paimon.table.sink.KeyAndBucketExtractor.bucketKeyHashCode; @@ -138,6 +144,26 @@ public void testBatchReadWrite() throws Exception { "2|21|201|binary|varbinary|mapKey:mapVal|multiset")); } + @Test + public void testReadWriteWithDataDirectory() throws Exception { + Consumer optionsSetter = options -> options.set(DATA_FILE_PATH_DIRECTORY, "data"); + writeData(optionsSetter); + FileStoreTable table = createFileStoreTable(optionsSetter); + + assertThat(table.fileIO().exists(new Path(tablePath, "data/pt=1"))).isTrue(); + + List splits = toSplits(table.newSnapshotReader().read().dataSplits()); + TableRead read = table.newRead(); + assertThat(getResult(read, splits, binaryRow(1), 0, BATCH_ROW_TO_STRING)) + .hasSameElementsAs( + Arrays.asList( + "1|10|100|binary|varbinary|mapKey:mapVal|multiset", + "1|11|101|binary|varbinary|mapKey:mapVal|multiset", + "1|12|102|binary|varbinary|mapKey:mapVal|multiset", + "1|11|101|binary|varbinary|mapKey:mapVal|multiset", + "1|12|102|binary|varbinary|mapKey:mapVal|multiset")); + } + @Test public void testBatchRecordsWrite() throws Exception { FileStoreTable table = createFileStoreTable(); @@ -220,10 +246,18 @@ public void testBatchProjection() throws Exception { .hasSameElementsAs(Arrays.asList("200|20", "201|21", "202|22", "201|21")); } - @Test - public void testBatchFilter() throws Exception { - writeData(); - FileStoreTable table = createFileStoreTable(); + @ParameterizedTest + @ValueSource(booleans = {true, false}) + public void testBatchFilter(boolean statsDenseStore) throws Exception { + Consumer optionsSetter = + options -> { + if (statsDenseStore) { + options.set(CoreOptions.METADATA_STATS_MODE, "none"); + options.set("fields.b.stats-mode", "full"); + } + }; + writeData(optionsSetter); + FileStoreTable table = createFileStoreTable(optionsSetter); PredicateBuilder builder = new PredicateBuilder(table.schema().logicalRowType()); Predicate predicate = builder.equal(2, 201L); @@ -548,6 +582,146 @@ public void testBloomFilterInDisk() throws Exception { reader.forEachRemaining(row -> assertThat(row.getString(1).toString()).isEqualTo("b")); } + @Test + public void testBSIAndBitmapIndexInMemory() throws Exception { + RowType rowType = + RowType.builder() + .field("id", DataTypes.INT()) + .field("event", DataTypes.STRING()) + .field("price", DataTypes.BIGINT()) + .build(); + // in unaware-bucket mode, we split files into splits all the time + FileStoreTable table = + createUnawareBucketFileStoreTable( + rowType, + options -> { + options.set(METADATA_STATS_MODE, "NONE"); + options.set( + FileIndexOptions.FILE_INDEX + + "." + + BitmapFileIndexFactory.BITMAP_INDEX + + "." + + CoreOptions.COLUMNS, + "event"); + options.set( + FileIndexOptions.FILE_INDEX + + "." + + BitSliceIndexBitmapFileIndexFactory.BSI_INDEX + + "." + + CoreOptions.COLUMNS, + "price"); + options.set(FILE_INDEX_IN_MANIFEST_THRESHOLD.key(), "1 MB"); + }); + + StreamTableWrite write = table.newWrite(commitUser); + StreamTableCommit commit = table.newCommit(commitUser); + + List result = new ArrayList<>(); + write.write(GenericRow.of(1, BinaryString.fromString("A"), 4L)); + write.write(GenericRow.of(1, BinaryString.fromString("B"), 2L)); + write.write(GenericRow.of(1, BinaryString.fromString("B"), 3L)); + write.write(GenericRow.of(1, BinaryString.fromString("C"), 3L)); + result.addAll(write.prepareCommit(true, 0)); + write.write(GenericRow.of(1, BinaryString.fromString("A"), 4L)); + write.write(GenericRow.of(1, BinaryString.fromString("B"), 3L)); + write.write(GenericRow.of(1, BinaryString.fromString("C"), 4L)); + write.write(GenericRow.of(1, BinaryString.fromString("D"), 2L)); + write.write(GenericRow.of(1, BinaryString.fromString("D"), 4L)); + result.addAll(write.prepareCommit(true, 0)); + commit.commit(0, result); + result.clear(); + + // test bitmap index and bsi index + Predicate predicate = + PredicateBuilder.and( + new PredicateBuilder(rowType).equal(1, BinaryString.fromString("C")), + new PredicateBuilder(rowType).greaterThan(2, 3L)); + TableScan.Plan plan = table.newScan().withFilter(predicate).plan(); + List metas = + plan.splits().stream() + .flatMap(split -> ((DataSplit) split).dataFiles().stream()) + .collect(Collectors.toList()); + assertThat(metas.size()).isEqualTo(1); + + RecordReader reader = + table.newRead().withFilter(predicate).createReader(plan.splits()); + reader.forEachRemaining( + row -> { + assertThat(row.getString(1).toString()).isEqualTo("C"); + assertThat(row.getLong(2)).isEqualTo(4L); + }); + } + + @Test + public void testBSIAndBitmapIndexInDisk() throws Exception { + RowType rowType = + RowType.builder() + .field("id", DataTypes.INT()) + .field("event", DataTypes.STRING()) + .field("price", DataTypes.BIGINT()) + .build(); + // in unaware-bucket mode, we split files into splits all the time + FileStoreTable table = + createUnawareBucketFileStoreTable( + rowType, + options -> { + options.set(METADATA_STATS_MODE, "NONE"); + options.set( + FileIndexOptions.FILE_INDEX + + "." + + BitmapFileIndexFactory.BITMAP_INDEX + + "." + + CoreOptions.COLUMNS, + "event"); + options.set( + FileIndexOptions.FILE_INDEX + + "." + + BitSliceIndexBitmapFileIndexFactory.BSI_INDEX + + "." + + CoreOptions.COLUMNS, + "price"); + options.set(FILE_INDEX_IN_MANIFEST_THRESHOLD.key(), "1 B"); + }); + + StreamTableWrite write = table.newWrite(commitUser); + StreamTableCommit commit = table.newCommit(commitUser); + + List result = new ArrayList<>(); + write.write(GenericRow.of(1, BinaryString.fromString("A"), 4L)); + write.write(GenericRow.of(1, BinaryString.fromString("B"), 2L)); + write.write(GenericRow.of(1, BinaryString.fromString("B"), 3L)); + write.write(GenericRow.of(1, BinaryString.fromString("C"), 3L)); + result.addAll(write.prepareCommit(true, 0)); + write.write(GenericRow.of(1, BinaryString.fromString("A"), 4L)); + write.write(GenericRow.of(1, BinaryString.fromString("B"), 3L)); + write.write(GenericRow.of(1, BinaryString.fromString("C"), 4L)); + write.write(GenericRow.of(1, BinaryString.fromString("D"), 2L)); + write.write(GenericRow.of(1, BinaryString.fromString("D"), 4L)); + result.addAll(write.prepareCommit(true, 0)); + commit.commit(0, result); + result.clear(); + + // test bitmap index and bsi index + Predicate predicate = + PredicateBuilder.and( + new PredicateBuilder(rowType).equal(1, BinaryString.fromString("C")), + new PredicateBuilder(rowType).greaterThan(2, 3L)); + TableScan.Plan plan = table.newScan().withFilter(predicate).plan(); + List metas = + plan.splits().stream() + .flatMap(split -> ((DataSplit) split).dataFiles().stream()) + .collect(Collectors.toList()); + assertThat(metas.size()).isEqualTo(2); + + RecordReader reader = + table.newRead().withFilter(predicate).createReader(plan.splits()); + reader.forEachRemaining( + row -> { + assertThat(row.getString(1).toString()).isEqualTo("C"); + assertThat(row.getLong(2)).isEqualTo(4L); + }); + } + @Test public void testWithShardAppendTable() throws Exception { FileStoreTable table = createFileStoreTable(conf -> conf.set(BUCKET, -1)); @@ -858,7 +1032,11 @@ public void testBatchOrderWithCompaction() throws Exception { } private void writeData() throws Exception { - FileStoreTable table = createFileStoreTable(); + writeData(options -> {}); + } + + private void writeData(Consumer optionsSetter) throws Exception { + FileStoreTable table = createFileStoreTable(optionsSetter); StreamTableWrite write = table.newWrite(commitUser); StreamTableCommit commit = table.newCommit(commitUser); diff --git a/paimon-core/src/test/java/org/apache/paimon/table/AppendOnlyTableColumnTypeFileDataTest.java b/paimon-core/src/test/java/org/apache/paimon/table/AppendOnlyTableColumnTypeFileDataTest.java index b4c16cef20a7..64d0c728d10b 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/AppendOnlyTableColumnTypeFileDataTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/AppendOnlyTableColumnTypeFileDataTest.java @@ -37,7 +37,7 @@ protected FileStoreTable createFileStoreTable(Map tableSchema SchemaManager schemaManager = new TestingSchemaManager(tablePath, tableSchemas); return new AppendOnlyFileStoreTable(fileIO, tablePath, schemaManager.latest().get()) { @Override - protected SchemaManager schemaManager() { + public SchemaManager schemaManager() { return schemaManager; } }; diff --git a/paimon-core/src/test/java/org/apache/paimon/table/AppendOnlyTableColumnTypeFileMetaTest.java b/paimon-core/src/test/java/org/apache/paimon/table/AppendOnlyTableColumnTypeFileMetaTest.java index f398d28cc524..300483a9f34b 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/AppendOnlyTableColumnTypeFileMetaTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/AppendOnlyTableColumnTypeFileMetaTest.java @@ -40,7 +40,7 @@ protected FileStoreTable createFileStoreTable(Map tableSchema SchemaManager schemaManager = new TestingSchemaManager(tablePath, tableSchemas); return new AppendOnlyFileStoreTable(fileIO, tablePath, schemaManager.latest().get()) { @Override - protected SchemaManager schemaManager() { + public SchemaManager schemaManager() { return schemaManager; } }; diff --git a/paimon-core/src/test/java/org/apache/paimon/table/AppendOnlyTableFileMetaFilterTest.java b/paimon-core/src/test/java/org/apache/paimon/table/AppendOnlyTableFileMetaFilterTest.java index f65546af75d8..85ed80299736 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/AppendOnlyTableFileMetaFilterTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/AppendOnlyTableFileMetaFilterTest.java @@ -40,7 +40,7 @@ protected FileStoreTable createFileStoreTable(Map tableSchema SchemaManager schemaManager = new TestingSchemaManager(tablePath, tableSchemas); return new AppendOnlyFileStoreTable(fileIO, tablePath, schemaManager.latest().get()) { @Override - protected SchemaManager schemaManager() { + public SchemaManager schemaManager() { return schemaManager; } }; diff --git a/paimon-core/src/test/java/org/apache/paimon/table/ColumnTypeFileMetaTestBase.java b/paimon-core/src/test/java/org/apache/paimon/table/ColumnTypeFileMetaTestBase.java index 12544a093e9f..7f264fa817b2 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/ColumnTypeFileMetaTestBase.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/ColumnTypeFileMetaTestBase.java @@ -26,8 +26,8 @@ import org.apache.paimon.predicate.PredicateBuilder; import org.apache.paimon.schema.TableSchema; import org.apache.paimon.stats.SimpleStats; -import org.apache.paimon.stats.SimpleStatsConverter; -import org.apache.paimon.stats.SimpleStatsConverters; +import org.apache.paimon.stats.SimpleStatsEvolution; +import org.apache.paimon.stats.SimpleStatsEvolutions; import org.apache.paimon.table.source.DataSplit; import org.apache.paimon.types.DataField; @@ -236,12 +236,13 @@ protected void validateValuesWithNewSchema( List filesName, List fileMetaList) { Function> schemaFields = id -> tableSchemas.get(id).fields(); - SimpleStatsConverters converters = new SimpleStatsConverters(schemaFields, schemaId); + SimpleStatsEvolutions converters = new SimpleStatsEvolutions(schemaFields, schemaId); for (DataFileMeta fileMeta : fileMetaList) { SimpleStats stats = getTableValueStats(fileMeta); - SimpleStatsConverter serializer = converters.getOrCreate(fileMeta.schemaId()); - InternalRow min = serializer.evolution(stats.minValues()); - InternalRow max = serializer.evolution(stats.maxValues()); + SimpleStatsEvolution.Result result = + converters.getOrCreate(fileMeta.schemaId()).evolution(stats, null, null); + InternalRow min = result.minValues(); + InternalRow max = result.maxValues(); assertThat(stats.minValues().getFieldCount()).isEqualTo(12); if (filesName.contains(fileMeta.fileName())) { checkTwoValues(min, max); diff --git a/paimon-core/src/test/java/org/apache/paimon/table/FileMetaFilterTestBase.java b/paimon-core/src/test/java/org/apache/paimon/table/FileMetaFilterTestBase.java index 3d1ebb3d528e..182a42fe4bfc 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/FileMetaFilterTestBase.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/FileMetaFilterTestBase.java @@ -23,8 +23,8 @@ import org.apache.paimon.predicate.Predicate; import org.apache.paimon.predicate.PredicateBuilder; import org.apache.paimon.stats.SimpleStats; -import org.apache.paimon.stats.SimpleStatsConverter; -import org.apache.paimon.stats.SimpleStatsConverters; +import org.apache.paimon.stats.SimpleStatsEvolution; +import org.apache.paimon.stats.SimpleStatsEvolutions; import org.apache.paimon.table.source.DataSplit; import org.apache.paimon.types.DataField; @@ -78,14 +78,16 @@ public void testTableSplit() throws Exception { .containsAll(filesName); Function> schemaFields = id -> schemas.get(id).fields(); - SimpleStatsConverters converters = - new SimpleStatsConverters(schemaFields, table.schema().id()); + SimpleStatsEvolutions converters = + new SimpleStatsEvolutions(schemaFields, table.schema().id()); for (DataFileMeta fileMeta : fileMetaList) { SimpleStats stats = getTableValueStats(fileMeta); - SimpleStatsConverter serializer = - converters.getOrCreate(fileMeta.schemaId()); - InternalRow min = serializer.evolution(stats.minValues()); - InternalRow max = serializer.evolution(stats.maxValues()); + SimpleStatsEvolution.Result result = + converters + .getOrCreate(fileMeta.schemaId()) + .evolution(stats, 100L, null); + InternalRow min = result.minValues(); + InternalRow max = result.maxValues(); assertThat(min.getFieldCount()).isEqualTo(6); @@ -184,16 +186,18 @@ public void testTableSplitFilterExistFields() throws Exception { assertThat(filterAllSplits).isEqualTo(allSplits); Function> schemaFields = id -> schemas.get(id).fields(); - SimpleStatsConverters converters = - new SimpleStatsConverters(schemaFields, table.schema().id()); + SimpleStatsEvolutions converters = + new SimpleStatsEvolutions(schemaFields, table.schema().id()); Set filterFileNames = new HashSet<>(); for (DataSplit dataSplit : filterAllSplits) { for (DataFileMeta dataFileMeta : dataSplit.dataFiles()) { SimpleStats stats = getTableValueStats(dataFileMeta); - SimpleStatsConverter serializer = - converters.getOrCreate(dataFileMeta.schemaId()); - InternalRow min = serializer.evolution(stats.minValues()); - InternalRow max = serializer.evolution(stats.maxValues()); + SimpleStatsEvolution.Result result = + converters + .getOrCreate(dataFileMeta.schemaId()) + .evolution(stats, 100L, null); + InternalRow min = result.minValues(); + InternalRow max = result.maxValues(); int minValue = min.getInt(1); int maxValue = max.getInt(1); if (minValue >= 14 @@ -263,15 +267,17 @@ public void testTableSplitFilterNewFields() throws Exception { Set filterFileNames = new HashSet<>(); Function> schemaFields = id -> schemas.get(id).fields(); - SimpleStatsConverters converters = - new SimpleStatsConverters(schemaFields, table.schema().id()); + SimpleStatsEvolutions converters = + new SimpleStatsEvolutions(schemaFields, table.schema().id()); for (DataSplit dataSplit : allSplits) { for (DataFileMeta dataFileMeta : dataSplit.dataFiles()) { SimpleStats stats = getTableValueStats(dataFileMeta); - SimpleStatsConverter serializer = - converters.getOrCreate(dataFileMeta.schemaId()); - InternalRow min = serializer.evolution(stats.minValues()); - InternalRow max = serializer.evolution(stats.maxValues()); + SimpleStatsEvolution.Result result = + converters + .getOrCreate(dataFileMeta.schemaId()) + .evolution(stats, 100L, null); + InternalRow min = result.minValues(); + InternalRow max = result.maxValues(); Integer minValue = min.isNullAt(3) ? null : min.getInt(3); Integer maxValue = max.isNullAt(3) ? null : max.getInt(3); if (minValue != null diff --git a/paimon-core/src/test/java/org/apache/paimon/table/FileStoreTableTestBase.java b/paimon-core/src/test/java/org/apache/paimon/table/FileStoreTableTestBase.java index f8b15c155f63..75e284a68c3a 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/FileStoreTableTestBase.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/FileStoreTableTestBase.java @@ -1136,8 +1136,9 @@ public void testCreateSameTagName() throws Exception { table.createTag("test-tag", 1); // verify that tag file exist assertThat(tagManager.tagExists("test-tag")).isTrue(); - // Create again - table.createTag("test-tag", 1); + // Create again failed if tag existed + Assertions.assertThatThrownBy(() -> table.createTag("test-tag", 1)) + .hasMessageContaining("Tag name 'test-tag' already exists."); Assertions.assertThatThrownBy(() -> table.createTag("test-tag", 2)) .hasMessageContaining("Tag name 'test-tag' already exists."); } @@ -1192,7 +1193,7 @@ public void testCreateBranch() throws Exception { SchemaManager schemaManager = new SchemaManager(new TraceableFileIO(), tablePath, "test-branch"); TableSchema branchSchema = - SchemaManager.fromPath(new TraceableFileIO(), schemaManager.toSchemaPath(0)); + TableSchema.fromPath(new TraceableFileIO(), schemaManager.toSchemaPath(0)); TableSchema schema0 = schemaManager.schema(0); assertThat(branchSchema.equals(schema0)).isTrue(); } @@ -1343,7 +1344,7 @@ public void testFastForward() throws Exception { // verify schema in branch1 and main branch is same SchemaManager schemaManager = new SchemaManager(new TraceableFileIO(), tablePath); TableSchema branchSchema = - SchemaManager.fromPath( + TableSchema.fromPath( new TraceableFileIO(), schemaManager.copyWithBranch(BRANCH_NAME).toSchemaPath(0)); TableSchema schema0 = schemaManager.schema(0); @@ -1472,10 +1473,10 @@ public void testAsyncExpireExecutionMode() throws Exception { TestFileStore.getFilesInUse( latestSnapshotId, snapshotManager, - store.newScan(), table.fileIO(), store.pathFactory(), - store.manifestListFactory().create()); + store.manifestListFactory().create(), + store.manifestFileFactory().create()); List unusedFileList = Files.walk(Paths.get(tempDir.toString())) diff --git a/paimon-core/src/test/java/org/apache/paimon/table/PrimaryKeyColumnTypeFileDataTest.java b/paimon-core/src/test/java/org/apache/paimon/table/PrimaryKeyColumnTypeFileDataTest.java index 8ba25c6617fe..64bb5f21abbb 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/PrimaryKeyColumnTypeFileDataTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/PrimaryKeyColumnTypeFileDataTest.java @@ -91,7 +91,7 @@ protected FileStoreTable createFileStoreTable(Map tableSchema SchemaManager schemaManager = new TestingSchemaManager(tablePath, tableSchemas); return new PrimaryKeyFileStoreTable(fileIO, tablePath, schemaManager.latest().get()) { @Override - protected SchemaManager schemaManager() { + public SchemaManager schemaManager() { return schemaManager; } }; diff --git a/paimon-core/src/test/java/org/apache/paimon/table/PrimaryKeyFileDataTableTest.java b/paimon-core/src/test/java/org/apache/paimon/table/PrimaryKeyFileDataTableTest.java index 1be321975466..ba9813804498 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/PrimaryKeyFileDataTableTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/PrimaryKeyFileDataTableTest.java @@ -247,7 +247,7 @@ protected FileStoreTable createFileStoreTable(Map tableSchema return new PrimaryKeyFileStoreTable(fileIO, tablePath, schemaManager.latest().get()) { @Override - protected SchemaManager schemaManager() { + public SchemaManager schemaManager() { return schemaManager; } }; diff --git a/paimon-core/src/test/java/org/apache/paimon/table/PrimaryKeyFileMetaFilterTest.java b/paimon-core/src/test/java/org/apache/paimon/table/PrimaryKeyFileMetaFilterTest.java index 618e8691c65d..88928fe991bc 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/PrimaryKeyFileMetaFilterTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/PrimaryKeyFileMetaFilterTest.java @@ -146,7 +146,7 @@ protected FileStoreTable createFileStoreTable(Map tableSchema SchemaManager schemaManager = new TestingSchemaManager(tablePath, tableSchemas); return new PrimaryKeyFileStoreTable(fileIO, tablePath, schemaManager.latest().get()) { @Override - protected SchemaManager schemaManager() { + public SchemaManager schemaManager() { return schemaManager; } }; diff --git a/paimon-core/src/test/java/org/apache/paimon/table/PrimaryKeyFileStoreTableTest.java b/paimon-core/src/test/java/org/apache/paimon/table/PrimaryKeyFileStoreTableTest.java index 7edc418186a8..fa635e2ab666 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/PrimaryKeyFileStoreTableTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/PrimaryKeyFileStoreTableTest.java @@ -84,6 +84,7 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Random; import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.BiFunction; @@ -99,6 +100,7 @@ import static org.apache.paimon.CoreOptions.ChangelogProducer.LOOKUP; import static org.apache.paimon.CoreOptions.DELETION_VECTORS_ENABLED; import static org.apache.paimon.CoreOptions.FILE_FORMAT; +import static org.apache.paimon.CoreOptions.FILE_INDEX_IN_MANIFEST_THRESHOLD; import static org.apache.paimon.CoreOptions.LOOKUP_LOCAL_FILE_TYPE; import static org.apache.paimon.CoreOptions.MERGE_ENGINE; import static org.apache.paimon.CoreOptions.MergeEngine; @@ -120,17 +122,6 @@ /** Tests for {@link PrimaryKeyFileStoreTable}. */ public class PrimaryKeyFileStoreTableTest extends FileStoreTableTestBase { - protected static final RowType COMPATIBILITY_ROW_TYPE = - RowType.of( - new DataType[] { - DataTypes.INT(), - DataTypes.INT(), - DataTypes.BIGINT(), - DataTypes.BINARY(1), - DataTypes.VARBINARY(1) - }, - new String[] {"pt", "a", "b", "c", "d"}); - protected static final Function COMPATIBILITY_BATCH_ROW_TO_STRING = rowData -> rowData.getInt(0) @@ -143,12 +134,6 @@ public class PrimaryKeyFileStoreTableTest extends FileStoreTableTestBase { + "|" + new String(rowData.getBinary(4)); - protected static final Function COMPATIBILITY_CHANGELOG_ROW_TO_STRING = - rowData -> - rowData.getRowKind().shortString() - + " " - + COMPATIBILITY_BATCH_ROW_TO_STRING.apply(rowData); - @Test public void testMultipleWriters() throws Exception { WriteSelector selector = @@ -359,10 +344,19 @@ public void testBatchProjection() throws Exception { .isEqualTo(Arrays.asList("20001|21", "202|22")); } - @Test - public void testBatchFilter() throws Exception { - writeData(); - FileStoreTable table = createFileStoreTable(); + @ParameterizedTest + @ValueSource(booleans = {true, false}) + public void testBatchFilter(boolean statsDenseStore) throws Exception { + Consumer optionsSetter = + options -> { + if (statsDenseStore) { + // pk table doesn't need value stats + options.set(CoreOptions.METADATA_STATS_MODE, "none"); + } + }; + writeData(optionsSetter); + FileStoreTable table = createFileStoreTable(optionsSetter); + PredicateBuilder builder = new PredicateBuilder(table.schema().logicalRowType()); Predicate predicate = and(builder.equal(2, 201L), builder.equal(1, 21)); @@ -604,7 +598,11 @@ private void innerTestStreamingFullChangelog(Consumer configure) throws } private void writeData() throws Exception { - FileStoreTable table = createFileStoreTable(); + writeData(options -> {}); + } + + private void writeData(Consumer optionsSetter) throws Exception { + FileStoreTable table = createFileStoreTable(optionsSetter); StreamTableWrite write = table.newWrite(commitUser); StreamTableCommit commit = table.newCommit(commitUser); @@ -658,10 +656,6 @@ private void writeBranchData(FileStoreTable table) throws Exception { @Test public void testReadFilter() throws Exception { FileStoreTable table = createFileStoreTable(); - if (table.coreOptions().fileFormat().getFormatIdentifier().equals("parquet")) { - // TODO support parquet reader filter push down - return; - } StreamTableWrite write = table.newWrite(commitUser); StreamTableCommit commit = table.newCommit(commitUser); @@ -777,6 +771,284 @@ public void testWithShardDeletionVectors() throws Exception { innerTestWithShard(table); } + @Test + public void testDeletionVectorsWithFileIndexInFile() throws Exception { + FileStoreTable table = + createFileStoreTable( + conf -> { + conf.set(BUCKET, 1); + conf.set(DELETION_VECTORS_ENABLED, true); + conf.set(TARGET_FILE_SIZE, MemorySize.ofBytes(1)); + conf.set("file-index.bloom-filter.columns", "b"); + }); + + StreamTableWrite write = + table.newWrite(commitUser).withIOManager(new IOManagerImpl(tempDir.toString())); + StreamTableCommit commit = table.newCommit(commitUser); + + write.write(rowData(1, 1, 300L)); + write.write(rowData(1, 2, 400L)); + write.write(rowData(1, 3, 200L)); + write.write(rowData(1, 4, 500L)); + commit.commit(0, write.prepareCommit(true, 0)); + + write.write(rowData(1, 5, 100L)); + write.write(rowData(1, 6, 600L)); + write.write(rowData(1, 7, 400L)); + commit.commit(1, write.prepareCommit(true, 1)); + + PredicateBuilder builder = new PredicateBuilder(ROW_TYPE); + List splits = toSplits(table.newSnapshotReader().read().dataSplits()); + assertThat(((DataSplit) splits.get(0)).dataFiles().size()).isEqualTo(2); + TableRead read = table.newRead().withFilter(builder.equal(2, 300L)); + assertThat(getResult(read, splits, BATCH_ROW_TO_STRING)) + .hasSameElementsAs( + Arrays.asList( + "1|1|300|binary|varbinary|mapKey:mapVal|multiset", + "1|2|400|binary|varbinary|mapKey:mapVal|multiset", + "1|3|200|binary|varbinary|mapKey:mapVal|multiset", + "1|4|500|binary|varbinary|mapKey:mapVal|multiset")); + } + + @Test + public void testDeletionVectorsWithParquetFilter() throws Exception { + // RowGroup record range [pk] : + // + // RowGroup-0 : [0-93421) + // RowGroup-1 : [93421-187794) + // RowGroup-2 : [187794-200000) + // + // ColumnPage record count : + // + // col-0 : 300 + // col-1 : 200 + // col-2 : 300 + // col-3 : 300 + // col-4 : 300 + // col-5 : 200 + // col-6 : 100 + // col-7 : 100 + // col-8 : 100 + // col-9 : 100 + // col-10 : 100 + // col-11 : 300 + + FileStoreTable table = + createFileStoreTable( + conf -> { + conf.set(BUCKET, 1); + conf.set(DELETION_VECTORS_ENABLED, true); + conf.set(FILE_FORMAT, "parquet"); + conf.set("parquet.block.size", "1048576"); + conf.set("parquet.page.size", "1024"); + }); + + BatchWriteBuilder writeBuilder = table.newBatchWriteBuilder(); + + BatchTableWrite write = + (BatchTableWrite) + writeBuilder + .newWrite() + .withIOManager(new IOManagerImpl(tempDir.toString())); + + for (int i = 0; i < 200000; i++) { + write.write(rowData(1, i, i * 100L)); + } + + List messages = write.prepareCommit(); + BatchTableCommit commit = writeBuilder.newCommit(); + commit.commit(messages); + write = + (BatchTableWrite) + writeBuilder + .newWrite() + .withIOManager(new IOManagerImpl(tempDir.toString())); + for (int i = 110000; i < 115000; i++) { + write.write(rowDataWithKind(RowKind.DELETE, 1, i, i * 100L)); + } + + for (int i = 130000; i < 135000; i++) { + write.write(rowDataWithKind(RowKind.DELETE, 1, i, i * 100L)); + } + + messages = write.prepareCommit(); + commit = writeBuilder.newCommit(); + commit.commit(messages); + + PredicateBuilder builder = new PredicateBuilder(ROW_TYPE); + List splits = toSplits(table.newSnapshotReader().read().dataSplits()); + Random random = new Random(); + + // point filter + + for (int i = 0; i < 10; i++) { + int value = random.nextInt(110000); + TableRead read = table.newRead().withFilter(builder.equal(1, value)).executeFilter(); + assertThat(getResult(read, splits, BATCH_ROW_TO_STRING)) + .isEqualTo( + Arrays.asList( + String.format( + "%d|%d|%d|binary|varbinary|mapKey:mapVal|multiset", + 1, value, value * 100L))); + } + + for (int i = 0; i < 10; i++) { + int value = 130000 + random.nextInt(5000); + TableRead read = table.newRead().withFilter(builder.equal(1, value)).executeFilter(); + assertThat(getResult(read, splits, BATCH_ROW_TO_STRING)).isEmpty(); + } + + TableRead tableRead = + table.newRead() + .withFilter( + PredicateBuilder.and( + builder.greaterOrEqual(1, 100000), + builder.lessThan(1, 150000))) + .executeFilter(); + + List result = getResult(tableRead, splits, BATCH_ROW_TO_STRING); + + assertThat(result.size()).isEqualTo(40000); // filter 10000 + + assertThat(result) + .doesNotContain("1|110000|11000000|binary|varbinary|mapKey:mapVal|multiset"); + assertThat(result) + .doesNotContain("1|114999|11499900|binary|varbinary|mapKey:mapVal|multiset"); + assertThat(result) + .doesNotContain("1|130000|13000000|binary|varbinary|mapKey:mapVal|multiset"); + assertThat(result) + .doesNotContain("1|134999|13499900|binary|varbinary|mapKey:mapVal|multiset"); + assertThat(result).contains("1|100000|10000000|binary|varbinary|mapKey:mapVal|multiset"); + assertThat(result).contains("1|149999|14999900|binary|varbinary|mapKey:mapVal|multiset"); + + assertThat(result).contains("1|101099|10109900|binary|varbinary|mapKey:mapVal|multiset"); + assertThat(result).contains("1|115000|11500000|binary|varbinary|mapKey:mapVal|multiset"); + assertThat(result).contains("1|129999|12999900|binary|varbinary|mapKey:mapVal|multiset"); + assertThat(result).contains("1|135000|13500000|binary|varbinary|mapKey:mapVal|multiset"); + } + + @Test + public void testDeletionVectorsWithFileIndexInMeta() throws Exception { + FileStoreTable table = + createFileStoreTable( + conf -> { + conf.set(BUCKET, 1); + conf.set(DELETION_VECTORS_ENABLED, true); + conf.set(TARGET_FILE_SIZE, MemorySize.ofBytes(1)); + conf.set("file-index.bloom-filter.columns", "b"); + conf.set("file-index.bloom-filter.b.items", "20"); + }); + + StreamTableWrite write = + table.newWrite(commitUser).withIOManager(new IOManagerImpl(tempDir.toString())); + StreamTableCommit commit = table.newCommit(commitUser); + + write.write(rowData(1, 1, 300L)); + write.write(rowData(1, 2, 400L)); + write.write(rowData(1, 3, 200L)); + write.write(rowData(1, 4, 500L)); + commit.commit(0, write.prepareCommit(true, 0)); + + write.write(rowData(1, 5, 100L)); + write.write(rowData(1, 6, 600L)); + write.write(rowData(1, 7, 400L)); + commit.commit(1, write.prepareCommit(true, 1)); + + PredicateBuilder builder = new PredicateBuilder(ROW_TYPE); + Predicate predicate = builder.equal(2, 300L); + + List splits = + toSplits(table.newSnapshotReader().withFilter(predicate).read().dataSplits()); + + assertThat(((DataSplit) splits.get(0)).dataFiles().size()).isEqualTo(1); + } + + @Test + public void testDeletionVectorsWithBitmapFileIndexInFile() throws Exception { + FileStoreTable table = + createFileStoreTable( + conf -> { + conf.set(BUCKET, 1); + conf.set(DELETION_VECTORS_ENABLED, true); + conf.set(TARGET_FILE_SIZE, MemorySize.ofBytes(1)); + conf.set(FILE_INDEX_IN_MANIFEST_THRESHOLD, MemorySize.ofBytes(1)); + conf.set("file-index.bitmap.columns", "b"); + }); + + StreamTableWrite write = + table.newWrite(commitUser).withIOManager(new IOManagerImpl(tempDir.toString())); + StreamTableCommit commit = table.newCommit(commitUser); + + write.write(rowData(1, 1, 300L)); + write.write(rowData(1, 2, 400L)); + write.write(rowData(1, 3, 100L)); + write.write(rowData(1, 4, 100L)); + commit.commit(0, write.prepareCommit(true, 0)); + + write.write(rowData(1, 1, 100L)); + write.write(rowData(1, 2, 100L)); + write.write(rowData(1, 3, 300L)); + write.write(rowData(1, 5, 100L)); + commit.commit(1, write.prepareCommit(true, 1)); + + write.write(rowData(1, 4, 200L)); + commit.commit(2, write.prepareCommit(true, 2)); + + PredicateBuilder builder = new PredicateBuilder(ROW_TYPE); + List splits = toSplits(table.newSnapshotReader().read().dataSplits()); + assertThat(((DataSplit) splits.get(0)).dataFiles().size()).isEqualTo(2); + TableRead read = table.newRead().withFilter(builder.equal(2, 100L)); + assertThat(getResult(read, splits, BATCH_ROW_TO_STRING)) + .hasSameElementsAs( + Arrays.asList( + "1|1|100|binary|varbinary|mapKey:mapVal|multiset", + "1|2|100|binary|varbinary|mapKey:mapVal|multiset", + "1|5|100|binary|varbinary|mapKey:mapVal|multiset")); + } + + @Test + public void testDeletionVectorsWithBitmapFileIndexInMeta() throws Exception { + FileStoreTable table = + createFileStoreTable( + conf -> { + conf.set(BUCKET, 1); + conf.set(DELETION_VECTORS_ENABLED, true); + conf.set(TARGET_FILE_SIZE, MemorySize.ofBytes(1)); + conf.set(FILE_INDEX_IN_MANIFEST_THRESHOLD, MemorySize.ofMebiBytes(1)); + conf.set("file-index.bitmap.columns", "b"); + }); + + StreamTableWrite write = + table.newWrite(commitUser).withIOManager(new IOManagerImpl(tempDir.toString())); + StreamTableCommit commit = table.newCommit(commitUser); + + write.write(rowData(1, 1, 300L)); + write.write(rowData(1, 2, 400L)); + write.write(rowData(1, 3, 100L)); + write.write(rowData(1, 4, 100L)); + commit.commit(0, write.prepareCommit(true, 0)); + + write.write(rowData(1, 1, 100L)); + write.write(rowData(1, 2, 100L)); + write.write(rowData(1, 3, 300L)); + write.write(rowData(1, 5, 100L)); + commit.commit(1, write.prepareCommit(true, 1)); + + write.write(rowData(1, 4, 200L)); + commit.commit(2, write.prepareCommit(true, 2)); + + PredicateBuilder builder = new PredicateBuilder(ROW_TYPE); + List splits = toSplits(table.newSnapshotReader().read().dataSplits()); + assertThat(((DataSplit) splits.get(0)).dataFiles().size()).isEqualTo(2); + TableRead read = table.newRead().withFilter(builder.equal(2, 100L)); + assertThat(getResult(read, splits, BATCH_ROW_TO_STRING)) + .hasSameElementsAs( + Arrays.asList( + "1|1|100|binary|varbinary|mapKey:mapVal|multiset", + "1|2|100|binary|varbinary|mapKey:mapVal|multiset", + "1|5|100|binary|varbinary|mapKey:mapVal|multiset")); + } + @Test public void testWithShardFirstRow() throws Exception { FileStoreTable table = @@ -1041,6 +1313,90 @@ public void testPartialUpdateRemoveRecordOnDelete() throws Exception { commit.close(); } + @Test + public void testPartialUpdateRemoveRecordOnSequenceGroup() throws Exception { + RowType rowType = + RowType.of( + new DataType[] { + DataTypes.INT(), + DataTypes.INT(), + DataTypes.INT(), + DataTypes.INT(), + DataTypes.INT(), + DataTypes.INT(), + DataTypes.INT() + }, + new String[] {"pt", "a", "b", "seq1", "c", "d", "seq2"}); + FileStoreTable table = + createFileStoreTable( + options -> { + options.set("merge-engine", "partial-update"); + options.set("fields.seq1.sequence-group", "b"); + options.set("fields.seq2.sequence-group", "c,d"); + options.set("partial-update.remove-record-on-sequence-group", "seq2"); + }, + rowType); + FileStoreTable wrongTable = + createFileStoreTable( + options -> { + options.set("merge-engine", "partial-update"); + options.set("fields.seq1.sequence-group", "b"); + options.set("fields.seq2.sequence-group", "c,d"); + options.set("partial-update.remove-record-on-sequence-group", "b"); + }, + rowType); + Function rowToString = row -> internalRowToString(row, rowType); + + assertThatThrownBy(() -> wrongTable.newWrite("")) + .hasMessageContaining( + "field 'b' defined in 'partial-update.remove-record-on-sequence-group' option must be part of sequence groups"); + + SnapshotReader snapshotReader = table.newSnapshotReader(); + TableRead read = table.newRead(); + StreamTableWrite write = table.newWrite(""); + StreamTableCommit commit = table.newCommit(""); + // 1. Inserts + write.write(GenericRow.of(1, 1, 10, 1, 20, 20, 1)); + write.write(GenericRow.of(1, 1, 11, 2, 25, 25, 0)); + write.write(GenericRow.of(1, 1, 12, 1, 29, 29, 2)); + commit.commit(0, write.prepareCommit(true, 0)); + List result = + getResult(read, toSplits(snapshotReader.read().dataSplits()), rowToString); + assertThat(result).containsExactlyInAnyOrder("+I[1, 1, 11, 2, 29, 29, 2]"); + + // 2. Update Before + write.write(GenericRow.ofKind(RowKind.UPDATE_BEFORE, 1, 1, 11, 2, 29, 29, 2)); + commit.commit(1, write.prepareCommit(true, 1)); + result = getResult(read, toSplits(snapshotReader.read().dataSplits()), rowToString); + assertThat(result).containsExactlyInAnyOrder("+I[1, 1, NULL, 2, NULL, NULL, 2]"); + + // 3. Update After + write.write(GenericRow.ofKind(RowKind.UPDATE_AFTER, 1, 1, 11, 2, 30, 30, 3)); + commit.commit(2, write.prepareCommit(true, 2)); + result = getResult(read, toSplits(snapshotReader.read().dataSplits()), rowToString); + assertThat(result).containsExactlyInAnyOrder("+I[1, 1, 11, 2, 30, 30, 3]"); + + // 4. Retracts + write.write(GenericRow.ofKind(RowKind.DELETE, 1, 1, 12, 3, 30, 30, 2)); + commit.commit(3, write.prepareCommit(true, 3)); + result = getResult(read, toSplits(snapshotReader.read().dataSplits()), rowToString); + assertThat(result).containsExactlyInAnyOrder("+I[1, 1, NULL, 3, 30, 30, 3]"); + + write.write(GenericRow.ofKind(RowKind.DELETE, 1, 1, 12, 2, 30, 31, 5)); + commit.commit(4, write.prepareCommit(true, 4)); + result = getResult(read, toSplits(snapshotReader.read().dataSplits()), rowToString); + assertThat(result).isEmpty(); + + // 5. Inserts + write.write(GenericRow.of(1, 1, 11, 2, 30, 31, 6)); + commit.commit(5, write.prepareCommit(true, 5)); + result = getResult(read, toSplits(snapshotReader.read().dataSplits()), rowToString); + assertThat(result).containsExactlyInAnyOrder("+I[1, 1, 11, 2, 30, 31, 6]"); + + write.close(); + commit.close(); + } + @Test public void testPartialUpdateWithAgg() throws Exception { RowType rowType = @@ -1416,15 +1772,22 @@ public void testStreamingReadOptimizedTable() throws Exception { .hasMessage("Unsupported streaming scan for read optimized table"); } - @Test - public void testReadDeletionVectorTable() throws Exception { - FileStoreTable table = - createFileStoreTable( - options -> { - // let level has many files - options.set(TARGET_FILE_SIZE, new MemorySize(1)); - options.set(DELETION_VECTORS_ENABLED, true); - }); + @ParameterizedTest + @ValueSource(booleans = {true, false}) + public void testReadDeletionVectorTable(boolean statsDenseStore) throws Exception { + Consumer optionsSetter = + options -> { + // let level has many files + options.set(TARGET_FILE_SIZE, new MemorySize(1)); + options.set(DELETION_VECTORS_ENABLED, true); + + if (statsDenseStore) { + options.set(CoreOptions.METADATA_STATS_MODE, "none"); + options.set("fields.b.stats-mode", "full"); + } + }; + + FileStoreTable table = createFileStoreTable(optionsSetter); StreamTableWrite write = table.newWrite(commitUser); IOManager ioManager = IOManager.create(tablePath.toString()); write.withIOManager(ioManager); diff --git a/paimon-core/src/test/java/org/apache/paimon/table/PrimaryKeyTableColumnTypeFileMetaTest.java b/paimon-core/src/test/java/org/apache/paimon/table/PrimaryKeyTableColumnTypeFileMetaTest.java index 45b67842b985..32a4138be564 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/PrimaryKeyTableColumnTypeFileMetaTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/PrimaryKeyTableColumnTypeFileMetaTest.java @@ -26,8 +26,8 @@ import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.schema.TableSchema; import org.apache.paimon.stats.SimpleStats; -import org.apache.paimon.stats.SimpleStatsConverter; -import org.apache.paimon.stats.SimpleStatsConverters; +import org.apache.paimon.stats.SimpleStatsEvolution; +import org.apache.paimon.stats.SimpleStatsEvolutions; import org.apache.paimon.table.source.DataSplit; import org.apache.paimon.types.DataField; @@ -54,7 +54,7 @@ protected FileStoreTable createFileStoreTable(Map tableSchema SchemaManager schemaManager = new TestingSchemaManager(tablePath, tableSchemas); return new PrimaryKeyFileStoreTable(fileIO, tablePath, schemaManager.latest().get()) { @Override - protected SchemaManager schemaManager() { + public SchemaManager schemaManager() { return schemaManager; } }; @@ -122,12 +122,13 @@ protected void validateValuesWithNewSchema( List fileMetaList) { Function> schemaFields = id -> tableSchemas.get(id).logicalTrimmedPrimaryKeysType().getFields(); - SimpleStatsConverters converters = new SimpleStatsConverters(schemaFields, schemaId); + SimpleStatsEvolutions converters = new SimpleStatsEvolutions(schemaFields, schemaId); for (DataFileMeta fileMeta : fileMetaList) { SimpleStats stats = getTableValueStats(fileMeta); - SimpleStatsConverter serializer = converters.getOrCreate(fileMeta.schemaId()); - InternalRow min = serializer.evolution(stats.minValues()); - InternalRow max = serializer.evolution(stats.maxValues()); + SimpleStatsEvolution.Result result = + converters.getOrCreate(fileMeta.schemaId()).evolution(stats, null, null); + InternalRow min = result.minValues(); + InternalRow max = result.maxValues(); assertThat(min.getFieldCount()).isEqualTo(4); if (filesName.contains(fileMeta.fileName())) { // parquet does not support padding diff --git a/paimon-core/src/test/java/org/apache/paimon/table/RecordLevelExpireWithMillisecondTest.java b/paimon-core/src/test/java/org/apache/paimon/table/RecordLevelExpireWithMillisecondTest.java index 14ec6885c608..15b6556c3481 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/RecordLevelExpireWithMillisecondTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/RecordLevelExpireWithMillisecondTest.java @@ -69,8 +69,6 @@ protected Options tableOptions() { options.set(CoreOptions.BUCKET, 1); options.set(CoreOptions.RECORD_LEVEL_EXPIRE_TIME, Duration.ofSeconds(1)); options.set(CoreOptions.RECORD_LEVEL_TIME_FIELD, "col1"); - options.set( - CoreOptions.RECORD_LEVEL_TIME_FIELD_TYPE, CoreOptions.TimeFieldType.MILLIS_LONG); return options; } diff --git a/paimon-core/src/test/java/org/apache/paimon/table/RecordLevelExpireWithTimestampBaseTest.java b/paimon-core/src/test/java/org/apache/paimon/table/RecordLevelExpireWithTimestampBaseTest.java new file mode 100644 index 000000000000..abcb8c1c76ca --- /dev/null +++ b/paimon-core/src/test/java/org/apache/paimon/table/RecordLevelExpireWithTimestampBaseTest.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.table; + +import org.apache.paimon.CoreOptions; +import org.apache.paimon.catalog.PrimaryKeyTableTestBase; +import org.apache.paimon.data.GenericRow; +import org.apache.paimon.data.Timestamp; +import org.apache.paimon.options.Options; + +import org.junit.jupiter.api.Test; + +import java.time.Duration; + +import static org.assertj.core.api.Assertions.assertThat; + +abstract class RecordLevelExpireWithTimestampBaseTest extends PrimaryKeyTableTestBase { + + @Override + protected Options tableOptions() { + Options options = new Options(); + options.set(CoreOptions.BUCKET, 1); + options.set(CoreOptions.RECORD_LEVEL_EXPIRE_TIME, Duration.ofSeconds(1)); + options.set(CoreOptions.RECORD_LEVEL_TIME_FIELD, "col1"); + return options; + } + + @Test + public void testTimestampTypeExpire() throws Exception { + long millis = System.currentTimeMillis(); + Timestamp timestamp1 = Timestamp.fromEpochMillis(millis - 60 * 1000); + Timestamp timestamp2 = Timestamp.fromEpochMillis(millis); + Timestamp timestamp3 = Timestamp.fromEpochMillis(millis + 60 * 1000); + + // create at least two files in one bucket + writeCommit(GenericRow.of(1, 1, timestamp1), GenericRow.of(1, 2, timestamp2)); + writeCommit(GenericRow.of(1, 3, timestamp3)); + + // no compaction, can be queried + assertThat(query(new int[] {0, 1})) + .containsExactlyInAnyOrder( + GenericRow.of(1, 1), GenericRow.of(1, 2), GenericRow.of(1, 3)); + Thread.sleep(2000); + + // compact, expired + compact(1); + assertThat(query(new int[] {0, 1})).containsExactlyInAnyOrder(GenericRow.of(1, 3)); + } +} diff --git a/paimon-core/src/test/java/org/apache/paimon/table/RecordLevelExpireWithTimestampLTZTest.java b/paimon-core/src/test/java/org/apache/paimon/table/RecordLevelExpireWithTimestampLTZTest.java new file mode 100644 index 000000000000..af834af276c4 --- /dev/null +++ b/paimon-core/src/test/java/org/apache/paimon/table/RecordLevelExpireWithTimestampLTZTest.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.table; + +import org.apache.paimon.catalog.Catalog; +import org.apache.paimon.catalog.CatalogContext; +import org.apache.paimon.catalog.CatalogFactory; +import org.apache.paimon.catalog.Identifier; +import org.apache.paimon.fs.Path; +import org.apache.paimon.schema.Schema; +import org.apache.paimon.types.DataTypes; +import org.apache.paimon.utils.TraceableFileIO; + +import org.junit.jupiter.api.BeforeEach; + +import java.util.UUID; + +class RecordLevelExpireWithTimestampLTZTest extends RecordLevelExpireWithTimestampBaseTest { + + @Override + @BeforeEach + public void beforeEachBase() throws Exception { + CatalogContext context = + CatalogContext.create( + new Path(TraceableFileIO.SCHEME + "://" + tempPath.toString())); + Catalog catalog = CatalogFactory.createCatalog(context); + Identifier identifier = new Identifier("default", "T"); + catalog.createDatabase(identifier.getDatabaseName(), true); + Schema schema = + Schema.newBuilder() + .column("pt", DataTypes.INT()) + .column("pk", DataTypes.INT()) + .column("col1", DataTypes.TIMESTAMP_WITH_LOCAL_TIME_ZONE()) + .partitionKeys("pt") + .primaryKey("pk", "pt") + .options(tableOptions().toMap()) + .build(); + catalog.createTable(identifier, schema, true); + table = (FileStoreTable) catalog.getTable(identifier); + commitUser = UUID.randomUUID().toString(); + } +} diff --git a/paimon-core/src/test/java/org/apache/paimon/table/RecordLevelExpireWithTimestampTest.java b/paimon-core/src/test/java/org/apache/paimon/table/RecordLevelExpireWithTimestampTest.java new file mode 100644 index 000000000000..3c4add8914f8 --- /dev/null +++ b/paimon-core/src/test/java/org/apache/paimon/table/RecordLevelExpireWithTimestampTest.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.table; + +import org.apache.paimon.catalog.Catalog; +import org.apache.paimon.catalog.CatalogContext; +import org.apache.paimon.catalog.CatalogFactory; +import org.apache.paimon.catalog.Identifier; +import org.apache.paimon.fs.Path; +import org.apache.paimon.schema.Schema; +import org.apache.paimon.types.DataTypes; +import org.apache.paimon.utils.TraceableFileIO; + +import org.junit.jupiter.api.BeforeEach; + +import java.util.UUID; + +class RecordLevelExpireWithTimestampTest extends RecordLevelExpireWithTimestampBaseTest { + + @Override + @BeforeEach + public void beforeEachBase() throws Exception { + CatalogContext context = + CatalogContext.create( + new Path(TraceableFileIO.SCHEME + "://" + tempPath.toString())); + Catalog catalog = CatalogFactory.createCatalog(context); + Identifier identifier = new Identifier("default", "T"); + catalog.createDatabase(identifier.getDatabaseName(), true); + Schema schema = + Schema.newBuilder() + .column("pt", DataTypes.INT()) + .column("pk", DataTypes.INT()) + .column("col1", DataTypes.TIMESTAMP()) + .partitionKeys("pt") + .primaryKey("pk", "pt") + .options(tableOptions().toMap()) + .build(); + catalog.createTable(identifier, schema, true); + table = (FileStoreTable) catalog.getTable(identifier); + commitUser = UUID.randomUUID().toString(); + } +} diff --git a/paimon-core/src/test/java/org/apache/paimon/table/SchemaEvolutionTest.java b/paimon-core/src/test/java/org/apache/paimon/table/SchemaEvolutionTest.java index 2b20b25c0b8b..951539299cbb 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/SchemaEvolutionTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/SchemaEvolutionTest.java @@ -41,7 +41,10 @@ import org.apache.paimon.types.DataTypes; import org.apache.paimon.types.RowType; +import org.apache.paimon.shade.guava30.com.google.common.collect.ImmutableList; +import org.apache.paimon.shade.guava30.com.google.common.collect.ImmutableMap; import org.apache.paimon.shade.guava30.com.google.common.collect.Lists; +import org.apache.paimon.shade.guava30.com.google.common.collect.Maps; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -57,8 +60,8 @@ import java.util.UUID; import java.util.function.Consumer; -import static org.apache.paimon.table.SystemFields.KEY_FIELD_PREFIX; -import static org.apache.paimon.table.SystemFields.SYSTEM_FIELD_NAMES; +import static org.apache.paimon.table.SpecialFields.KEY_FIELD_PREFIX; +import static org.apache.paimon.table.SpecialFields.SYSTEM_FIELD_NAMES; import static org.apache.paimon.testutils.assertj.PaimonAssertions.anyCauseMatches; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -350,7 +353,52 @@ public void testRenameField() throws Exception { .hasMessage( String.format( "Column %s already exists in the %s table.", - "f0", identifier.getFullName())); + "f1", identifier.getFullName())); + } + + @Test + public void testRenamePrimaryKeyColumn() throws Exception { + Schema schema = + new Schema( + RowType.of(DataTypes.INT(), DataTypes.BIGINT()).getFields(), + Lists.newArrayList("f0"), + Lists.newArrayList("f0", "f1"), + Maps.newHashMap(), + ""); + + schemaManager.createTable(schema); + assertThat(schemaManager.latest().get().fieldNames()).containsExactly("f0", "f1"); + + schemaManager.commitChanges(SchemaChange.renameColumn("f1", "f1_")); + TableSchema newSchema = schemaManager.latest().get(); + assertThat(newSchema.fieldNames()).containsExactly("f0", "f1_"); + assertThat(newSchema.primaryKeys()).containsExactlyInAnyOrder("f0", "f1_"); + + assertThatThrownBy( + () -> schemaManager.commitChanges(SchemaChange.renameColumn("f0", "f0_"))) + .isInstanceOf(UnsupportedOperationException.class) + .hasMessage("Cannot rename partition column: [f0]"); + } + + @Test + public void testRenameBucketKeyColumn() throws Exception { + Schema schema = + new Schema( + RowType.of(DataTypes.INT(), DataTypes.BIGINT()).getFields(), + ImmutableList.of(), + Lists.newArrayList("f0", "f1"), + ImmutableMap.of( + CoreOptions.BUCKET_KEY.key(), + "f1,f0", + CoreOptions.BUCKET.key(), + "16"), + ""); + + schemaManager.createTable(schema); + schemaManager.commitChanges(SchemaChange.renameColumn("f0", "f0_")); + TableSchema newSchema = schemaManager.latest().get(); + + assertThat(newSchema.options().get(CoreOptions.BUCKET_KEY.key())).isEqualTo("f1,f0_"); } @Test @@ -379,14 +427,14 @@ public void testDropField() throws Exception { schemaManager.commitChanges( Collections.singletonList(SchemaChange.dropColumn("f0")))) .isInstanceOf(UnsupportedOperationException.class) - .hasMessage(String.format("Cannot drop/rename partition key[%s]", "f0")); + .hasMessage(String.format("Cannot drop partition key or primary key: [%s]", "f0")); assertThatThrownBy( () -> schemaManager.commitChanges( Collections.singletonList(SchemaChange.dropColumn("f2")))) .isInstanceOf(UnsupportedOperationException.class) - .hasMessage(String.format("Cannot drop/rename primary key[%s]", "f2")); + .hasMessage(String.format("Cannot drop partition key or primary key: [%s]", "f2")); } @Test diff --git a/paimon-core/src/test/java/org/apache/paimon/table/TableTestBase.java b/paimon-core/src/test/java/org/apache/paimon/table/TableTestBase.java index eaaf8ca70bc8..7d7617cf8bd1 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/TableTestBase.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/TableTestBase.java @@ -57,6 +57,7 @@ import java.util.Map; import java.util.Random; import java.util.UUID; +import java.util.function.Function; import java.util.function.Predicate; import static org.assertj.core.api.Assertions.assertThat; @@ -110,6 +111,19 @@ protected final void write(Table table, Pair... rows) thro } } + protected void writeWithBucketAssigner( + Table table, Function bucketAssigner, InternalRow... rows) + throws Exception { + BatchWriteBuilder writeBuilder = table.newBatchWriteBuilder(); + try (BatchTableWrite write = writeBuilder.newWrite(); + BatchTableCommit commit = writeBuilder.newCommit()) { + for (InternalRow row : rows) { + write.write(row, bucketAssigner.apply(row)); + } + commit.commit(write.prepareCommit()); + } + } + protected void write(Table table, InternalRow... rows) throws Exception { write(table, null, rows); } @@ -146,6 +160,10 @@ protected void compact( } } + public void dropTableDefault() throws Exception { + catalog.dropTable(identifier(), true); + } + protected List read(Table table, Pair, String>... dynamicOptions) throws Exception { return read(table, null, dynamicOptions); diff --git a/paimon-core/src/test/java/org/apache/paimon/table/sink/CommitMessageSerializerTest.java b/paimon-core/src/test/java/org/apache/paimon/table/sink/CommitMessageSerializerTest.java index eb9105189b71..1f87838aea31 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/sink/CommitMessageSerializerTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/sink/CommitMessageSerializerTest.java @@ -48,7 +48,7 @@ public void test() throws IOException { CommitMessageImpl committable = new CommitMessageImpl(row(0), 1, dataIncrement, compactIncrement, indexIncrement); CommitMessageImpl newCommittable = - (CommitMessageImpl) serializer.deserialize(3, serializer.serialize(committable)); + (CommitMessageImpl) serializer.deserialize(5, serializer.serialize(committable)); assertThat(newCommittable.compactIncrement()).isEqualTo(committable.compactIncrement()); assertThat(newCommittable.newFilesIncrement()).isEqualTo(committable.newFilesIncrement()); assertThat(newCommittable.indexIncrement()).isEqualTo(committable.indexIncrement()); diff --git a/paimon-core/src/test/java/org/apache/paimon/table/source/SplitGeneratorTest.java b/paimon-core/src/test/java/org/apache/paimon/table/source/SplitGeneratorTest.java index 14faaa671096..a1f7d69e2877 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/source/SplitGeneratorTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/source/SplitGeneratorTest.java @@ -43,10 +43,10 @@ public class SplitGeneratorTest { public static DataFileMeta newFileFromSequence( - String name, int rowCount, long minSequence, long maxSequence) { + String name, int fileSize, long minSequence, long maxSequence) { return new DataFileMeta( name, - rowCount, + fileSize, 1, EMPTY_ROW, EMPTY_ROW, @@ -58,7 +58,8 @@ public static DataFileMeta newFileFromSequence( 0, 0L, null, - FileSource.APPEND); + FileSource.APPEND, + null); } @Test diff --git a/paimon-core/src/test/java/org/apache/paimon/table/source/SplitTest.java b/paimon-core/src/test/java/org/apache/paimon/table/source/SplitTest.java index 4bc9b6f8b15a..0219941a0ac0 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/source/SplitTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/source/SplitTest.java @@ -26,6 +26,7 @@ import org.apache.paimon.io.DataFileTestDataGenerator; import org.apache.paimon.io.DataInputDeserializer; import org.apache.paimon.io.DataOutputViewStreamWrapper; +import org.apache.paimon.manifest.FileSource; import org.apache.paimon.stats.SimpleStats; import org.apache.paimon.utils.IOUtils; import org.apache.paimon.utils.InstantiationUtil; @@ -48,6 +49,41 @@ /** Test for {@link DataSplit}. */ public class SplitTest { + @Test + public void testSplitMergedRowCount() { + // not rawConvertible + List dataFiles = + Arrays.asList(newDataFile(1000L), newDataFile(2000L), newDataFile(3000L)); + DataSplit split = newDataSplit(false, dataFiles, null); + assertThat(split.partialMergedRowCount()).isEqualTo(0L); + assertThat(split.mergedRowCountAvailable()).isEqualTo(false); + + // rawConvertible without deletion files + split = newDataSplit(true, dataFiles, null); + assertThat(split.partialMergedRowCount()).isEqualTo(6000L); + assertThat(split.mergedRowCountAvailable()).isEqualTo(true); + assertThat(split.mergedRowCount()).isEqualTo(6000L); + + // rawConvertible with deletion files without cardinality + ArrayList deletionFiles = new ArrayList<>(); + deletionFiles.add(null); + deletionFiles.add(new DeletionFile("p", 1, 2, null)); + deletionFiles.add(new DeletionFile("p", 1, 2, 100L)); + split = newDataSplit(true, dataFiles, deletionFiles); + assertThat(split.partialMergedRowCount()).isEqualTo(3900L); + assertThat(split.mergedRowCountAvailable()).isEqualTo(false); + + // rawConvertible with deletion files with cardinality + deletionFiles = new ArrayList<>(); + deletionFiles.add(null); + deletionFiles.add(new DeletionFile("p", 1, 2, 200L)); + deletionFiles.add(new DeletionFile("p", 1, 2, 100L)); + split = newDataSplit(true, dataFiles, deletionFiles); + assertThat(split.partialMergedRowCount()).isEqualTo(5700L); + assertThat(split.mergedRowCountAvailable()).isEqualTo(true); + assertThat(split.mergedRowCount()).isEqualTo(5700L); + } + @Test public void testSerializer() throws IOException { DataFileTestDataGenerator gen = DataFileTestDataGenerator.builder().build(); @@ -73,7 +109,7 @@ public void testSerializer() throws IOException { } @Test - public void testSerializerCompatible() throws Exception { + public void testSerializerNormal() throws Exception { SimpleStats keyStats = new SimpleStats( singleColumn("min_key"), @@ -102,6 +138,62 @@ public void testSerializerCompatible() throws Exception { Timestamp.fromLocalDateTime(LocalDateTime.parse("2022-03-02T20:20:12")), 11L, new byte[] {1, 2, 4}, + FileSource.COMPACT, + Arrays.asList("field1", "field2", "field3")); + List dataFiles = Collections.singletonList(dataFile); + + DeletionFile deletionFile = new DeletionFile("deletion_file", 100, 22, 33L); + List deletionFiles = Collections.singletonList(deletionFile); + + BinaryRow partition = new BinaryRow(1); + BinaryRowWriter binaryRowWriter = new BinaryRowWriter(partition); + binaryRowWriter.writeString(0, BinaryString.fromString("aaaaa")); + binaryRowWriter.complete(); + + DataSplit split = + DataSplit.builder() + .withSnapshot(18) + .withPartition(partition) + .withBucket(20) + .withDataFiles(dataFiles) + .withDataDeletionFiles(deletionFiles) + .withBucketPath("my path") + .build(); + + assertThat(InstantiationUtil.clone(split)).isEqualTo(split); + } + + @Test + public void testSerializerCompatibleV1() throws Exception { + SimpleStats keyStats = + new SimpleStats( + singleColumn("min_key"), + singleColumn("max_key"), + fromLongArray(new Long[] {0L})); + SimpleStats valueStats = + new SimpleStats( + singleColumn("min_value"), + singleColumn("max_value"), + fromLongArray(new Long[] {0L})); + + DataFileMeta dataFile = + new DataFileMeta( + "my_file", + 1024 * 1024, + 1024, + singleColumn("min_key"), + singleColumn("max_key"), + keyStats, + valueStats, + 15, + 200, + 5, + 3, + Arrays.asList("extra1", "extra2"), + Timestamp.fromLocalDateTime(LocalDateTime.parse("2022-03-02T20:20:12")), + 11L, + new byte[] {1, 2, 4}, + null, null); List dataFiles = Collections.singletonList(dataFile); @@ -130,4 +222,160 @@ public void testSerializerCompatible() throws Exception { InstantiationUtil.deserializeObject(v2Bytes, DataSplit.class.getClassLoader()); assertThat(actual).isEqualTo(split); } + + @Test + public void testSerializerCompatibleV2() throws Exception { + SimpleStats keyStats = + new SimpleStats( + singleColumn("min_key"), + singleColumn("max_key"), + fromLongArray(new Long[] {0L})); + SimpleStats valueStats = + new SimpleStats( + singleColumn("min_value"), + singleColumn("max_value"), + fromLongArray(new Long[] {0L})); + + DataFileMeta dataFile = + new DataFileMeta( + "my_file", + 1024 * 1024, + 1024, + singleColumn("min_key"), + singleColumn("max_key"), + keyStats, + valueStats, + 15, + 200, + 5, + 3, + Arrays.asList("extra1", "extra2"), + Timestamp.fromLocalDateTime(LocalDateTime.parse("2022-03-02T20:20:12")), + 11L, + new byte[] {1, 2, 4}, + FileSource.COMPACT, + null); + List dataFiles = Collections.singletonList(dataFile); + + BinaryRow partition = new BinaryRow(1); + BinaryRowWriter binaryRowWriter = new BinaryRowWriter(partition); + binaryRowWriter.writeString(0, BinaryString.fromString("aaaaa")); + binaryRowWriter.complete(); + + DataSplit split = + DataSplit.builder() + .withSnapshot(18) + .withPartition(partition) + .withBucket(20) + .withDataFiles(dataFiles) + .withBucketPath("my path") + .build(); + + byte[] v2Bytes = + IOUtils.readFully( + SplitTest.class + .getClassLoader() + .getResourceAsStream("compatibility/datasplit-v2"), + true); + + DataSplit actual = + InstantiationUtil.deserializeObject(v2Bytes, DataSplit.class.getClassLoader()); + assertThat(actual).isEqualTo(split); + } + + @Test + public void testSerializerCompatibleV3() throws Exception { + SimpleStats keyStats = + new SimpleStats( + singleColumn("min_key"), + singleColumn("max_key"), + fromLongArray(new Long[] {0L})); + SimpleStats valueStats = + new SimpleStats( + singleColumn("min_value"), + singleColumn("max_value"), + fromLongArray(new Long[] {0L})); + + DataFileMeta dataFile = + new DataFileMeta( + "my_file", + 1024 * 1024, + 1024, + singleColumn("min_key"), + singleColumn("max_key"), + keyStats, + valueStats, + 15, + 200, + 5, + 3, + Arrays.asList("extra1", "extra2"), + Timestamp.fromLocalDateTime(LocalDateTime.parse("2022-03-02T20:20:12")), + 11L, + new byte[] {1, 2, 4}, + FileSource.COMPACT, + Arrays.asList("field1", "field2", "field3")); + List dataFiles = Collections.singletonList(dataFile); + + DeletionFile deletionFile = new DeletionFile("deletion_file", 100, 22, null); + List deletionFiles = Collections.singletonList(deletionFile); + + BinaryRow partition = new BinaryRow(1); + BinaryRowWriter binaryRowWriter = new BinaryRowWriter(partition); + binaryRowWriter.writeString(0, BinaryString.fromString("aaaaa")); + binaryRowWriter.complete(); + + DataSplit split = + DataSplit.builder() + .withSnapshot(18) + .withPartition(partition) + .withBucket(20) + .withDataFiles(dataFiles) + .withDataDeletionFiles(deletionFiles) + .withBucketPath("my path") + .build(); + + byte[] v2Bytes = + IOUtils.readFully( + SplitTest.class + .getClassLoader() + .getResourceAsStream("compatibility/datasplit-v3"), + true); + + DataSplit actual = + InstantiationUtil.deserializeObject(v2Bytes, DataSplit.class.getClassLoader()); + assertThat(actual).isEqualTo(split); + } + + private DataFileMeta newDataFile(long rowCount) { + return DataFileMeta.forAppend( + "my_data_file.parquet", + 1024 * 1024, + rowCount, + null, + 0L, + rowCount, + 1, + Collections.emptyList(), + null, + null, + null); + } + + private DataSplit newDataSplit( + boolean rawConvertible, + List dataFiles, + List deletionFiles) { + DataSplit.Builder builder = DataSplit.builder(); + builder.withSnapshot(1) + .withPartition(BinaryRow.EMPTY_ROW) + .withBucket(1) + .withBucketPath("my path") + .rawConvertible(rawConvertible) + .withDataFiles(dataFiles); + if (deletionFiles != null) { + builder.withDataDeletionFiles(deletionFiles); + } + return builder.build(); + } } diff --git a/paimon-core/src/test/java/org/apache/paimon/table/source/snapshot/TimeTravelUtilsTest.java b/paimon-core/src/test/java/org/apache/paimon/table/source/snapshot/TimeTravelUtilsTest.java new file mode 100644 index 000000000000..4d5ea873f108 --- /dev/null +++ b/paimon-core/src/test/java/org/apache/paimon/table/source/snapshot/TimeTravelUtilsTest.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.table.source.snapshot; + +import org.apache.paimon.CoreOptions; +import org.apache.paimon.Snapshot; +import org.apache.paimon.table.sink.StreamTableCommit; +import org.apache.paimon.table.sink.StreamTableWrite; +import org.apache.paimon.utils.SnapshotManager; + +import org.junit.jupiter.api.Test; + +import java.util.HashMap; + +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertThrows; + +/** Tests for {@link TimeTravelUtil}. */ +public class TimeTravelUtilsTest extends ScannerTestBase { + + @Test + public void testResolveSnapshotFromOptions() throws Exception { + SnapshotManager snapshotManager = table.snapshotManager(); + StreamTableWrite write = table.newWrite(commitUser); + StreamTableCommit commit = table.newCommit(commitUser); + + write.write(rowData(1, 10, 100L)); + commit.commit(0, write.prepareCommit(true, 0)); + + long ts = System.currentTimeMillis(); + + write.write(rowData(2, 30, 101L)); + commit.commit(1, write.prepareCommit(true, 1)); + + write.write(rowData(3, 50, 500L)); + commit.commit(2, write.prepareCommit(true, 2)); + + HashMap optMap = new HashMap<>(4); + optMap.put("scan.snapshot-id", "2"); + CoreOptions options = CoreOptions.fromMap(optMap); + Snapshot snapshot = TimeTravelUtil.resolveSnapshotFromOptions(options, snapshotManager); + assertNotNull(snapshot); + assertTrue(snapshot.id() == 2); + + optMap.clear(); + optMap.put("scan.timestamp-millis", ts + ""); + options = CoreOptions.fromMap(optMap); + snapshot = TimeTravelUtil.resolveSnapshotFromOptions(options, snapshotManager); + assertTrue(snapshot.id() == 1); + + table.createTag("tag3", 3); + optMap.clear(); + optMap.put("scan.tag-name", "tag3"); + options = CoreOptions.fromMap(optMap); + snapshot = TimeTravelUtil.resolveSnapshotFromOptions(options, snapshotManager); + assertTrue(snapshot.id() == 3); + + // if contain more scan.xxx config would throw out + optMap.put("scan.snapshot-id", "2"); + CoreOptions options1 = CoreOptions.fromMap(optMap); + assertThrows( + IllegalArgumentException.class, + () -> TimeTravelUtil.resolveSnapshotFromOptions(options1, snapshotManager), + "scan.snapshot-id scan.tag-name scan.watermark and scan.timestamp-millis can contains only one"); + write.close(); + commit.close(); + } +} diff --git a/paimon-core/src/test/java/org/apache/paimon/table/system/FilesTableTest.java b/paimon-core/src/test/java/org/apache/paimon/table/system/FilesTableTest.java index 89fb201faba2..f0280560c267 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/system/FilesTableTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/system/FilesTableTest.java @@ -191,7 +191,16 @@ private List getExpectedResult(long snapshotId) { BinaryString.fromString( Arrays.toString(new String[] {partition1, partition2})), fileEntry.bucket(), - BinaryString.fromString(file.fileName()), + BinaryString.fromString( + table.location() + + "/pt1=" + + partition1 + + "/pt2=" + + partition2 + + "/bucket-" + + fileEntry.bucket() + + "/" + + file.fileName()), BinaryString.fromString(file.fileFormat()), file.schemaId(), file.level(), @@ -211,7 +220,9 @@ private List getExpectedResult(long snapshotId) { maxCol1, maxKey, partition1, partition2)), file.minSequenceNumber(), file.maxSequenceNumber(), - file.creationTime())); + file.creationTime(), + BinaryString.fromString( + file.fileSource().map(Object::toString).orElse(null)))); } return expectedRow; } diff --git a/paimon-core/src/test/java/org/apache/paimon/table/system/ManifestsTableTest.java b/paimon-core/src/test/java/org/apache/paimon/table/system/ManifestsTableTest.java index 4125a3f83dfe..a39e6f6fa807 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/system/ManifestsTableTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/system/ManifestsTableTest.java @@ -136,13 +136,27 @@ public void testReadManifestsFromSpecifiedTagName() throws Exception { } @Test - public void testReadManifestsFromNotExistSnapshot() throws Exception { + public void testReadManifestsFromSpecifiedTimestampMillis() throws Exception { + write(table, GenericRow.of(3, 1, 1), GenericRow.of(3, 2, 1)); + List expectedRow = getExpectedResult(3L); + manifestsTable = + (ManifestsTable) + manifestsTable.copy( + Collections.singletonMap( + CoreOptions.SCAN_TIMESTAMP_MILLIS.key(), + String.valueOf(System.currentTimeMillis()))); + List result = read(manifestsTable); + assertThat(result).containsExactlyElementsOf(expectedRow); + } + + @Test + public void testReadManifestsFromNotExistSnapshot() { manifestsTable = (ManifestsTable) manifestsTable.copy( Collections.singletonMap(CoreOptions.SCAN_SNAPSHOT_ID.key(), "3")); assertThrows( - "Specified scan.snapshot-id 3 is not exist, you can set it in range from 1 to 2", + "Specified parameter scan.snapshot-id = 3 is not exist, you can set it in range from 1 to 2", SnapshotNotExistException.class, () -> read(manifestsTable)); } diff --git a/paimon-core/src/test/java/org/apache/paimon/table/system/PartitionsTableTest.java b/paimon-core/src/test/java/org/apache/paimon/table/system/PartitionsTableTest.java index a17dc75466a6..8d12dc707bf5 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/system/PartitionsTableTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/system/PartitionsTableTest.java @@ -40,6 +40,7 @@ import org.junit.jupiter.api.Test; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import static org.assertj.core.api.Assertions.assertThat; @@ -76,7 +77,7 @@ public void before() throws Exception { partitionsTable = (PartitionsTable) catalog.getTable(filesTableId); // snapshot 1: append - write(table, GenericRow.of(1, 1, 1), GenericRow.of(1, 2, 5)); + write(table, GenericRow.of(1, 1, 1), GenericRow.of(1, 3, 5)); write(table, GenericRow.of(1, 1, 3), GenericRow.of(1, 2, 4)); } @@ -85,19 +86,36 @@ public void before() throws Exception { public void testPartitionRecordCount() throws Exception { List expectedRow = new ArrayList<>(); expectedRow.add(GenericRow.of(BinaryString.fromString("[1]"), 2L)); - expectedRow.add(GenericRow.of(BinaryString.fromString("[2]"), 2L)); + expectedRow.add(GenericRow.of(BinaryString.fromString("[2]"), 1L)); + expectedRow.add(GenericRow.of(BinaryString.fromString("[3]"), 1L)); // Only read partition and record count, record size may not stable. List result = read(partitionsTable, new int[][] {{0}, {1}}); assertThat(result).containsExactlyInAnyOrderElementsOf(expectedRow); } + @Test + public void testPartitionTimeTravel() throws Exception { + List expectedRow = new ArrayList<>(); + expectedRow.add(GenericRow.of(BinaryString.fromString("[1]"), 1L)); + expectedRow.add(GenericRow.of(BinaryString.fromString("[3]"), 1L)); + + // Only read partition and record count, record size may not stable. + List result = + read( + partitionsTable.copy( + Collections.singletonMap(CoreOptions.SCAN_VERSION.key(), "1")), + new int[][] {{0}, {1}}); + assertThat(result).containsExactlyInAnyOrderElementsOf(expectedRow); + } + @Test public void testPartitionValue() throws Exception { write(table, GenericRow.of(2, 1, 3), GenericRow.of(3, 1, 4)); List expectedRow = new ArrayList<>(); expectedRow.add(GenericRow.of(BinaryString.fromString("[1]"), 4L, 3L)); - expectedRow.add(GenericRow.of(BinaryString.fromString("[2]"), 2L, 2L)); + expectedRow.add(GenericRow.of(BinaryString.fromString("[2]"), 1L, 1L)); + expectedRow.add(GenericRow.of(BinaryString.fromString("[3]"), 1L, 1L)); List result = read(partitionsTable, new int[][] {{0}, {1}, {3}}); assertThat(result).containsExactlyInAnyOrderElementsOf(expectedRow); diff --git a/paimon-core/src/test/java/org/apache/paimon/table/system/TagsTableTest.java b/paimon-core/src/test/java/org/apache/paimon/table/system/TagsTableTest.java index 8f8029cde785..7ee8cd53d75c 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/system/TagsTableTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/system/TagsTableTest.java @@ -40,6 +40,8 @@ import java.time.LocalDateTime; import java.util.ArrayList; import java.util.List; +import java.util.Map; +import java.util.TreeMap; import static org.assertj.core.api.Assertions.assertThat; @@ -89,11 +91,12 @@ void testTagsTable() throws Exception { } private List getExpectedResult() { - List internalRows = new ArrayList<>(); + Map tagToRows = new TreeMap<>(); for (Pair snapshot : tagManager.tagObjects()) { Tag tag = snapshot.getKey(); String tagName = snapshot.getValue(); - internalRows.add( + tagToRows.put( + tagName, GenericRow.of( BinaryString.fromString(tagName), tag.id(), @@ -109,6 +112,11 @@ private List getExpectedResult() { : BinaryString.fromString( tag.getTagTimeRetained().toString()))); } + + List internalRows = new ArrayList<>(); + for (Map.Entry entry : tagToRows.entrySet()) { + internalRows.add(entry.getValue()); + } return internalRows; } } diff --git a/paimon-core/src/test/java/org/apache/paimon/utils/FileStorePathFactoryTest.java b/paimon-core/src/test/java/org/apache/paimon/utils/FileStorePathFactoryTest.java index 994367d0cd06..6ca15cf1503d 100644 --- a/paimon-core/src/test/java/org/apache/paimon/utils/FileStorePathFactoryTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/utils/FileStorePathFactoryTest.java @@ -87,7 +87,11 @@ public void testCreateDataFilePathFactoryWithPartition() { "default", CoreOptions.FILE_FORMAT.defaultValue().toString(), CoreOptions.DATA_FILE_PREFIX.defaultValue(), - CoreOptions.CHANGELOG_FILE_PREFIX.defaultValue()); + CoreOptions.CHANGELOG_FILE_PREFIX.defaultValue(), + CoreOptions.PARTITION_GENERATE_LEGCY_NAME.defaultValue(), + CoreOptions.FILE_SUFFIX_INCLUDE_COMPRESSION.defaultValue(), + CoreOptions.FILE_COMPRESSION.defaultValue(), + null); assertPartition("20211224", 16, pathFactory, "/dt=20211224/hr=16"); assertPartition("20211224", null, pathFactory, "/dt=20211224/hr=default"); @@ -124,6 +128,10 @@ public static FileStorePathFactory createNonPartFactory(Path root) { PARTITION_DEFAULT_NAME.defaultValue(), CoreOptions.FILE_FORMAT.defaultValue().toString(), CoreOptions.DATA_FILE_PREFIX.defaultValue(), - CoreOptions.CHANGELOG_FILE_PREFIX.defaultValue()); + CoreOptions.CHANGELOG_FILE_PREFIX.defaultValue(), + CoreOptions.PARTITION_GENERATE_LEGCY_NAME.defaultValue(), + CoreOptions.FILE_SUFFIX_INCLUDE_COMPRESSION.defaultValue(), + CoreOptions.FILE_COMPRESSION.defaultValue(), + null); } } diff --git a/paimon-core/src/test/java/org/apache/paimon/utils/FormatReaderMappingTest.java b/paimon-core/src/test/java/org/apache/paimon/utils/FormatReaderMappingTest.java new file mode 100644 index 000000000000..dd00d142c83a --- /dev/null +++ b/paimon-core/src/test/java/org/apache/paimon/utils/FormatReaderMappingTest.java @@ -0,0 +1,146 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.utils; + +import org.apache.paimon.schema.IndexCastMapping; +import org.apache.paimon.schema.SchemaEvolutionUtil; +import org.apache.paimon.table.SpecialFields; +import org.apache.paimon.types.DataField; +import org.apache.paimon.types.DataTypes; +import org.apache.paimon.types.RowType; + +import org.assertj.core.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.List; + +/** Test for {@link FormatReaderMapping.Builder}. */ +public class FormatReaderMappingTest { + + @Test + public void testTrimKeyFields() { + List keyFields = new ArrayList<>(); + List allFields = new ArrayList<>(); + List testFields = new ArrayList<>(); + + for (int i = 0; i < 10; i++) { + keyFields.add( + new DataField( + SpecialFields.KEY_FIELD_ID_START + i, + SpecialFields.KEY_FIELD_PREFIX + i, + DataTypes.STRING())); + } + + allFields.addAll(keyFields); + for (int i = 0; i < 20; i++) { + allFields.add(new DataField(i, String.valueOf(i), DataTypes.STRING())); + } + + testFields.add( + new DataField( + SpecialFields.KEY_FIELD_ID_START + 1, + SpecialFields.KEY_FIELD_PREFIX + 1, + DataTypes.STRING())); + testFields.add( + new DataField( + SpecialFields.KEY_FIELD_ID_START + 3, + SpecialFields.KEY_FIELD_PREFIX + 3, + DataTypes.STRING())); + testFields.add( + new DataField( + SpecialFields.KEY_FIELD_ID_START + 5, + SpecialFields.KEY_FIELD_PREFIX + 5, + DataTypes.STRING())); + testFields.add( + new DataField( + SpecialFields.KEY_FIELD_ID_START + 7, + SpecialFields.KEY_FIELD_PREFIX + 7, + DataTypes.STRING())); + testFields.add(new DataField(3, String.valueOf(3), DataTypes.STRING())); + testFields.add(new DataField(4, String.valueOf(4), DataTypes.STRING())); + testFields.add(new DataField(5, String.valueOf(5), DataTypes.STRING())); + testFields.add(new DataField(1, String.valueOf(1), DataTypes.STRING())); + testFields.add(new DataField(6, String.valueOf(6), DataTypes.STRING())); + + Pair res = FormatReaderMapping.Builder.trimKeyFields(testFields, allFields); + + Assertions.assertThat(res.getKey()).containsExactly(0, 1, 2, 3, 1, 4, 2, 0, 5); + + List fields = res.getRight().getFields(); + Assertions.assertThat(fields.size()).isEqualTo(6); + Assertions.assertThat(fields.get(0).id()).isEqualTo(1); + Assertions.assertThat(fields.get(1).id()).isEqualTo(3); + Assertions.assertThat(fields.get(2).id()).isEqualTo(5); + Assertions.assertThat(fields.get(3).id()).isEqualTo(7); + Assertions.assertThat(fields.get(4).id()).isEqualTo(4); + Assertions.assertThat(fields.get(5).id()).isEqualTo(6); + } + + @Test + public void testTrimKeyWithIndexMapping() { + List readTableFields = new ArrayList<>(); + List readDataFields = new ArrayList<>(); + + readTableFields.add( + new DataField( + SpecialFields.KEY_FIELD_ID_START + 1, + SpecialFields.KEY_FIELD_PREFIX + "a", + DataTypes.STRING())); + readTableFields.add(new DataField(0, "0", DataTypes.STRING())); + readTableFields.add(new DataField(1, "a", DataTypes.STRING())); + readTableFields.add(new DataField(2, "2", DataTypes.STRING())); + readTableFields.add(new DataField(3, "3", DataTypes.STRING())); + + readDataFields.add( + new DataField( + SpecialFields.KEY_FIELD_ID_START + 1, + SpecialFields.KEY_FIELD_PREFIX + "a", + DataTypes.STRING())); + readDataFields.add(new DataField(0, "0", DataTypes.STRING())); + readDataFields.add(new DataField(1, "a", DataTypes.STRING())); + readDataFields.add(new DataField(3, "3", DataTypes.STRING())); + + // build index cast mapping + IndexCastMapping indexCastMapping = + SchemaEvolutionUtil.createIndexCastMapping(readTableFields, readDataFields); + + // map from key fields reading to value fields reading + Pair trimmedKeyPair = + FormatReaderMapping.Builder.trimKeyFields(readDataFields, readDataFields); + + FormatReaderMapping formatReaderMapping = + new FormatReaderMapping( + indexCastMapping.getIndexMapping(), + indexCastMapping.getCastMapping(), + trimmedKeyPair.getLeft(), + null, + null, + null, + null); + + Assertions.assertThat(formatReaderMapping.getIndexMapping()) + .containsExactly(0, 1, 0, -1, 2); + List trimmed = trimmedKeyPair.getRight().getFields(); + Assertions.assertThat(trimmed.get(0).id()).isEqualTo(1); + Assertions.assertThat(trimmed.get(1).id()).isEqualTo(0); + Assertions.assertThat(trimmed.get(2).id()).isEqualTo(3); + Assertions.assertThat(trimmed.size()).isEqualTo(3); + } +} diff --git a/paimon-core/src/test/java/org/apache/paimon/utils/ObjectsCacheTest.java b/paimon-core/src/test/java/org/apache/paimon/utils/ObjectsCacheTest.java index 8a4f0b0612e7..9d3275e3ab48 100644 --- a/paimon-core/src/test/java/org/apache/paimon/utils/ObjectsCacheTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/utils/ObjectsCacheTest.java @@ -58,17 +58,23 @@ public void test() throws IOException { // test empty map.put("k1", Collections.emptyList()); - List values = cache.read("k1", null, Filter.alwaysTrue(), Filter.alwaysTrue()); + List values = + cache.read( + "k1", null, Filter.alwaysTrue(), Filter.alwaysTrue(), Filter.alwaysTrue()); assertThat(values).isEmpty(); // test values List expect = Arrays.asList("v1", "v2", "v3"); map.put("k2", expect); - values = cache.read("k2", null, Filter.alwaysTrue(), Filter.alwaysTrue()); + values = + cache.read( + "k2", null, Filter.alwaysTrue(), Filter.alwaysTrue(), Filter.alwaysTrue()); assertThat(values).containsExactlyElementsOf(expect); // test cache - values = cache.read("k2", null, Filter.alwaysTrue(), Filter.alwaysTrue()); + values = + cache.read( + "k2", null, Filter.alwaysTrue(), Filter.alwaysTrue(), Filter.alwaysTrue()); assertThat(values).containsExactlyElementsOf(expect); // test filter @@ -77,7 +83,8 @@ public void test() throws IOException { "k2", null, Filter.alwaysTrue(), - r -> r.getString(0).toString().endsWith("2")); + r -> r.getString(0).toString().endsWith("2"), + Filter.alwaysTrue()); assertThat(values).containsExactly("v2"); // test load filter @@ -88,6 +95,7 @@ public void test() throws IOException { "k3", null, r -> r.getString(0).toString().endsWith("2"), + Filter.alwaysTrue(), Filter.alwaysTrue()); assertThat(values).containsExactly("v2"); @@ -99,6 +107,7 @@ public void test() throws IOException { "k4", null, r -> r.getString(0).toString().endsWith("5"), + Filter.alwaysTrue(), Filter.alwaysTrue()); assertThat(values).isEmpty(); @@ -117,6 +126,7 @@ public void test() throws IOException { k, null, Filter.alwaysTrue(), + Filter.alwaysTrue(), Filter.alwaysTrue())) .containsExactly(k); } catch (IOException e) { diff --git a/paimon-core/src/test/java/org/apache/paimon/utils/SnapshotManagerTest.java b/paimon-core/src/test/java/org/apache/paimon/utils/SnapshotManagerTest.java index 6b7b28263af0..26480cf411bb 100644 --- a/paimon-core/src/test/java/org/apache/paimon/utils/SnapshotManagerTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/utils/SnapshotManagerTest.java @@ -281,8 +281,8 @@ public void testLatestSnapshotOfUser() throws IOException, InterruptedException @Test public void testTraversalSnapshotsFromLatestSafely() throws IOException, InterruptedException { FileIO localFileIO = LocalFileIO.create(); - SnapshotManager snapshotManager = - new SnapshotManager(localFileIO, new Path(tempDir.toString())); + Path path = new Path(tempDir.toString()); + SnapshotManager snapshotManager = new SnapshotManager(localFileIO, path); // create 10 snapshots for (long i = 0; i < 10; i++) { Snapshot snapshot = diff --git a/paimon-core/src/test/resources/META-INF/services/org.apache.paimon.factories.Factory b/paimon-core/src/test/resources/META-INF/services/org.apache.paimon.factories.Factory new file mode 100644 index 000000000000..7eb517ab9835 --- /dev/null +++ b/paimon-core/src/test/resources/META-INF/services/org.apache.paimon.factories.Factory @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +org.apache.paimon.mergetree.compact.aggregate.TestCustomAggFactory \ No newline at end of file diff --git a/paimon-core/src/test/resources/compatibility/datasplit-v2 b/paimon-core/src/test/resources/compatibility/datasplit-v2 new file mode 100644 index 000000000000..8d2014984b8f Binary files /dev/null and b/paimon-core/src/test/resources/compatibility/datasplit-v2 differ diff --git a/paimon-core/src/test/resources/compatibility/datasplit-v3 b/paimon-core/src/test/resources/compatibility/datasplit-v3 new file mode 100644 index 000000000000..6b19fe2d958d Binary files /dev/null and b/paimon-core/src/test/resources/compatibility/datasplit-v3 differ diff --git a/paimon-core/src/test/resources/compatibility/manifest-committable-v3 b/paimon-core/src/test/resources/compatibility/manifest-committable-v3 new file mode 100644 index 000000000000..58ad2983cfb4 Binary files /dev/null and b/paimon-core/src/test/resources/compatibility/manifest-committable-v3 differ diff --git a/paimon-core/src/test/resources/compatibility/manifest-committable-v4 b/paimon-core/src/test/resources/compatibility/manifest-committable-v4 new file mode 100644 index 000000000000..9c095669a34b Binary files /dev/null and b/paimon-core/src/test/resources/compatibility/manifest-committable-v4 differ diff --git a/paimon-docs/pom.xml b/paimon-docs/pom.xml index 95de61841962..6c4216943552 100644 --- a/paimon-docs/pom.xml +++ b/paimon-docs/pom.xml @@ -59,7 +59,7 @@ under the License. org.apache.paimon - paimon-spark-common + paimon-spark-common_${scala.binary.version} ${project.version} provided diff --git a/paimon-docs/src/main/java/org/apache/paimon/docs/configuration/ConfigOptionsDocGenerator.java b/paimon-docs/src/main/java/org/apache/paimon/docs/configuration/ConfigOptionsDocGenerator.java index 6f700724ac01..1d35559af658 100644 --- a/paimon-docs/src/main/java/org/apache/paimon/docs/configuration/ConfigOptionsDocGenerator.java +++ b/paimon-docs/src/main/java/org/apache/paimon/docs/configuration/ConfigOptionsDocGenerator.java @@ -77,6 +77,7 @@ public class ConfigOptionsDocGenerator { new OptionsClassLocation("paimon-core", "org.apache.paimon.lookup"), new OptionsClassLocation("paimon-core", "org.apache.paimon.catalog"), new OptionsClassLocation("paimon-core", "org.apache.paimon.jdbc"), + new OptionsClassLocation("paimon-core", "org.apache.paimon.table"), new OptionsClassLocation("paimon-format", "org.apache.paimon.format"), new OptionsClassLocation( "paimon-flink/paimon-flink-common", "org.apache.paimon.flink"), diff --git a/paimon-e2e-tests/pom.xml b/paimon-e2e-tests/pom.xml index 7ca13535e470..6f025c9d9f9f 100644 --- a/paimon-e2e-tests/pom.xml +++ b/paimon-e2e-tests/pom.xml @@ -34,7 +34,7 @@ under the License. 2.8.3-10.0 3.1.1 - flink-sql-connector-hive-2.3.10_${scala.binary.version} + flink-sql-connector-hive-2.3.10_${flink.scala.binary.version} @@ -63,7 +63,7 @@ under the License. org.apache.paimon - paimon-spark-3.2 + paimon-spark-${test.spark.main.version} ${project.version} runtime @@ -185,7 +185,7 @@ under the License. org.apache.paimon - paimon-spark-3.2 + paimon-spark-${test.spark.main.version} ${project.version} paimon-spark.jar jar diff --git a/paimon-e2e-tests/src/test/java/org/apache/paimon/tests/E2eTestBase.java b/paimon-e2e-tests/src/test/java/org/apache/paimon/tests/E2eTestBase.java index 55f9e984e5de..0547ac4f27d1 100644 --- a/paimon-e2e-tests/src/test/java/org/apache/paimon/tests/E2eTestBase.java +++ b/paimon-e2e-tests/src/test/java/org/apache/paimon/tests/E2eTestBase.java @@ -111,7 +111,8 @@ public void before() throws Exception { for (String s : kafkaServices) { environment.withLogConsumer(s + "-1", new Slf4jLogConsumer(LOG)); } - environment.waitingFor("kafka-1", buildWaitStrategy(".*Recorded new controller.*", 2)); + environment.waitingFor( + "kafka-1", buildWaitStrategy(".*Recorded new ZK controller.*", 2)); } if (withHive) { List hiveServices = diff --git a/paimon-e2e-tests/src/test/resources-filtered/docker-compose.yaml b/paimon-e2e-tests/src/test/resources-filtered/docker-compose.yaml index 400f88c385c9..b89420c8125a 100644 --- a/paimon-e2e-tests/src/test/resources-filtered/docker-compose.yaml +++ b/paimon-e2e-tests/src/test/resources-filtered/docker-compose.yaml @@ -77,7 +77,7 @@ services: # ---------------------------------------- zookeeper: - image: confluentinc/cp-zookeeper:7.0.1 + image: confluentinc/cp-zookeeper:7.8.0 networks: testnetwork: aliases: @@ -89,7 +89,7 @@ services: - "2181" kafka: - image: confluentinc/cp-kafka:7.0.1 + image: confluentinc/cp-kafka:7.8.0 networks: testnetwork: aliases: @@ -193,7 +193,7 @@ services: # ---------------------------------------- spark-master: - image: bde2020/spark-master:3.3.0-hadoop3.3 + image: bde2020/spark-master:${test.spark.version}-hadoop3.3 volumes: - testdata:/test-data - /tmp/paimon-e2e-tests-jars:/jars @@ -205,7 +205,7 @@ services: - INIT_DAEMON_STEP=setup_spark spark-worker: - image: bde2020/spark-worker:3.3.0-hadoop3.3 + image: bde2020/spark-worker:${test.spark.version}-hadoop3.3 depends_on: - spark-master volumes: diff --git a/paimon-filesystems/paimon-oss-impl/src/main/java/org/apache/paimon/oss/HadoopCompliantFileIO.java b/paimon-filesystems/paimon-oss-impl/src/main/java/org/apache/paimon/oss/HadoopCompliantFileIO.java index 4d86c12a6e52..67027eabadfb 100644 --- a/paimon-filesystems/paimon-oss-impl/src/main/java/org/apache/paimon/oss/HadoopCompliantFileIO.java +++ b/paimon-filesystems/paimon-oss-impl/src/main/java/org/apache/paimon/oss/HadoopCompliantFileIO.java @@ -286,5 +286,15 @@ public Path getPath() { public long getModificationTime() { return status.getModificationTime(); } + + @Override + public long getAccessTime() { + return status.getAccessTime(); + } + + @Override + public String getOwner() { + return status.getOwner(); + } } } diff --git a/paimon-filesystems/paimon-s3-impl/src/main/java/org/apache/paimon/s3/HadoopCompliantFileIO.java b/paimon-filesystems/paimon-s3-impl/src/main/java/org/apache/paimon/s3/HadoopCompliantFileIO.java index abfe0fabba61..80f3df582096 100644 --- a/paimon-filesystems/paimon-s3-impl/src/main/java/org/apache/paimon/s3/HadoopCompliantFileIO.java +++ b/paimon-filesystems/paimon-s3-impl/src/main/java/org/apache/paimon/s3/HadoopCompliantFileIO.java @@ -286,5 +286,15 @@ public Path getPath() { public long getModificationTime() { return status.getModificationTime(); } + + @Override + public long getAccessTime() { + return status.getAccessTime(); + } + + @Override + public String getOwner() { + return status.getOwner(); + } } } diff --git a/paimon-flink/paimon-flink-1.15/pom.xml b/paimon-flink/paimon-flink-1.15/pom.xml index bfc6ec53c404..21c179226bf0 100644 --- a/paimon-flink/paimon-flink-1.15/pom.xml +++ b/paimon-flink/paimon-flink-1.15/pom.xml @@ -87,14 +87,14 @@ under the License. org.apache.flink - flink-table-planner_${scala.binary.version} + flink-table-planner_${flink.scala.binary.version} ${flink.version} test org.apache.flink - flink-table-planner_${scala.binary.version} + flink-table-planner_${flink.scala.binary.version} ${flink.version} test-jar test diff --git a/paimon-flink/paimon-flink-1.15/src/main/java/org/apache/flink/api/common/functions/OpenContext.java b/paimon-flink/paimon-flink-1.15/src/main/java/org/apache/flink/api/common/functions/OpenContext.java new file mode 100644 index 000000000000..4ff5484b3b08 --- /dev/null +++ b/paimon-flink/paimon-flink-1.15/src/main/java/org/apache/flink/api/common/functions/OpenContext.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.api.common.functions; + +import org.apache.flink.annotation.PublicEvolving; + +/** + * The {@link OpenContext} interface provides necessary information required by the {@link + * RichFunction} when it is opened. The {@link OpenContext} is currently empty because it can be + * used to add more methods without affecting the signature of {@code RichFunction#open}. + */ +@PublicEvolving +public interface OpenContext {} diff --git a/paimon-flink/paimon-flink-1.15/src/main/java/org/apache/flink/api/common/serialization/SerializerConfig.java b/paimon-flink/paimon-flink-1.15/src/main/java/org/apache/flink/api/common/serialization/SerializerConfig.java new file mode 100644 index 000000000000..16987469a948 --- /dev/null +++ b/paimon-flink/paimon-flink-1.15/src/main/java/org/apache/flink/api/common/serialization/SerializerConfig.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.api.common.serialization; + +/** Placeholder class to resolve compatibility issues. */ +public interface SerializerConfig {} diff --git a/paimon-flink/paimon-flink-1.15/src/main/java/org/apache/flink/api/common/serialization/SerializerConfigImpl.java b/paimon-flink/paimon-flink-1.15/src/main/java/org/apache/flink/api/common/serialization/SerializerConfigImpl.java new file mode 100644 index 000000000000..374d33f6500d --- /dev/null +++ b/paimon-flink/paimon-flink-1.15/src/main/java/org/apache/flink/api/common/serialization/SerializerConfigImpl.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.api.common.serialization; + +/** Placeholder class to resolve compatibility issues. */ +public class SerializerConfigImpl implements SerializerConfig {} diff --git a/paimon-flink/paimon-flink-1.15/src/main/java/org/apache/flink/api/connector/sink2/WriterInitContext.java b/paimon-flink/paimon-flink-1.15/src/main/java/org/apache/flink/api/connector/sink2/WriterInitContext.java new file mode 100644 index 000000000000..563dbbe75e7e --- /dev/null +++ b/paimon-flink/paimon-flink-1.15/src/main/java/org/apache/flink/api/connector/sink2/WriterInitContext.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.api.connector.sink2; + +/** Placeholder class to resolve compatibility issues. */ +public interface WriterInitContext extends org.apache.flink.api.connector.sink2.Sink.InitContext {} diff --git a/paimon-flink/paimon-flink-1.15/src/main/java/org/apache/flink/api/connector/source/SupportsHandleExecutionAttemptSourceEvent.java b/paimon-flink/paimon-flink-1.15/src/main/java/org/apache/flink/api/connector/source/SupportsHandleExecutionAttemptSourceEvent.java new file mode 100644 index 000000000000..9d1ca7a43b8f --- /dev/null +++ b/paimon-flink/paimon-flink-1.15/src/main/java/org/apache/flink/api/connector/source/SupportsHandleExecutionAttemptSourceEvent.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.api.connector.source; + +import org.apache.flink.annotation.PublicEvolving; + +/** + * An decorative interface of {@link SplitEnumerator} which allows to handle {@link SourceEvent} + * sent from a specific execution attempt. + * + *

    The split enumerator must implement this interface if it needs to deal with custom source + * events and is used in cases that a subtask can have multiple concurrent execution attempts, e.g. + * if speculative execution is enabled. Otherwise an error will be thrown when the split enumerator + * receives a custom source event. + */ +@PublicEvolving +public interface SupportsHandleExecutionAttemptSourceEvent { + + /** + * Handles a custom source event from the source reader. It is similar to {@link + * SplitEnumerator#handleSourceEvent(int, SourceEvent)} but is aware of the subtask execution + * attempt who sent this event. + * + * @param subtaskId the subtask id of the source reader who sent the source event. + * @param attemptNumber the attempt number of the source reader who sent the source event. + * @param sourceEvent the source event from the source reader. + */ + void handleSourceEvent(int subtaskId, int attemptNumber, SourceEvent sourceEvent); +} diff --git a/paimon-flink/paimon-flink-1.15/src/main/java/org/apache/flink/streaming/api/functions/sink/v2/DiscardingSink.java b/paimon-flink/paimon-flink-1.15/src/main/java/org/apache/flink/streaming/api/functions/sink/v2/DiscardingSink.java new file mode 100644 index 000000000000..98aaf6418ff7 --- /dev/null +++ b/paimon-flink/paimon-flink-1.15/src/main/java/org/apache/flink/streaming/api/functions/sink/v2/DiscardingSink.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.streaming.api.functions.sink.v2; + +import org.apache.flink.annotation.PublicEvolving; +import org.apache.flink.api.connector.sink2.Sink; +import org.apache.flink.api.connector.sink2.SinkWriter; + +import java.io.IOException; + +/** + * A special sink that ignores all elements. + * + * @param The type of elements received by the sink. + */ +@PublicEvolving +public class DiscardingSink implements Sink { + private static final long serialVersionUID = 1L; + + @Override + public SinkWriter createWriter(InitContext context) throws IOException { + return new DiscardingElementWriter(); + } + + private class DiscardingElementWriter implements SinkWriter { + + @Override + public void write(IN element, Context context) throws IOException, InterruptedException { + // discard it. + } + + @Override + public void flush(boolean endOfInput) throws IOException, InterruptedException { + // this writer has no pending data. + } + + @Override + public void close() throws Exception { + // do nothing. + } + } +} diff --git a/paimon-flink/paimon-flink-1.15/src/main/java/org/apache/paimon/flink/source/DataTableSource.java b/paimon-flink/paimon-flink-1.15/src/main/java/org/apache/paimon/flink/source/DataTableSource.java index ee00d41832cd..f41f8da6c820 100644 --- a/paimon-flink/paimon-flink-1.15/src/main/java/org/apache/paimon/flink/source/DataTableSource.java +++ b/paimon-flink/paimon-flink-1.15/src/main/java/org/apache/paimon/flink/source/DataTableSource.java @@ -51,7 +51,7 @@ public DataTableSource( null, null, null, - false); + null); } public DataTableSource( @@ -64,7 +64,7 @@ public DataTableSource( @Nullable int[][] projectFields, @Nullable Long limit, @Nullable WatermarkStrategy watermarkStrategy, - boolean isBatchCountStar) { + @Nullable Long countPushed) { super( tableIdentifier, table, @@ -75,7 +75,7 @@ public DataTableSource( projectFields, limit, watermarkStrategy, - isBatchCountStar); + countPushed); } @Override @@ -90,7 +90,7 @@ public DataTableSource copy() { projectFields, limit, watermarkStrategy, - isBatchCountStar); + countPushed); } @Override diff --git a/paimon-flink/paimon-flink-1.15/src/main/java/org/apache/paimon/flink/utils/RuntimeContextUtils.java b/paimon-flink/paimon-flink-1.15/src/main/java/org/apache/paimon/flink/utils/RuntimeContextUtils.java new file mode 100644 index 000000000000..460fea55ad7a --- /dev/null +++ b/paimon-flink/paimon-flink-1.15/src/main/java/org/apache/paimon/flink/utils/RuntimeContextUtils.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.flink.utils; + +import org.apache.flink.api.common.functions.RuntimeContext; + +/** Utility methods about Flink runtime context to resolve compatibility issues. */ +public class RuntimeContextUtils { + public static int getNumberOfParallelSubtasks(RuntimeContext context) { + return context.getNumberOfParallelSubtasks(); + } + + public static int getIndexOfThisSubtask(RuntimeContext context) { + return context.getIndexOfThisSubtask(); + } +} diff --git a/paimon-flink/paimon-flink-1.16/pom.xml b/paimon-flink/paimon-flink-1.16/pom.xml index 0e1abbe6674a..c5a334d6fce6 100644 --- a/paimon-flink/paimon-flink-1.16/pom.xml +++ b/paimon-flink/paimon-flink-1.16/pom.xml @@ -35,6 +35,8 @@ under the License. 1.16.3 + 1.16 + 1.5.2 @@ -88,7 +90,7 @@ under the License. org.apache.flink - flink-table-planner_${scala.binary.version} + flink-table-planner_${flink.scala.binary.version} ${flink.version} test @@ -100,6 +102,12 @@ under the License. test + + org.apache.iceberg + iceberg-flink-${iceberg.flink.version} + ${iceberg.version} + test + diff --git a/paimon-flink/paimon-flink-1.16/src/main/java/org/apache/flink/api/common/functions/OpenContext.java b/paimon-flink/paimon-flink-1.16/src/main/java/org/apache/flink/api/common/functions/OpenContext.java new file mode 100644 index 000000000000..4ff5484b3b08 --- /dev/null +++ b/paimon-flink/paimon-flink-1.16/src/main/java/org/apache/flink/api/common/functions/OpenContext.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.api.common.functions; + +import org.apache.flink.annotation.PublicEvolving; + +/** + * The {@link OpenContext} interface provides necessary information required by the {@link + * RichFunction} when it is opened. The {@link OpenContext} is currently empty because it can be + * used to add more methods without affecting the signature of {@code RichFunction#open}. + */ +@PublicEvolving +public interface OpenContext {} diff --git a/paimon-flink/paimon-flink-1.16/src/main/java/org/apache/flink/api/common/serialization/SerializerConfig.java b/paimon-flink/paimon-flink-1.16/src/main/java/org/apache/flink/api/common/serialization/SerializerConfig.java new file mode 100644 index 000000000000..16987469a948 --- /dev/null +++ b/paimon-flink/paimon-flink-1.16/src/main/java/org/apache/flink/api/common/serialization/SerializerConfig.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.api.common.serialization; + +/** Placeholder class to resolve compatibility issues. */ +public interface SerializerConfig {} diff --git a/paimon-flink/paimon-flink-1.16/src/main/java/org/apache/flink/api/common/serialization/SerializerConfigImpl.java b/paimon-flink/paimon-flink-1.16/src/main/java/org/apache/flink/api/common/serialization/SerializerConfigImpl.java new file mode 100644 index 000000000000..374d33f6500d --- /dev/null +++ b/paimon-flink/paimon-flink-1.16/src/main/java/org/apache/flink/api/common/serialization/SerializerConfigImpl.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.api.common.serialization; + +/** Placeholder class to resolve compatibility issues. */ +public class SerializerConfigImpl implements SerializerConfig {} diff --git a/paimon-flink/paimon-flink-1.16/src/main/java/org/apache/flink/api/connector/sink2/WriterInitContext.java b/paimon-flink/paimon-flink-1.16/src/main/java/org/apache/flink/api/connector/sink2/WriterInitContext.java new file mode 100644 index 000000000000..563dbbe75e7e --- /dev/null +++ b/paimon-flink/paimon-flink-1.16/src/main/java/org/apache/flink/api/connector/sink2/WriterInitContext.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.api.connector.sink2; + +/** Placeholder class to resolve compatibility issues. */ +public interface WriterInitContext extends org.apache.flink.api.connector.sink2.Sink.InitContext {} diff --git a/paimon-flink/paimon-flink-1.16/src/main/java/org/apache/flink/streaming/api/functions/sink/v2/DiscardingSink.java b/paimon-flink/paimon-flink-1.16/src/main/java/org/apache/flink/streaming/api/functions/sink/v2/DiscardingSink.java new file mode 100644 index 000000000000..98aaf6418ff7 --- /dev/null +++ b/paimon-flink/paimon-flink-1.16/src/main/java/org/apache/flink/streaming/api/functions/sink/v2/DiscardingSink.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.streaming.api.functions.sink.v2; + +import org.apache.flink.annotation.PublicEvolving; +import org.apache.flink.api.connector.sink2.Sink; +import org.apache.flink.api.connector.sink2.SinkWriter; + +import java.io.IOException; + +/** + * A special sink that ignores all elements. + * + * @param The type of elements received by the sink. + */ +@PublicEvolving +public class DiscardingSink implements Sink { + private static final long serialVersionUID = 1L; + + @Override + public SinkWriter createWriter(InitContext context) throws IOException { + return new DiscardingElementWriter(); + } + + private class DiscardingElementWriter implements SinkWriter { + + @Override + public void write(IN element, Context context) throws IOException, InterruptedException { + // discard it. + } + + @Override + public void flush(boolean endOfInput) throws IOException, InterruptedException { + // this writer has no pending data. + } + + @Override + public void close() throws Exception { + // do nothing. + } + } +} diff --git a/paimon-flink/paimon-flink-1.16/src/main/java/org/apache/paimon/flink/utils/RuntimeContextUtils.java b/paimon-flink/paimon-flink-1.16/src/main/java/org/apache/paimon/flink/utils/RuntimeContextUtils.java new file mode 100644 index 000000000000..460fea55ad7a --- /dev/null +++ b/paimon-flink/paimon-flink-1.16/src/main/java/org/apache/paimon/flink/utils/RuntimeContextUtils.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.flink.utils; + +import org.apache.flink.api.common.functions.RuntimeContext; + +/** Utility methods about Flink runtime context to resolve compatibility issues. */ +public class RuntimeContextUtils { + public static int getNumberOfParallelSubtasks(RuntimeContext context) { + return context.getNumberOfParallelSubtasks(); + } + + public static int getIndexOfThisSubtask(RuntimeContext context) { + return context.getIndexOfThisSubtask(); + } +} diff --git a/paimon-flink/paimon-flink-1.16/src/test/java/org/apache/paimon/flink/iceberg/Flink116IcebergITCase.java b/paimon-flink/paimon-flink-1.16/src/test/java/org/apache/paimon/flink/iceberg/Flink116IcebergITCase.java new file mode 100644 index 000000000000..3001fefe4bb3 --- /dev/null +++ b/paimon-flink/paimon-flink-1.16/src/test/java/org/apache/paimon/flink/iceberg/Flink116IcebergITCase.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.flink.iceberg; + +/** IT cases for Paimon Iceberg compatibility in Flink 1.16. */ +public class Flink116IcebergITCase extends FlinkIcebergITCaseBase { + + @Override + public void testNestedTypes(String format) { + // Flink 1.16 (or maybe Calcite?) will mistakenly cast the result to VARCHAR(5), + // so we skip this test in Flink 1.16. + } +} diff --git a/paimon-flink/paimon-flink-1.17/pom.xml b/paimon-flink/paimon-flink-1.17/pom.xml index 99492af0f248..6e85d71016e5 100644 --- a/paimon-flink/paimon-flink-1.17/pom.xml +++ b/paimon-flink/paimon-flink-1.17/pom.xml @@ -35,6 +35,7 @@ under the License. 1.17.2 + 1.17 @@ -95,7 +96,7 @@ under the License. org.apache.flink - flink-table-planner_${scala.binary.version} + flink-table-planner_${flink.scala.binary.version} ${flink.version} test @@ -106,6 +107,13 @@ under the License. ${flink.version} test + + + org.apache.iceberg + iceberg-flink-${iceberg.flink.version} + ${iceberg.version} + test + diff --git a/paimon-flink/paimon-flink-1.17/src/main/java/org/apache/flink/api/common/functions/OpenContext.java b/paimon-flink/paimon-flink-1.17/src/main/java/org/apache/flink/api/common/functions/OpenContext.java new file mode 100644 index 000000000000..4ff5484b3b08 --- /dev/null +++ b/paimon-flink/paimon-flink-1.17/src/main/java/org/apache/flink/api/common/functions/OpenContext.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.api.common.functions; + +import org.apache.flink.annotation.PublicEvolving; + +/** + * The {@link OpenContext} interface provides necessary information required by the {@link + * RichFunction} when it is opened. The {@link OpenContext} is currently empty because it can be + * used to add more methods without affecting the signature of {@code RichFunction#open}. + */ +@PublicEvolving +public interface OpenContext {} diff --git a/paimon-flink/paimon-flink-1.17/src/main/java/org/apache/flink/api/common/serialization/SerializerConfig.java b/paimon-flink/paimon-flink-1.17/src/main/java/org/apache/flink/api/common/serialization/SerializerConfig.java new file mode 100644 index 000000000000..16987469a948 --- /dev/null +++ b/paimon-flink/paimon-flink-1.17/src/main/java/org/apache/flink/api/common/serialization/SerializerConfig.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.api.common.serialization; + +/** Placeholder class to resolve compatibility issues. */ +public interface SerializerConfig {} diff --git a/paimon-flink/paimon-flink-1.17/src/main/java/org/apache/flink/api/common/serialization/SerializerConfigImpl.java b/paimon-flink/paimon-flink-1.17/src/main/java/org/apache/flink/api/common/serialization/SerializerConfigImpl.java new file mode 100644 index 000000000000..374d33f6500d --- /dev/null +++ b/paimon-flink/paimon-flink-1.17/src/main/java/org/apache/flink/api/common/serialization/SerializerConfigImpl.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.api.common.serialization; + +/** Placeholder class to resolve compatibility issues. */ +public class SerializerConfigImpl implements SerializerConfig {} diff --git a/paimon-flink/paimon-flink-1.17/src/main/java/org/apache/flink/api/connector/sink2/WriterInitContext.java b/paimon-flink/paimon-flink-1.17/src/main/java/org/apache/flink/api/connector/sink2/WriterInitContext.java new file mode 100644 index 000000000000..db4500042572 --- /dev/null +++ b/paimon-flink/paimon-flink-1.17/src/main/java/org/apache/flink/api/connector/sink2/WriterInitContext.java @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.api.connector.sink2; + +import org.apache.flink.annotation.Public; + +/** Placeholder class to resolve compatibility issues. */ +@Public +public interface WriterInitContext extends org.apache.flink.api.connector.sink2.Sink.InitContext {} diff --git a/paimon-flink/paimon-flink-1.17/src/main/java/org/apache/flink/streaming/api/functions/sink/v2/DiscardingSink.java b/paimon-flink/paimon-flink-1.17/src/main/java/org/apache/flink/streaming/api/functions/sink/v2/DiscardingSink.java new file mode 100644 index 000000000000..fc7eb0d48356 --- /dev/null +++ b/paimon-flink/paimon-flink-1.17/src/main/java/org/apache/flink/streaming/api/functions/sink/v2/DiscardingSink.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.streaming.api.functions.sink.v2; + +import org.apache.flink.annotation.PublicEvolving; +import org.apache.flink.api.common.SupportsConcurrentExecutionAttempts; +import org.apache.flink.api.connector.sink2.Sink; +import org.apache.flink.api.connector.sink2.SinkWriter; + +import java.io.IOException; + +/** + * A special sink that ignores all elements. + * + * @param The type of elements received by the sink. + */ +@PublicEvolving +public class DiscardingSink implements Sink, SupportsConcurrentExecutionAttempts { + private static final long serialVersionUID = 1L; + + @Override + public SinkWriter createWriter(InitContext context) throws IOException { + return new DiscardingElementWriter(); + } + + private class DiscardingElementWriter implements SinkWriter { + + @Override + public void write(IN element, Context context) throws IOException, InterruptedException { + // discard it. + } + + @Override + public void flush(boolean endOfInput) throws IOException, InterruptedException { + // this writer has no pending data. + } + + @Override + public void close() throws Exception { + // do nothing. + } + } +} diff --git a/paimon-flink/paimon-flink-1.17/src/main/java/org/apache/paimon/flink/utils/RuntimeContextUtils.java b/paimon-flink/paimon-flink-1.17/src/main/java/org/apache/paimon/flink/utils/RuntimeContextUtils.java new file mode 100644 index 000000000000..460fea55ad7a --- /dev/null +++ b/paimon-flink/paimon-flink-1.17/src/main/java/org/apache/paimon/flink/utils/RuntimeContextUtils.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.flink.utils; + +import org.apache.flink.api.common.functions.RuntimeContext; + +/** Utility methods about Flink runtime context to resolve compatibility issues. */ +public class RuntimeContextUtils { + public static int getNumberOfParallelSubtasks(RuntimeContext context) { + return context.getNumberOfParallelSubtasks(); + } + + public static int getIndexOfThisSubtask(RuntimeContext context) { + return context.getIndexOfThisSubtask(); + } +} diff --git a/paimon-flink/paimon-flink-1.17/src/test/java/org/apache/paimon/flink/iceberg/Flink117IcebergITCase.java b/paimon-flink/paimon-flink-1.17/src/test/java/org/apache/paimon/flink/iceberg/Flink117IcebergITCase.java new file mode 100644 index 000000000000..3628043bdfa6 --- /dev/null +++ b/paimon-flink/paimon-flink-1.17/src/test/java/org/apache/paimon/flink/iceberg/Flink117IcebergITCase.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.flink.iceberg; + +/** IT cases for Paimon Iceberg compatibility in Flink 1.17. */ +public class Flink117IcebergITCase extends FlinkIcebergITCaseBase {} diff --git a/paimon-flink/paimon-flink-1.18/pom.xml b/paimon-flink/paimon-flink-1.18/pom.xml index 0220bcc19612..d1c33f43f958 100644 --- a/paimon-flink/paimon-flink-1.18/pom.xml +++ b/paimon-flink/paimon-flink-1.18/pom.xml @@ -35,6 +35,7 @@ under the License. 1.18.1 + 1.18 @@ -88,7 +89,7 @@ under the License. org.apache.flink - flink-table-planner_${scala.binary.version} + flink-table-planner_${flink.scala.binary.version} ${flink.version} test @@ -99,6 +100,13 @@ under the License. ${flink.version} test + + + org.apache.iceberg + iceberg-flink-${iceberg.flink.version} + ${iceberg.version} + test + diff --git a/paimon-flink/paimon-flink-1.18/src/main/java/org/apache/flink/api/common/functions/OpenContext.java b/paimon-flink/paimon-flink-1.18/src/main/java/org/apache/flink/api/common/functions/OpenContext.java new file mode 100644 index 000000000000..4ff5484b3b08 --- /dev/null +++ b/paimon-flink/paimon-flink-1.18/src/main/java/org/apache/flink/api/common/functions/OpenContext.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.api.common.functions; + +import org.apache.flink.annotation.PublicEvolving; + +/** + * The {@link OpenContext} interface provides necessary information required by the {@link + * RichFunction} when it is opened. The {@link OpenContext} is currently empty because it can be + * used to add more methods without affecting the signature of {@code RichFunction#open}. + */ +@PublicEvolving +public interface OpenContext {} diff --git a/paimon-flink/paimon-flink-1.18/src/main/java/org/apache/flink/api/common/serialization/SerializerConfig.java b/paimon-flink/paimon-flink-1.18/src/main/java/org/apache/flink/api/common/serialization/SerializerConfig.java new file mode 100644 index 000000000000..16987469a948 --- /dev/null +++ b/paimon-flink/paimon-flink-1.18/src/main/java/org/apache/flink/api/common/serialization/SerializerConfig.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.api.common.serialization; + +/** Placeholder class to resolve compatibility issues. */ +public interface SerializerConfig {} diff --git a/paimon-flink/paimon-flink-1.18/src/main/java/org/apache/flink/api/common/serialization/SerializerConfigImpl.java b/paimon-flink/paimon-flink-1.18/src/main/java/org/apache/flink/api/common/serialization/SerializerConfigImpl.java new file mode 100644 index 000000000000..374d33f6500d --- /dev/null +++ b/paimon-flink/paimon-flink-1.18/src/main/java/org/apache/flink/api/common/serialization/SerializerConfigImpl.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.api.common.serialization; + +/** Placeholder class to resolve compatibility issues. */ +public class SerializerConfigImpl implements SerializerConfig {} diff --git a/paimon-flink/paimon-flink-1.18/src/main/java/org/apache/flink/api/connector/sink2/WriterInitContext.java b/paimon-flink/paimon-flink-1.18/src/main/java/org/apache/flink/api/connector/sink2/WriterInitContext.java new file mode 100644 index 000000000000..563dbbe75e7e --- /dev/null +++ b/paimon-flink/paimon-flink-1.18/src/main/java/org/apache/flink/api/connector/sink2/WriterInitContext.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.api.connector.sink2; + +/** Placeholder class to resolve compatibility issues. */ +public interface WriterInitContext extends org.apache.flink.api.connector.sink2.Sink.InitContext {} diff --git a/paimon-flink/paimon-flink-1.18/src/main/java/org/apache/flink/streaming/api/functions/sink/v2/DiscardingSink.java b/paimon-flink/paimon-flink-1.18/src/main/java/org/apache/flink/streaming/api/functions/sink/v2/DiscardingSink.java new file mode 100644 index 000000000000..fc7eb0d48356 --- /dev/null +++ b/paimon-flink/paimon-flink-1.18/src/main/java/org/apache/flink/streaming/api/functions/sink/v2/DiscardingSink.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.streaming.api.functions.sink.v2; + +import org.apache.flink.annotation.PublicEvolving; +import org.apache.flink.api.common.SupportsConcurrentExecutionAttempts; +import org.apache.flink.api.connector.sink2.Sink; +import org.apache.flink.api.connector.sink2.SinkWriter; + +import java.io.IOException; + +/** + * A special sink that ignores all elements. + * + * @param The type of elements received by the sink. + */ +@PublicEvolving +public class DiscardingSink implements Sink, SupportsConcurrentExecutionAttempts { + private static final long serialVersionUID = 1L; + + @Override + public SinkWriter createWriter(InitContext context) throws IOException { + return new DiscardingElementWriter(); + } + + private class DiscardingElementWriter implements SinkWriter { + + @Override + public void write(IN element, Context context) throws IOException, InterruptedException { + // discard it. + } + + @Override + public void flush(boolean endOfInput) throws IOException, InterruptedException { + // this writer has no pending data. + } + + @Override + public void close() throws Exception { + // do nothing. + } + } +} diff --git a/paimon-flink/paimon-flink-1.18/src/main/java/org/apache/paimon/flink/procedure/CompactDatabaseProcedure.java b/paimon-flink/paimon-flink-1.18/src/main/java/org/apache/paimon/flink/procedure/CompactDatabaseProcedure.java index 99f205bacb58..ac4340c11336 100644 --- a/paimon-flink/paimon-flink-1.18/src/main/java/org/apache/paimon/flink/procedure/CompactDatabaseProcedure.java +++ b/paimon-flink/paimon-flink-1.18/src/main/java/org/apache/paimon/flink/procedure/CompactDatabaseProcedure.java @@ -26,6 +26,8 @@ import java.util.Map; +import static org.apache.paimon.flink.action.ActionFactory.FULL; +import static org.apache.paimon.flink.action.CompactActionFactory.checkCompactStrategy; import static org.apache.paimon.utils.ParameterUtils.parseCommaSeparatedKeyValues; /** @@ -51,6 +53,7 @@ * * -- set table options ('k=v,...') * CALL sys.compact_database('includingDatabases', 'mode', 'includingTables', 'excludingTables', 'tableOptions') + * * */ public class CompactDatabaseProcedure extends ProcedureBase { @@ -106,7 +109,8 @@ public String[] call( includingTables, excludingTables, tableOptions, - ""); + "", + null); } public String[] call( @@ -116,7 +120,8 @@ public String[] call( String includingTables, String excludingTables, String tableOptions, - String partitionIdleTime) + String partitionIdleTime, + String compactStrategy) throws Exception { String warehouse = catalog.warehouse(); Map catalogOptions = catalog.options(); @@ -133,6 +138,10 @@ public String[] call( action.withPartitionIdleTime(TimeUtils.parseDuration(partitionIdleTime)); } + if (checkCompactStrategy(compactStrategy)) { + action.withFullCompaction(compactStrategy.trim().equalsIgnoreCase(FULL)); + } + return execute(procedureContext, action, "Compact database job"); } diff --git a/paimon-flink/paimon-flink-1.18/src/main/java/org/apache/paimon/flink/procedure/CompactProcedure.java b/paimon-flink/paimon-flink-1.18/src/main/java/org/apache/paimon/flink/procedure/CompactProcedure.java index c9be24404946..560e532a6dbb 100644 --- a/paimon-flink/paimon-flink-1.18/src/main/java/org/apache/paimon/flink/procedure/CompactProcedure.java +++ b/paimon-flink/paimon-flink-1.18/src/main/java/org/apache/paimon/flink/procedure/CompactProcedure.java @@ -31,6 +31,9 @@ import java.util.Collections; import java.util.Map; +import static org.apache.paimon.flink.action.ActionFactory.FULL; +import static org.apache.paimon.flink.action.CompactActionFactory.checkCompactStrategy; + /** * Stay compatible with 1.18 procedure which doesn't support named argument. Usage: * @@ -44,8 +47,14 @@ * CALL sys.compact('tableId', 'pt1=A,pt2=a;pt1=B,pt2=b') * * -- compact a table with sorting + * CALL sys.compact('tableId', 'ORDER/ZORDER', 'col1,col2') + * + * -- compact specific partitions with sorting * CALL sys.compact('tableId', 'partitions', 'ORDER/ZORDER', 'col1,col2', 'sink.parallelism=6') * + * -- compact with specific compact strategy + * CALL sys.compact('tableId', 'partitions', 'ORDER/ZORDER', 'col1,col2', 'sink.parallelism=6', 'minor') + * * */ public class CompactProcedure extends ProcedureBase { @@ -61,6 +70,15 @@ public String[] call(ProcedureContext procedureContext, String tableId, String p return call(procedureContext, tableId, partitions, "", "", "", ""); } + public String[] call( + ProcedureContext procedureContext, + String tableId, + String orderStrategy, + String orderByColumns) + throws Exception { + return call(procedureContext, tableId, "", orderStrategy, orderByColumns, "", ""); + } + public String[] call( ProcedureContext procedureContext, String tableId, @@ -106,7 +124,8 @@ public String[] call( orderByColumns, tableOptions, whereSql, - ""); + "", + null); } public String[] call( @@ -117,7 +136,8 @@ public String[] call( String orderByColumns, String tableOptions, String whereSql, - String partitionIdleTime) + String partitionIdleTime, + String compactStrategy) throws Exception { String warehouse = catalog.warehouse(); @@ -140,6 +160,10 @@ public String[] call( if (!(StringUtils.isNullOrWhitespaceOnly(partitionIdleTime))) { action.withPartitionIdleTime(TimeUtils.parseDuration(partitionIdleTime)); } + + if (checkCompactStrategy(compactStrategy)) { + action.withFullCompaction(compactStrategy.trim().equalsIgnoreCase(FULL)); + } jobName = "Compact Job"; } else if (!orderStrategy.isEmpty() && !orderByColumns.isEmpty()) { Preconditions.checkArgument( diff --git a/paimon-flink/paimon-flink-1.18/src/main/java/org/apache/paimon/flink/procedure/CreateOrReplaceTagBaseProcedure.java b/paimon-flink/paimon-flink-1.18/src/main/java/org/apache/paimon/flink/procedure/CreateOrReplaceTagBaseProcedure.java new file mode 100644 index 000000000000..2b7dadc05e9b --- /dev/null +++ b/paimon-flink/paimon-flink-1.18/src/main/java/org/apache/paimon/flink/procedure/CreateOrReplaceTagBaseProcedure.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.flink.procedure; + +import org.apache.paimon.catalog.Catalog; +import org.apache.paimon.catalog.Identifier; +import org.apache.paimon.table.Table; +import org.apache.paimon.utils.TimeUtils; + +import org.apache.flink.table.procedure.ProcedureContext; + +import javax.annotation.Nullable; + +import java.time.Duration; + +/** A base procedure to create or replace a tag. */ +public abstract class CreateOrReplaceTagBaseProcedure extends ProcedureBase { + + public String[] call(ProcedureContext procedureContext, String tableId, String tagName) + throws Catalog.TableNotExistException { + return innerCall(tableId, tagName, null, null); + } + + public String[] call( + ProcedureContext procedureContext, String tableId, String tagName, long snapshotId) + throws Catalog.TableNotExistException { + return innerCall(tableId, tagName, snapshotId, null); + } + + public String[] call( + ProcedureContext procedureContext, + String tableId, + String tagName, + long snapshotId, + String timeRetained) + throws Catalog.TableNotExistException { + return innerCall(tableId, tagName, snapshotId, timeRetained); + } + + public String[] call( + ProcedureContext procedureContext, String tableId, String tagName, String timeRetained) + throws Catalog.TableNotExistException { + return innerCall(tableId, tagName, null, timeRetained); + } + + private String[] innerCall( + String tableId, + String tagName, + @Nullable Long snapshotId, + @Nullable String timeRetained) + throws Catalog.TableNotExistException { + Table table = catalog.getTable(Identifier.fromString(tableId)); + createOrReplaceTag(table, tagName, snapshotId, toDuration(timeRetained)); + return new String[] {"Success"}; + } + + abstract void createOrReplaceTag( + Table table, String tagName, Long snapshotId, Duration timeRetained); + + @Nullable + private static Duration toDuration(@Nullable String s) { + if (s == null) { + return null; + } + + return TimeUtils.parseDuration(s); + } +} diff --git a/paimon-flink/paimon-flink-1.18/src/main/java/org/apache/paimon/flink/procedure/CreateTagProcedure.java b/paimon-flink/paimon-flink-1.18/src/main/java/org/apache/paimon/flink/procedure/CreateTagProcedure.java index 1a7b03ef6512..b1af1c93942f 100644 --- a/paimon-flink/paimon-flink-1.18/src/main/java/org/apache/paimon/flink/procedure/CreateTagProcedure.java +++ b/paimon-flink/paimon-flink-1.18/src/main/java/org/apache/paimon/flink/procedure/CreateTagProcedure.java @@ -18,14 +18,7 @@ package org.apache.paimon.flink.procedure; -import org.apache.paimon.catalog.Catalog; -import org.apache.paimon.catalog.Identifier; import org.apache.paimon.table.Table; -import org.apache.paimon.utils.TimeUtils; - -import org.apache.flink.table.procedure.ProcedureContext; - -import javax.annotation.Nullable; import java.time.Duration; @@ -36,59 +29,17 @@ * CALL sys.create_tag('tableId', 'tagName', snapshotId, 'timeRetained') * */ -public class CreateTagProcedure extends ProcedureBase { +public class CreateTagProcedure extends CreateOrReplaceTagBaseProcedure { public static final String IDENTIFIER = "create_tag"; - public String[] call( - ProcedureContext procedureContext, String tableId, String tagName, long snapshotId) - throws Catalog.TableNotExistException { - return innerCall(tableId, tagName, snapshotId, null); - } - - public String[] call(ProcedureContext procedureContext, String tableId, String tagName) - throws Catalog.TableNotExistException { - return innerCall(tableId, tagName, null, null); - } - - public String[] call( - ProcedureContext procedureContext, - String tableId, - String tagName, - long snapshotId, - String timeRetained) - throws Catalog.TableNotExistException { - return innerCall(tableId, tagName, snapshotId, timeRetained); - } - - public String[] call( - ProcedureContext procedureContext, String tableId, String tagName, String timeRetained) - throws Catalog.TableNotExistException { - return innerCall(tableId, tagName, null, timeRetained); - } - - private String[] innerCall( - String tableId, - String tagName, - @Nullable Long snapshotId, - @Nullable String timeRetained) - throws Catalog.TableNotExistException { - Table table = catalog.getTable(Identifier.fromString(tableId)); + @Override + void createOrReplaceTag(Table table, String tagName, Long snapshotId, Duration timeRetained) { if (snapshotId == null) { - table.createTag(tagName, toDuration(timeRetained)); + table.createTag(tagName, timeRetained); } else { - table.createTag(tagName, snapshotId, toDuration(timeRetained)); + table.createTag(tagName, snapshotId, timeRetained); } - return new String[] {"Success"}; - } - - @Nullable - private static Duration toDuration(@Nullable String s) { - if (s == null) { - return null; - } - - return TimeUtils.parseDuration(s); } @Override diff --git a/paimon-flink/paimon-flink-1.18/src/main/java/org/apache/paimon/flink/procedure/ExpirePartitionsProcedure.java b/paimon-flink/paimon-flink-1.18/src/main/java/org/apache/paimon/flink/procedure/ExpirePartitionsProcedure.java index c0e5a65c49ef..1c0d73cfbe38 100644 --- a/paimon-flink/paimon-flink-1.18/src/main/java/org/apache/paimon/flink/procedure/ExpirePartitionsProcedure.java +++ b/paimon-flink/paimon-flink-1.18/src/main/java/org/apache/paimon/flink/procedure/ExpirePartitionsProcedure.java @@ -93,9 +93,10 @@ public String[] call( .catalogEnvironment() .metastoreClientFactory()) .map(MetastoreClient.Factory::create) - .orElse(null)); + .orElse(null), + fileStore.options().partitionExpireMaxNum()); if (maxExpires != null) { - partitionExpire.withMaxExpires(maxExpires); + partitionExpire.withMaxExpireNum(maxExpires); } List> expired = partitionExpire.expire(Long.MAX_VALUE); return expired == null || expired.isEmpty() diff --git a/paimon-flink/paimon-flink-1.18/src/main/java/org/apache/paimon/flink/procedure/ExpireTagsProcedure.java b/paimon-flink/paimon-flink-1.18/src/main/java/org/apache/paimon/flink/procedure/ExpireTagsProcedure.java new file mode 100644 index 000000000000..183e39dc66a8 --- /dev/null +++ b/paimon-flink/paimon-flink-1.18/src/main/java/org/apache/paimon/flink/procedure/ExpireTagsProcedure.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.flink.procedure; + +import org.apache.paimon.catalog.Catalog; +import org.apache.paimon.table.FileStoreTable; +import org.apache.paimon.tag.TagTimeExpire; +import org.apache.paimon.utils.DateTimeUtils; + +import org.apache.flink.table.procedure.ProcedureContext; + +import java.time.LocalDateTime; +import java.util.List; +import java.util.TimeZone; + +/** A procedure to expire tags by time. */ +public class ExpireTagsProcedure extends ProcedureBase { + + private static final String IDENTIFIER = "expire_tags"; + + public String[] call(ProcedureContext procedureContext, String tableId) + throws Catalog.TableNotExistException { + return call(procedureContext, tableId, null); + } + + public String[] call(ProcedureContext procedureContext, String tableId, String olderThanStr) + throws Catalog.TableNotExistException { + FileStoreTable fileStoreTable = (FileStoreTable) table(tableId); + TagTimeExpire tagTimeExpire = + fileStoreTable.store().newTagCreationManager().getTagTimeExpire(); + if (olderThanStr != null) { + LocalDateTime olderThanTime = + DateTimeUtils.parseTimestampData(olderThanStr, 3, TimeZone.getDefault()) + .toLocalDateTime(); + tagTimeExpire.withOlderThanTime(olderThanTime); + } + List expired = tagTimeExpire.expire(); + return expired.isEmpty() + ? new String[] {"No expired tags."} + : expired.toArray(new String[0]); + } + + @Override + public String identifier() { + return IDENTIFIER; + } +} diff --git a/paimon-flink/paimon-flink-1.18/src/main/java/org/apache/paimon/flink/procedure/MigrateFileProcedure.java b/paimon-flink/paimon-flink-1.18/src/main/java/org/apache/paimon/flink/procedure/MigrateFileProcedure.java index 5d68cc0f5722..1e581c38cb97 100644 --- a/paimon-flink/paimon-flink-1.18/src/main/java/org/apache/paimon/flink/procedure/MigrateFileProcedure.java +++ b/paimon-flink/paimon-flink-1.18/src/main/java/org/apache/paimon/flink/procedure/MigrateFileProcedure.java @@ -18,6 +18,7 @@ package org.apache.paimon.flink.procedure; +import org.apache.paimon.catalog.Catalog; import org.apache.paimon.catalog.Identifier; import org.apache.paimon.flink.utils.TableMigrationUtils; import org.apache.paimon.migrate.Migrator; @@ -88,7 +89,9 @@ public void migrateHandle( Identifier sourceTableId = Identifier.fromString(sourceTablePath); Identifier targetTableId = Identifier.fromString(targetPaimonTablePath); - if (!(catalog.tableExists(targetTableId))) { + try { + catalog.getTable(targetTableId); + } catch (Catalog.TableNotExistException e) { throw new IllegalArgumentException( "Target paimon table does not exist: " + targetPaimonTablePath); } diff --git a/paimon-flink/paimon-flink-1.18/src/main/java/org/apache/paimon/flink/procedure/PurgeFilesProcedure.java b/paimon-flink/paimon-flink-1.18/src/main/java/org/apache/paimon/flink/procedure/PurgeFilesProcedure.java new file mode 100644 index 000000000000..3053eae3c7cf --- /dev/null +++ b/paimon-flink/paimon-flink-1.18/src/main/java/org/apache/paimon/flink/procedure/PurgeFilesProcedure.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.flink.procedure; + +import org.apache.paimon.catalog.Catalog; +import org.apache.paimon.catalog.Identifier; +import org.apache.paimon.fs.FileIO; +import org.apache.paimon.fs.Path; +import org.apache.paimon.table.FileStoreTable; +import org.apache.paimon.table.Table; + +import org.apache.flink.table.procedure.ProcedureContext; + +import java.io.IOException; +import java.util.Arrays; + +/** A procedure to purge files for a table. */ +public class PurgeFilesProcedure extends ProcedureBase { + public static final String IDENTIFIER = "purge_files"; + + public String[] call(ProcedureContext procedureContext, String tableId) + throws Catalog.TableNotExistException { + Table table = catalog.getTable(Identifier.fromString(tableId)); + FileStoreTable fileStoreTable = (FileStoreTable) table; + FileIO fileIO = fileStoreTable.fileIO(); + Path tablePath = fileStoreTable.snapshotManager().tablePath(); + try { + Arrays.stream(fileIO.listStatus(tablePath)) + .filter(f -> !f.getPath().getName().contains("schema")) + .forEach( + fileStatus -> { + try { + fileIO.delete(fileStatus.getPath(), true); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + } catch (IOException e) { + throw new RuntimeException(e); + } + return new String[] { + String.format("Success purge files with table: %s.", fileStoreTable.name()) + }; + } + + @Override + public String identifier() { + return IDENTIFIER; + } +} diff --git a/paimon-flink/paimon-flink-1.18/src/main/java/org/apache/paimon/flink/procedure/RemoveOrphanFilesProcedure.java b/paimon-flink/paimon-flink-1.18/src/main/java/org/apache/paimon/flink/procedure/RemoveOrphanFilesProcedure.java index c5fa7b7ba34a..b4a3a6b359d9 100644 --- a/paimon-flink/paimon-flink-1.18/src/main/java/org/apache/paimon/flink/procedure/RemoveOrphanFilesProcedure.java +++ b/paimon-flink/paimon-flink-1.18/src/main/java/org/apache/paimon/flink/procedure/RemoveOrphanFilesProcedure.java @@ -20,9 +20,13 @@ import org.apache.paimon.catalog.Identifier; import org.apache.paimon.flink.orphan.FlinkOrphanFilesClean; +import org.apache.paimon.operation.CleanOrphanFilesResult; +import org.apache.paimon.operation.LocalOrphanFilesClean; import org.apache.flink.table.procedure.ProcedureContext; +import java.util.Locale; + import static org.apache.paimon.operation.OrphanFilesClean.createFileCleaner; import static org.apache.paimon.operation.OrphanFilesClean.olderThanMillis; @@ -66,20 +70,61 @@ public String[] call( boolean dryRun, Integer parallelism) throws Exception { + return call(procedureContext, tableId, olderThan, dryRun, parallelism, null); + } + + public String[] call( + ProcedureContext procedureContext, + String tableId, + String olderThan, + boolean dryRun, + Integer parallelism, + String mode) + throws Exception { Identifier identifier = Identifier.fromString(tableId); String databaseName = identifier.getDatabaseName(); String tableName = identifier.getObjectName(); + if (mode == null) { + mode = "DISTRIBUTED"; + } - long deleted = - FlinkOrphanFilesClean.executeDatabaseOrphanFiles( - procedureContext.getExecutionEnvironment(), - catalog, - olderThanMillis(olderThan), - createFileCleaner(catalog, dryRun), - parallelism, - databaseName, - tableName); - return new String[] {String.valueOf(deleted)}; + CleanOrphanFilesResult cleanOrphanFilesResult; + try { + switch (mode.toUpperCase(Locale.ROOT)) { + case "DISTRIBUTED": + cleanOrphanFilesResult = + FlinkOrphanFilesClean.executeDatabaseOrphanFiles( + procedureContext.getExecutionEnvironment(), + catalog, + olderThanMillis(olderThan), + createFileCleaner(catalog, dryRun), + parallelism, + databaseName, + tableName); + break; + case "LOCAL": + cleanOrphanFilesResult = + LocalOrphanFilesClean.executeDatabaseOrphanFiles( + catalog, + databaseName, + tableName, + olderThanMillis(olderThan), + createFileCleaner(catalog, dryRun), + parallelism); + break; + default: + throw new IllegalArgumentException( + "Unknown mode: " + + mode + + ". Only 'DISTRIBUTED' and 'LOCAL' are supported."); + } + return new String[] { + String.valueOf(cleanOrphanFilesResult.getDeletedFileCount()), + String.valueOf(cleanOrphanFilesResult.getDeletedFileTotalLenInBytes()) + }; + } catch (Exception e) { + throw new RuntimeException(e); + } } @Override diff --git a/paimon-common/src/main/java/org/apache/paimon/lineage/LineageMetaFactory.java b/paimon-flink/paimon-flink-1.18/src/main/java/org/apache/paimon/flink/procedure/ReplaceTagProcedure.java similarity index 60% rename from paimon-common/src/main/java/org/apache/paimon/lineage/LineageMetaFactory.java rename to paimon-flink/paimon-flink-1.18/src/main/java/org/apache/paimon/flink/procedure/ReplaceTagProcedure.java index 11c6d3a1173c..6ed6ecc0e512 100644 --- a/paimon-common/src/main/java/org/apache/paimon/lineage/LineageMetaFactory.java +++ b/paimon-flink/paimon-flink-1.18/src/main/java/org/apache/paimon/flink/procedure/ReplaceTagProcedure.java @@ -16,22 +16,24 @@ * limitations under the License. */ -package org.apache.paimon.lineage; +package org.apache.paimon.flink.procedure; -import org.apache.paimon.factories.Factory; -import org.apache.paimon.options.Options; +import org.apache.paimon.table.Table; -import java.io.Serializable; +import java.time.Duration; -/** Factory to create {@link LineageMeta}. Each factory should have a unique identifier. */ -public interface LineageMetaFactory extends Factory, Serializable { +/** A procedure to replace a tag. */ +public class ReplaceTagProcedure extends CreateOrReplaceTagBaseProcedure { - LineageMeta create(LineageMetaContext context); + private static final String IDENTIFIER = "replace_tag"; - /** - * Context has all options in a catalog and is used in factory to create {@link LineageMeta}. - */ - interface LineageMetaContext { - Options options(); + @Override + void createOrReplaceTag(Table table, String tagName, Long snapshotId, Duration timeRetained) { + table.replaceTag(tagName, snapshotId, timeRetained); + } + + @Override + public String identifier() { + return IDENTIFIER; } } diff --git a/paimon-flink/paimon-flink-1.18/src/main/java/org/apache/paimon/flink/procedure/ResetConsumerProcedure.java b/paimon-flink/paimon-flink-1.18/src/main/java/org/apache/paimon/flink/procedure/ResetConsumerProcedure.java index 0355d6dc1cab..7777ccda19de 100644 --- a/paimon-flink/paimon-flink-1.18/src/main/java/org/apache/paimon/flink/procedure/ResetConsumerProcedure.java +++ b/paimon-flink/paimon-flink-1.18/src/main/java/org/apache/paimon/flink/procedure/ResetConsumerProcedure.java @@ -49,6 +49,7 @@ public String[] call( throws Catalog.TableNotExistException { FileStoreTable fileStoreTable = (FileStoreTable) catalog.getTable(Identifier.fromString(tableId)); + fileStoreTable.snapshotManager().snapshot(nextSnapshotId); ConsumerManager consumerManager = new ConsumerManager( fileStoreTable.fileIO(), diff --git a/paimon-flink/paimon-flink-1.18/src/main/java/org/apache/paimon/flink/procedure/RollbackToTimestampProcedure.java b/paimon-flink/paimon-flink-1.18/src/main/java/org/apache/paimon/flink/procedure/RollbackToTimestampProcedure.java new file mode 100644 index 000000000000..2e511f67a84d --- /dev/null +++ b/paimon-flink/paimon-flink-1.18/src/main/java/org/apache/paimon/flink/procedure/RollbackToTimestampProcedure.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.flink.procedure; + +import org.apache.paimon.Snapshot; +import org.apache.paimon.catalog.Catalog; +import org.apache.paimon.catalog.Identifier; +import org.apache.paimon.table.FileStoreTable; +import org.apache.paimon.table.Table; +import org.apache.paimon.utils.Preconditions; + +import org.apache.flink.table.procedure.ProcedureContext; + +/** + * Rollback to timestamp procedure. Usage: + * + * 

    
+ *  -- rollback to the snapshot which earlier or equal than timestamp.
+ *  CALL sys.rollback_to_timestamp('tableId', timestamp)
+ *
    + */ +public class RollbackToTimestampProcedure extends ProcedureBase { + + public static final String IDENTIFIER = "rollback_to_timestamp"; + + public String[] call(ProcedureContext procedureContext, String tableId, long timestamp) + throws Catalog.TableNotExistException { + Preconditions.checkNotNull(tableId, "table can not be empty"); + Table table = catalog.getTable(Identifier.fromString(tableId)); + FileStoreTable fileStoreTable = (FileStoreTable) table; + Snapshot snapshot = fileStoreTable.snapshotManager().earlierOrEqualTimeMills(timestamp); + Preconditions.checkNotNull( + snapshot, String.format("count not find snapshot earlier than %s", timestamp)); + long snapshotId = snapshot.id(); + fileStoreTable.rollbackTo(snapshotId); + return new String[] {String.format("Success roll back to snapshot: %s .", snapshotId)}; + } + + @Override + public String identifier() { + return IDENTIFIER; + } +} diff --git a/paimon-flink/paimon-flink-1.18/src/main/java/org/apache/paimon/flink/procedure/RollbackToWatermarkProcedure.java b/paimon-flink/paimon-flink-1.18/src/main/java/org/apache/paimon/flink/procedure/RollbackToWatermarkProcedure.java new file mode 100644 index 000000000000..da0b38f16b54 --- /dev/null +++ b/paimon-flink/paimon-flink-1.18/src/main/java/org/apache/paimon/flink/procedure/RollbackToWatermarkProcedure.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.flink.procedure; + +import org.apache.paimon.Snapshot; +import org.apache.paimon.catalog.Catalog; +import org.apache.paimon.catalog.Identifier; +import org.apache.paimon.table.FileStoreTable; +import org.apache.paimon.table.Table; +import org.apache.paimon.utils.Preconditions; + +import org.apache.flink.table.procedure.ProcedureContext; + +/** + * Rollback to watermark procedure. Usage: + * + * 
    
+ *  -- rollback to the snapshot which earlier or equal than watermark.
+ *  CALL sys.rollback_to_watermark('tableId', watermark)
+ *
    + */ +public class RollbackToWatermarkProcedure extends ProcedureBase { + + public static final String IDENTIFIER = "rollback_to_watermark"; + + public String[] call(ProcedureContext procedureContext, String tableId, long watermark) + throws Catalog.TableNotExistException { + Preconditions.checkNotNull(tableId, "table can not be empty"); + Table table = catalog.getTable(Identifier.fromString(tableId)); + FileStoreTable fileStoreTable = (FileStoreTable) table; + Snapshot snapshot = fileStoreTable.snapshotManager().earlierOrEqualWatermark(watermark); + Preconditions.checkNotNull( + snapshot, String.format("count not find snapshot earlier than %s", watermark)); + long snapshotId = snapshot.id(); + fileStoreTable.rollbackTo(snapshotId); + return new String[] {String.format("Success roll back to snapshot: %s .", snapshotId)}; + } + + @Override + public String identifier() { + return IDENTIFIER; + } +} diff --git a/paimon-flink/paimon-flink-1.18/src/main/java/org/apache/paimon/flink/utils/RuntimeContextUtils.java b/paimon-flink/paimon-flink-1.18/src/main/java/org/apache/paimon/flink/utils/RuntimeContextUtils.java new file mode 100644 index 000000000000..460fea55ad7a --- /dev/null +++ b/paimon-flink/paimon-flink-1.18/src/main/java/org/apache/paimon/flink/utils/RuntimeContextUtils.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.flink.utils; + +import org.apache.flink.api.common.functions.RuntimeContext; + +/** Utility methods about Flink runtime context to resolve compatibility issues. */ +public class RuntimeContextUtils { + public static int getNumberOfParallelSubtasks(RuntimeContext context) { + return context.getNumberOfParallelSubtasks(); + } + + public static int getIndexOfThisSubtask(RuntimeContext context) { + return context.getIndexOfThisSubtask(); + } +} diff --git a/paimon-flink/paimon-flink-1.18/src/test/java/org/apache/paimon/flink/RemoveOrphanFilesActionITCase.java b/paimon-flink/paimon-flink-1.18/src/test/java/org/apache/paimon/flink/RemoveOrphanFilesActionITCase.java new file mode 100644 index 000000000000..a168c3785c7c --- /dev/null +++ b/paimon-flink/paimon-flink-1.18/src/test/java/org/apache/paimon/flink/RemoveOrphanFilesActionITCase.java @@ -0,0 +1,299 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.flink; + +import org.apache.paimon.CoreOptions; +import org.apache.paimon.data.BinaryString; +import org.apache.paimon.data.GenericRow; +import org.apache.paimon.flink.action.ActionITCaseBase; +import org.apache.paimon.flink.action.RemoveOrphanFilesAction; +import org.apache.paimon.fs.FileIO; +import org.apache.paimon.fs.Path; +import org.apache.paimon.options.Options; +import org.apache.paimon.schema.SchemaChange; +import org.apache.paimon.schema.SchemaManager; +import org.apache.paimon.schema.TableSchema; +import org.apache.paimon.table.FileStoreTable; +import org.apache.paimon.table.FileStoreTableFactory; +import org.apache.paimon.table.sink.StreamTableCommit; +import org.apache.paimon.table.sink.StreamTableWrite; +import org.apache.paimon.table.sink.StreamWriteBuilder; +import org.apache.paimon.types.DataType; +import org.apache.paimon.types.DataTypes; +import org.apache.paimon.types.RowType; + +import org.apache.paimon.shade.guava30.com.google.common.collect.ImmutableList; + +import org.apache.flink.types.Row; +import org.apache.flink.util.CloseableIterator; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.ThreadLocalRandom; + +import static org.apache.paimon.CoreOptions.SCAN_FALLBACK_BRANCH; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatCode; + +/** IT cases base for {@link RemoveOrphanFilesAction} in Flink 1.18. */ +public class RemoveOrphanFilesActionITCase extends ActionITCaseBase { + private static final String ORPHAN_FILE_1 = "bucket-0/orphan_file1"; + private static final String ORPHAN_FILE_2 = "bucket-0/orphan_file2"; + + private FileStoreTable createTableAndWriteData(String tableName) throws Exception { + RowType rowType = + RowType.of( + new DataType[] {DataTypes.BIGINT(), DataTypes.STRING()}, + new String[] {"k", "v"}); + + FileStoreTable table = + createFileStoreTable( + tableName, + rowType, + Collections.emptyList(), + Collections.singletonList("k"), + Collections.emptyList(), + Collections.emptyMap()); + + StreamWriteBuilder writeBuilder = table.newStreamWriteBuilder().withCommitUser(commitUser); + write = writeBuilder.newWrite(); + commit = writeBuilder.newCommit(); + + writeData(rowData(1L, BinaryString.fromString("Hi"))); + + Path orphanFile1 = getOrphanFilePath(table, ORPHAN_FILE_1); + Path orphanFile2 = getOrphanFilePath(table, ORPHAN_FILE_2); + + FileIO fileIO = table.fileIO(); + fileIO.writeFile(orphanFile1, "a", true); + Thread.sleep(2000); + fileIO.writeFile(orphanFile2, "b", true); + + return table; + } + + private Path getOrphanFilePath(FileStoreTable table, String orphanFile) { + return new Path(table.location(), orphanFile); + } + + @Test + public void testRunWithoutException() throws Exception { + createTableAndWriteData(tableName); + + List args = + new ArrayList<>( + Arrays.asList( + "remove_orphan_files", + "--warehouse", + warehouse, + "--database", + database, + "--table", + tableName)); + RemoveOrphanFilesAction action1 = createAction(RemoveOrphanFilesAction.class, args); + assertThatCode(action1::run).doesNotThrowAnyException(); + + args.add("--older_than"); + args.add("2023-12-31 23:59:59"); + RemoveOrphanFilesAction action2 = createAction(RemoveOrphanFilesAction.class, args); + assertThatCode(action2::run).doesNotThrowAnyException(); + + String withoutOlderThan = + String.format("CALL sys.remove_orphan_files('%s.%s')", database, tableName); + + CloseableIterator withoutOlderThanCollect = executeSQL(withoutOlderThan); + assertThat(ImmutableList.copyOf(withoutOlderThanCollect)).containsOnly(Row.of("0")); + + String withDryRun = + String.format( + "CALL sys.remove_orphan_files('%s.%s', '2999-12-31 23:59:59', true)", + database, tableName); + ImmutableList actualDryRunDeleteFile = ImmutableList.copyOf(executeSQL(withDryRun)); + assertThat(actualDryRunDeleteFile).containsOnly(Row.of("2")); + + String withOlderThan = + String.format( + "CALL sys.remove_orphan_files('%s.%s', '2999-12-31 23:59:59')", + database, tableName); + ImmutableList actualDeleteFile = ImmutableList.copyOf(executeSQL(withOlderThan)); + + assertThat(actualDeleteFile).containsExactlyInAnyOrder(Row.of("2"), Row.of("2")); + } + + @Test + public void testRemoveDatabaseOrphanFilesITCase() throws Exception { + createTableAndWriteData("tableName1"); + createTableAndWriteData("tableName2"); + + List args = + new ArrayList<>( + Arrays.asList( + "remove_orphan_files", + "--warehouse", + warehouse, + "--database", + database, + "--table", + "*")); + RemoveOrphanFilesAction action1 = createAction(RemoveOrphanFilesAction.class, args); + assertThatCode(action1::run).doesNotThrowAnyException(); + + args.add("--older_than"); + args.add("2023-12-31 23:59:59"); + RemoveOrphanFilesAction action2 = createAction(RemoveOrphanFilesAction.class, args); + assertThatCode(action2::run).doesNotThrowAnyException(); + + args.add("--parallelism"); + args.add("5"); + RemoveOrphanFilesAction action3 = createAction(RemoveOrphanFilesAction.class, args); + assertThatCode(action3::run).doesNotThrowAnyException(); + + String withoutOlderThan = + String.format("CALL sys.remove_orphan_files('%s.%s')", database, "*"); + CloseableIterator withoutOlderThanCollect = executeSQL(withoutOlderThan); + assertThat(ImmutableList.copyOf(withoutOlderThanCollect)).containsOnly(Row.of("0")); + + String withParallelism = + String.format("CALL sys.remove_orphan_files('%s.%s','',true,5)", database, "*"); + CloseableIterator withParallelismCollect = executeSQL(withParallelism); + assertThat(ImmutableList.copyOf(withParallelismCollect)).containsOnly(Row.of("0")); + + String withDryRun = + String.format( + "CALL sys.remove_orphan_files('%s.%s', '2999-12-31 23:59:59', true)", + database, "*"); + ImmutableList actualDryRunDeleteFile = ImmutableList.copyOf(executeSQL(withDryRun)); + assertThat(actualDryRunDeleteFile).containsOnly(Row.of("4")); + + String withOlderThan = + String.format( + "CALL sys.remove_orphan_files('%s.%s', '2999-12-31 23:59:59')", + database, "*"); + ImmutableList actualDeleteFile = ImmutableList.copyOf(executeSQL(withOlderThan)); + + assertThat(actualDeleteFile).containsOnly(Row.of("4")); + } + + @Test + public void testCleanWithBranch() throws Exception { + // create main branch + FileStoreTable table = createTableAndWriteData(tableName); + + // create first branch and write some data + table.createBranch("br"); + SchemaManager schemaManager = new SchemaManager(table.fileIO(), table.location(), "br"); + TableSchema branchSchema = + schemaManager.commitChanges(SchemaChange.addColumn("v2", DataTypes.INT())); + Options branchOptions = new Options(branchSchema.options()); + branchOptions.set(CoreOptions.BRANCH, "br"); + branchSchema = branchSchema.copy(branchOptions.toMap()); + FileStoreTable branchTable = + FileStoreTableFactory.create(table.fileIO(), table.location(), branchSchema); + + String commitUser = UUID.randomUUID().toString(); + StreamTableWrite write = branchTable.newWrite(commitUser); + StreamTableCommit commit = branchTable.newCommit(commitUser); + write.write(GenericRow.of(2L, BinaryString.fromString("Hello"), 20)); + commit.commit(1, write.prepareCommit(false, 1)); + write.close(); + commit.close(); + + // create orphan file in snapshot directory of first branch + Path orphanFile3 = new Path(table.location(), "branch/branch-br/snapshot/orphan_file3"); + branchTable.fileIO().writeFile(orphanFile3, "x", true); + + // create second branch, which is empty + table.createBranch("br2"); + + // create orphan file in snapshot directory of second branch + Path orphanFile4 = new Path(table.location(), "branch/branch-br2/snapshot/orphan_file4"); + branchTable.fileIO().writeFile(orphanFile4, "y", true); + + if (ThreadLocalRandom.current().nextBoolean()) { + executeSQL( + String.format( + "ALTER TABLE `%s`.`%s` SET ('%s' = 'br')", + database, tableName, SCAN_FALLBACK_BRANCH.key()), + false, + true); + } + String procedure = + String.format( + "CALL sys.remove_orphan_files('%s.%s', '2999-12-31 23:59:59')", + database, "*"); + ImmutableList actualDeleteFile = ImmutableList.copyOf(executeSQL(procedure)); + assertThat(actualDeleteFile).containsOnly(Row.of("4")); + } + + @Test + public void testRunWithMode() throws Exception { + createTableAndWriteData(tableName); + + List args = + new ArrayList<>( + Arrays.asList( + "remove_orphan_files", + "--warehouse", + warehouse, + "--database", + database, + "--table", + tableName)); + RemoveOrphanFilesAction action1 = createAction(RemoveOrphanFilesAction.class, args); + assertThatCode(action1::run).doesNotThrowAnyException(); + + args.add("--older_than"); + args.add("2023-12-31 23:59:59"); + RemoveOrphanFilesAction action2 = createAction(RemoveOrphanFilesAction.class, args); + assertThatCode(action2::run).doesNotThrowAnyException(); + + String withoutOlderThan = + String.format("CALL sys.remove_orphan_files('%s.%s')", database, tableName); + CloseableIterator withoutOlderThanCollect = executeSQL(withoutOlderThan); + assertThat(ImmutableList.copyOf(withoutOlderThanCollect)).containsOnly(Row.of("0")); + + String withLocalMode = + String.format( + "CALL sys.remove_orphan_files('%s.%s', '2999-12-31 23:59:59', true, 5, 'local')", + database, tableName); + ImmutableList actualLocalRunDeleteFile = + ImmutableList.copyOf(executeSQL(withLocalMode)); + assertThat(actualLocalRunDeleteFile).containsOnly(Row.of("2")); + + String withDistributedMode = + String.format( + "CALL sys.remove_orphan_files('%s.%s', '2999-12-31 23:59:59', true, 5, 'distributed')", + database, tableName); + ImmutableList actualDistributedRunDeleteFile = + ImmutableList.copyOf(executeSQL(withDistributedMode)); + assertThat(actualDistributedRunDeleteFile).containsOnly(Row.of("2")); + + String withInvalidMode = + String.format( + "CALL sys.remove_orphan_files('%s.%s', '2999-12-31 23:59:59', true, 5, 'unknown')", + database, tableName); + assertThatCode(() -> executeSQL(withInvalidMode)) + .isInstanceOf(RuntimeException.class) + .hasMessageContaining("Unknown mode"); + } +} diff --git a/paimon-flink/paimon-flink-1.18/src/test/java/org/apache/paimon/flink/iceberg/Flink118IcebergITCase.java b/paimon-flink/paimon-flink-1.18/src/test/java/org/apache/paimon/flink/iceberg/Flink118IcebergITCase.java new file mode 100644 index 000000000000..bf309a9f4182 --- /dev/null +++ b/paimon-flink/paimon-flink-1.18/src/test/java/org/apache/paimon/flink/iceberg/Flink118IcebergITCase.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.flink.iceberg; + +/** IT cases for Paimon Iceberg compatibility in Flink 1.18. */ +public class Flink118IcebergITCase extends FlinkIcebergITCaseBase {} diff --git a/paimon-flink/paimon-flink-1.18/src/test/java/org/apache/paimon/flink/procedure/ProcedurePositionalArgumentsITCase.java b/paimon-flink/paimon-flink-1.18/src/test/java/org/apache/paimon/flink/procedure/ProcedurePositionalArgumentsITCase.java index a48de667bf3d..f79d6fb716b4 100644 --- a/paimon-flink/paimon-flink-1.18/src/test/java/org/apache/paimon/flink/procedure/ProcedurePositionalArgumentsITCase.java +++ b/paimon-flink/paimon-flink-1.18/src/test/java/org/apache/paimon/flink/procedure/ProcedurePositionalArgumentsITCase.java @@ -24,6 +24,7 @@ import org.apache.flink.types.Row; import org.apache.flink.util.CloseableIterator; +import org.assertj.core.api.Assertions; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.function.Executable; @@ -56,6 +57,8 @@ public void testCompactDatabaseAndTable() { assertThatCode(() -> sql("CALL sys.compact('default.T')")).doesNotThrowAnyException(); assertThatCode(() -> sql("CALL sys.compact('default.T', 'pt=1')")) .doesNotThrowAnyException(); + assertThatCode(() -> sql("CALL sys.compact('default.T', '', '')")) + .doesNotThrowAnyException(); assertThatCode(() -> sql("CALL sys.compact('default.T', 'pt=1', '', '')")) .doesNotThrowAnyException(); assertThatCode(() -> sql("CALL sys.compact('default.T', '', '', '', 'sink.parallelism=1')")) @@ -65,10 +68,16 @@ public void testCompactDatabaseAndTable() { sql( "CALL sys.compact('default.T', '', '', '', 'sink.parallelism=1','pt=1')")) .doesNotThrowAnyException(); - assertThatCode(() -> sql("CALL sys.compact('default.T', '', 'zorder', 'k', '','','5s')")) + assertThatCode( + () -> + sql( + "CALL sys.compact('default.T', '' ,'zorder', 'k', '','','5s', '')")) .message() .contains("sort compact do not support 'partition_idle_time'."); + assertThatCode(() -> sql("CALL sys.compact('default.T', '', '' ,'', '', '', '', 'full')")) + .doesNotThrowAnyException(); + assertThatCode(() -> sql("CALL sys.compact_database('default')")) .doesNotThrowAnyException(); } @@ -320,13 +329,13 @@ public void testCreateDeleteAndForwardBranch() throws Exception { sql("CALL sys.create_branch('default.T', 'test', 'tag1')"); sql("CALL sys.create_branch('default.T', 'test2', 'tag2')"); - assertThat(collectToString("SELECT branch_name, created_from_snapshot FROM `T$branches`")) - .containsExactlyInAnyOrder("+I[test, 1]", "+I[test2, 2]"); + assertThat(collectToString("SELECT branch_name FROM `T$branches`")) + .containsExactlyInAnyOrder("+I[test]", "+I[test2]"); sql("CALL sys.delete_branch('default.T', 'test')"); - assertThat(collectToString("SELECT branch_name, created_from_snapshot FROM `T$branches`")) - .containsExactlyInAnyOrder("+I[test2, 2]"); + assertThat(collectToString("SELECT branch_name FROM `T$branches`")) + .containsExactlyInAnyOrder("+I[test2]"); sql("CALL sys.fast_forward('default.T', 'test2')"); @@ -489,4 +498,60 @@ public void testRewriteFileIndex() { assertThatCode(() -> sql("CALL sys.rewrite_file_index('default.T', 'pt = 0')")) .doesNotThrowAnyException(); } + + @Test + public void testExpireTags() throws Exception { + sql( + "CREATE TABLE T (" + + " k STRING," + + " dt STRING," + + " PRIMARY KEY (k, dt) NOT ENFORCED" + + ") PARTITIONED BY (dt) WITH (" + + " 'bucket' = '1'" + + ")"); + FileStoreTable table = paimonTable("T"); + for (int i = 1; i <= 3; i++) { + sql("INSERT INTO T VALUES ('" + i + "', '" + i + "')"); + } + assertThat(table.snapshotManager().snapshotCount()).isEqualTo(3L); + + sql("CALL sys.create_tag('default.T', 'tag-1', 1)"); + sql("CALL sys.create_tag('default.T', 'tag-2', 2, '1d')"); + sql("CALL sys.create_tag('default.T', 'tag-3', 3, '1s')"); + + assertThat(sql("select count(*) from `T$tags`")).containsExactly(Row.of(3L)); + + Thread.sleep(1000); + assertThat(sql("CALL sys.expire_tags('default.T')")) + .containsExactlyInAnyOrder(Row.of("tag-3")); + } + + @Test + public void testReplaceTags() throws Exception { + sql( + "CREATE TABLE T (" + + " id INT," + + " NAME STRING," + + " PRIMARY KEY (id) NOT ENFORCED" + + ") WITH ('bucket' = '1'" + + ")"); + sql("INSERT INTO T VALUES (1, 'a')"); + sql("INSERT INTO T VALUES (2, 'b')"); + assertThat(paimonTable("T").snapshotManager().snapshotCount()).isEqualTo(2L); + + Assertions.assertThatThrownBy(() -> sql("CALL sys.replace_tag('default.T', 'test_tag')")) + .hasMessageContaining("Tag name 'test_tag' does not exist."); + + sql("CALL sys.create_tag('default.T', 'test_tag')"); + assertThat(sql("select tag_name,snapshot_id,time_retained from `T$tags`")) + .containsExactly(Row.of("test_tag", 2L, null)); + + sql("CALL sys.replace_tag('default.T', 'test_tag', 1)"); + assertThat(sql("select tag_name,snapshot_id,time_retained from `T$tags`")) + .containsExactly(Row.of("test_tag", 1L, null)); + + sql("CALL sys.replace_tag('default.T', 'test_tag', 2, '1 d')"); + assertThat(sql("select tag_name,snapshot_id,time_retained from `T$tags`")) + .containsExactly(Row.of("test_tag", 2L, "PT24H")); + } } diff --git a/paimon-flink/paimon-flink-1.19/pom.xml b/paimon-flink/paimon-flink-1.19/pom.xml index 4f1b5bbfe418..736ac4c3076d 100644 --- a/paimon-flink/paimon-flink-1.19/pom.xml +++ b/paimon-flink/paimon-flink-1.19/pom.xml @@ -35,6 +35,7 @@ under the License. 1.19.1 + 1.19 @@ -95,7 +96,7 @@ under the License. org.apache.flink - flink-table-planner_${scala.binary.version} + flink-table-planner_${flink.scala.binary.version} ${flink.version} test @@ -113,6 +114,13 @@ under the License. 1.21 test + + + org.apache.iceberg + iceberg-flink-${iceberg.flink.version} + ${iceberg.version} + test + diff --git a/paimon-flink/paimon-flink-1.19/src/test/java/org/apache/paimon/flink/RemoveOrphanFilesActionITCase.java b/paimon-flink/paimon-flink-1.19/src/test/java/org/apache/paimon/flink/RemoveOrphanFilesActionITCase.java new file mode 100644 index 000000000000..e1be410b8cb1 --- /dev/null +++ b/paimon-flink/paimon-flink-1.19/src/test/java/org/apache/paimon/flink/RemoveOrphanFilesActionITCase.java @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.flink; + +import org.apache.paimon.flink.action.RemoveOrphanFilesAction; +import org.apache.paimon.flink.action.RemoveOrphanFilesActionITCaseBase; + +/** IT cases base for {@link RemoveOrphanFilesAction} in Flink 1.19. */ +public class RemoveOrphanFilesActionITCase extends RemoveOrphanFilesActionITCaseBase {} diff --git a/paimon-flink/paimon-flink-1.19/src/test/java/org/apache/paimon/flink/iceberg/Flink119IcebergITCase.java b/paimon-flink/paimon-flink-1.19/src/test/java/org/apache/paimon/flink/iceberg/Flink119IcebergITCase.java new file mode 100644 index 000000000000..eb64a6c9de37 --- /dev/null +++ b/paimon-flink/paimon-flink-1.19/src/test/java/org/apache/paimon/flink/iceberg/Flink119IcebergITCase.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.flink.iceberg; + +/** IT cases for Paimon Iceberg compatibility in Flink 1.19. */ +public class Flink119IcebergITCase extends FlinkIcebergITCaseBase {} diff --git a/paimon-flink/paimon-flink-1.20/pom.xml b/paimon-flink/paimon-flink-1.20/pom.xml index 7cf1d8e98df7..f15792d2bea9 100644 --- a/paimon-flink/paimon-flink-1.20/pom.xml +++ b/paimon-flink/paimon-flink-1.20/pom.xml @@ -55,6 +55,20 @@ under the License. + + + org.apache.flink + flink-streaming-java + ${flink.version} + provided + + + + org.apache.flink + flink-table-common + ${flink.version} + provided + diff --git a/paimon-flink/paimon-flink-cdc/pom.xml b/paimon-flink/paimon-flink-cdc/pom.xml index 48f0d5a13d63..792c6c14378b 100644 --- a/paimon-flink/paimon-flink-cdc/pom.xml +++ b/paimon-flink/paimon-flink-cdc/pom.xml @@ -34,7 +34,7 @@ under the License. Paimon : Flink : CDC - 1.18.1 + 1.20.0 3.1.1 3.1.1 1.11.4 @@ -43,7 +43,7 @@ under the License. 1.19.1 4.0.0-1.17 7.5.0 - 3.0.1-1.18 + 3.3.0-1.20 @@ -167,6 +167,13 @@ under the License. + + commons-codec + commons-codec + 1.9 + test + + org.apache.paimon paimon-common @@ -185,14 +192,14 @@ under the License. org.apache.flink - flink-table-planner_${scala.binary.version} + flink-table-planner_${flink.scala.binary.version} ${flink.version} test org.apache.flink - flink-table-planner_${scala.binary.version} + flink-table-planner_${flink.scala.binary.version} ${flink.version} test test-jar diff --git a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/CdcActionCommonUtils.java b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/CdcActionCommonUtils.java index 8f96022bde35..c8af6f91c420 100644 --- a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/CdcActionCommonUtils.java +++ b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/CdcActionCommonUtils.java @@ -56,6 +56,9 @@ public class CdcActionCommonUtils { public static final String PULSAR_CONF = "pulsar_conf"; public static final String TABLE_PREFIX = "table_prefix"; public static final String TABLE_SUFFIX = "table_suffix"; + public static final String TABLE_PREFIX_DB = "table_prefix_db"; + public static final String TABLE_SUFFIX_DB = "table_suffix_db"; + public static final String TABLE_MAPPING = "table_mapping"; public static final String INCLUDING_TABLES = "including_tables"; public static final String EXCLUDING_TABLES = "excluding_tables"; public static final String TYPE_MAPPING = "type_mapping"; diff --git a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/SyncDatabaseActionBase.java b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/SyncDatabaseActionBase.java index 4ab56bdcf118..4fb1339c5193 100644 --- a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/SyncDatabaseActionBase.java +++ b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/SyncDatabaseActionBase.java @@ -52,6 +52,9 @@ public abstract class SyncDatabaseActionBase extends SynchronizationActionBase { protected MultiTablesSinkMode mode = COMBINED; protected String tablePrefix = ""; protected String tableSuffix = ""; + protected Map tableMapping = new HashMap<>(); + protected Map dbPrefix = new HashMap<>(); + protected Map dbSuffix = new HashMap<>(); protected String includingTables = ".*"; protected List partitionKeys = new ArrayList<>(); protected List primaryKeys = new ArrayList<>(); @@ -97,6 +100,37 @@ public SyncDatabaseActionBase withTableSuffix(@Nullable String tableSuffix) { return this; } + public SyncDatabaseActionBase withDbPrefix(Map dbPrefix) { + if (dbPrefix != null) { + this.dbPrefix = + dbPrefix.entrySet().stream() + .collect( + HashMap::new, + (m, e) -> m.put(e.getKey().toLowerCase(), e.getValue()), + HashMap::putAll); + } + return this; + } + + public SyncDatabaseActionBase withDbSuffix(Map dbSuffix) { + if (dbSuffix != null) { + this.dbSuffix = + dbSuffix.entrySet().stream() + .collect( + HashMap::new, + (m, e) -> m.put(e.getKey().toLowerCase(), e.getValue()), + HashMap::putAll); + } + return this; + } + + public SyncDatabaseActionBase withTableMapping(Map tableMapping) { + if (tableMapping != null) { + this.tableMapping = tableMapping; + } + return this; + } + public SyncDatabaseActionBase includingTables(@Nullable String includingTables) { if (includingTables != null) { this.includingTables = includingTables; @@ -155,7 +189,14 @@ protected EventParser.Factory buildEventParserFactory() Pattern excludingPattern = excludingTables == null ? null : Pattern.compile(excludingTables); TableNameConverter tableNameConverter = - new TableNameConverter(allowUpperCase, mergeShards, tablePrefix, tableSuffix); + new TableNameConverter( + allowUpperCase, + mergeShards, + dbPrefix, + dbSuffix, + tablePrefix, + tableSuffix, + tableMapping); Set createdTables; try { createdTables = new HashSet<>(catalog.listTables(database)); diff --git a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/SyncDatabaseActionFactoryBase.java b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/SyncDatabaseActionFactoryBase.java index e7a386979d4e..d497b588c2af 100644 --- a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/SyncDatabaseActionFactoryBase.java +++ b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/SyncDatabaseActionFactoryBase.java @@ -29,8 +29,11 @@ import static org.apache.paimon.flink.action.cdc.CdcActionCommonUtils.MULTIPLE_TABLE_PARTITION_KEYS; import static org.apache.paimon.flink.action.cdc.CdcActionCommonUtils.PARTITION_KEYS; import static org.apache.paimon.flink.action.cdc.CdcActionCommonUtils.PRIMARY_KEYS; +import static org.apache.paimon.flink.action.cdc.CdcActionCommonUtils.TABLE_MAPPING; import static org.apache.paimon.flink.action.cdc.CdcActionCommonUtils.TABLE_PREFIX; +import static org.apache.paimon.flink.action.cdc.CdcActionCommonUtils.TABLE_PREFIX_DB; import static org.apache.paimon.flink.action.cdc.CdcActionCommonUtils.TABLE_SUFFIX; +import static org.apache.paimon.flink.action.cdc.CdcActionCommonUtils.TABLE_SUFFIX_DB; import static org.apache.paimon.flink.action.cdc.CdcActionCommonUtils.TYPE_MAPPING; /** Base {@link ActionFactory} for synchronizing into database. */ @@ -51,6 +54,9 @@ public Optional create(MultipleParameterToolAdapter params) { protected void withParams(MultipleParameterToolAdapter params, T action) { action.withTablePrefix(params.get(TABLE_PREFIX)) .withTableSuffix(params.get(TABLE_SUFFIX)) + .withDbPrefix(optionalConfigMap(params, TABLE_PREFIX_DB)) + .withDbSuffix(optionalConfigMap(params, TABLE_SUFFIX_DB)) + .withTableMapping(optionalConfigMap(params, TABLE_MAPPING)) .includingTables(params.get(INCLUDING_TABLES)) .excludingTables(params.get(EXCLUDING_TABLES)) .withPartitionKeyMultiple( diff --git a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/SyncTableActionBase.java b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/SyncTableActionBase.java index 4c7db6d28b62..87efeb2a19cf 100644 --- a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/SyncTableActionBase.java +++ b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/SyncTableActionBase.java @@ -122,7 +122,7 @@ protected void validateCaseSensitivity() { protected void beforeBuildingSourceSink() throws Exception { Identifier identifier = new Identifier(database, table); // Check if table exists before trying to get or create it - if (catalog.tableExists(identifier)) { + try { fileStoreTable = (FileStoreTable) catalog.getTable(identifier); fileStoreTable = alterTableOptions(identifier, fileStoreTable); try { @@ -146,7 +146,7 @@ protected void beforeBuildingSourceSink() throws Exception { // check partition keys and primary keys in case that user specified them checkConstraints(); } - } else { + } catch (Catalog.TableNotExistException e) { Schema retrievedSchema = retrieveSchema(); computedColumns = buildComputedColumns(computedColumnArgs, retrievedSchema.fields()); Schema paimonSchema = buildPaimonSchema(retrievedSchema); diff --git a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/SynchronizationActionBase.java b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/SynchronizationActionBase.java index 2b9b08917700..a7c770347410 100644 --- a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/SynchronizationActionBase.java +++ b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/SynchronizationActionBase.java @@ -32,13 +32,14 @@ import org.apache.paimon.schema.SchemaChange; import org.apache.paimon.table.FileStoreTable; +import org.apache.flink.api.common.RuntimeExecutionMode; import org.apache.flink.api.common.eventtime.WatermarkStrategy; import org.apache.flink.api.common.functions.FlatMapFunction; import org.apache.flink.api.connector.source.Source; import org.apache.flink.configuration.Configuration; +import org.apache.flink.configuration.ExecutionOptions; import org.apache.flink.streaming.api.datastream.DataStream; import org.apache.flink.streaming.api.datastream.DataStreamSource; -import org.apache.flink.streaming.api.functions.source.SourceFunction; import java.time.Duration; import java.util.HashMap; @@ -53,6 +54,7 @@ import static org.apache.paimon.flink.FlinkConnectorOptions.SCAN_WATERMARK_ALIGNMENT_MAX_DRIFT; import static org.apache.paimon.flink.FlinkConnectorOptions.SCAN_WATERMARK_ALIGNMENT_UPDATE_INTERVAL; import static org.apache.paimon.flink.FlinkConnectorOptions.SCAN_WATERMARK_IDLE_TIMEOUT; +import static org.apache.paimon.utils.Preconditions.checkArgument; /** Base {@link Action} for table/database synchronizing job. */ public abstract class SynchronizationActionBase extends ActionBase { @@ -128,7 +130,7 @@ public void build() throws Exception { protected void beforeBuildingSourceSink() throws Exception {} - protected Object buildSource() { + protected Source buildSource() { return syncJobHandler.provideSource(); } @@ -137,41 +139,39 @@ protected CdcTimestampExtractor createCdcTimestampExtractor() { "Unsupported timestamp extractor for current cdc source."); } - private DataStreamSource buildDataStreamSource(Object source) { - if (source instanceof Source) { - boolean isAutomaticWatermarkCreationEnabled = - tableConfig.containsKey(CoreOptions.TAG_AUTOMATIC_CREATION.key()) - && Objects.equals( - tableConfig.get(CoreOptions.TAG_AUTOMATIC_CREATION.key()), - WATERMARK.toString()); - - Options options = Options.fromMap(tableConfig); - Duration idleTimeout = options.get(SCAN_WATERMARK_IDLE_TIMEOUT); - String watermarkAlignGroup = options.get(SCAN_WATERMARK_ALIGNMENT_GROUP); - WatermarkStrategy watermarkStrategy = - isAutomaticWatermarkCreationEnabled - ? watermarkAlignGroup != null - ? new CdcWatermarkStrategy(createCdcTimestampExtractor()) - .withWatermarkAlignment( - watermarkAlignGroup, - options.get(SCAN_WATERMARK_ALIGNMENT_MAX_DRIFT), - options.get( - SCAN_WATERMARK_ALIGNMENT_UPDATE_INTERVAL)) - : new CdcWatermarkStrategy(createCdcTimestampExtractor()) - : WatermarkStrategy.noWatermarks(); - if (idleTimeout != null) { - watermarkStrategy = watermarkStrategy.withIdleness(idleTimeout); - } - return env.fromSource( - (Source) source, - watermarkStrategy, - syncJobHandler.provideSourceName()); - } - if (source instanceof SourceFunction) { - return env.addSource( - (SourceFunction) source, syncJobHandler.provideSourceName()); + protected void validateRuntimeExecutionMode() { + checkArgument( + env.getConfiguration().get(ExecutionOptions.RUNTIME_MODE) + == RuntimeExecutionMode.STREAMING, + "It's only support STREAMING mode for flink-cdc sync table action."); + } + + private DataStreamSource buildDataStreamSource( + Source source) { + boolean isAutomaticWatermarkCreationEnabled = + tableConfig.containsKey(CoreOptions.TAG_AUTOMATIC_CREATION.key()) + && Objects.equals( + tableConfig.get(CoreOptions.TAG_AUTOMATIC_CREATION.key()), + WATERMARK.toString()); + + Options options = Options.fromMap(tableConfig); + Duration idleTimeout = options.get(SCAN_WATERMARK_IDLE_TIMEOUT); + String watermarkAlignGroup = options.get(SCAN_WATERMARK_ALIGNMENT_GROUP); + WatermarkStrategy watermarkStrategy = + isAutomaticWatermarkCreationEnabled + ? watermarkAlignGroup != null + ? new CdcWatermarkStrategy(createCdcTimestampExtractor()) + .withWatermarkAlignment( + watermarkAlignGroup, + options.get(SCAN_WATERMARK_ALIGNMENT_MAX_DRIFT), + options.get( + SCAN_WATERMARK_ALIGNMENT_UPDATE_INTERVAL)) + : new CdcWatermarkStrategy(createCdcTimestampExtractor()) + : WatermarkStrategy.noWatermarks(); + if (idleTimeout != null) { + watermarkStrategy = watermarkStrategy.withIdleness(idleTimeout); } - throw new UnsupportedOperationException("Unrecognized source type"); + return env.fromSource(source, watermarkStrategy, syncJobHandler.provideSourceName()); } protected abstract FlatMapFunction recordParse(); diff --git a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/TableNameConverter.java b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/TableNameConverter.java index 67c70aa58cdb..15fc3507ce2d 100644 --- a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/TableNameConverter.java +++ b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/TableNameConverter.java @@ -21,6 +21,8 @@ import org.apache.paimon.catalog.Identifier; import java.io.Serializable; +import java.util.HashMap; +import java.util.Map; /** Used to convert a MySQL source table name to corresponding Paimon table name. */ public class TableNameConverter implements Serializable { @@ -29,24 +31,70 @@ public class TableNameConverter implements Serializable { private final boolean caseSensitive; private final boolean mergeShards; + private final Map dbPrefix; + private final Map dbSuffix; private final String prefix; private final String suffix; + private final Map tableMapping; public TableNameConverter(boolean caseSensitive) { - this(caseSensitive, true, "", ""); + this(caseSensitive, true, "", "", null); } public TableNameConverter( - boolean caseSensitive, boolean mergeShards, String prefix, String suffix) { + boolean caseSensitive, + boolean mergeShards, + String prefix, + String suffix, + Map tableMapping) { + this( + caseSensitive, + mergeShards, + new HashMap<>(), + new HashMap<>(), + prefix, + suffix, + tableMapping); + } + + public TableNameConverter( + boolean caseSensitive, + boolean mergeShards, + Map dbPrefix, + Map dbSuffix, + String prefix, + String suffix, + Map tableMapping) { this.caseSensitive = caseSensitive; this.mergeShards = mergeShards; + this.dbPrefix = dbPrefix; + this.dbSuffix = dbSuffix; this.prefix = prefix; this.suffix = suffix; + this.tableMapping = lowerMapKey(tableMapping); } - public String convert(String originName) { - String tableName = caseSensitive ? originName : originName.toLowerCase(); - return prefix + tableName + suffix; + public String convert(String originDbName, String originTblName) { + // top priority: table mapping + if (tableMapping.containsKey(originTblName.toLowerCase())) { + String mappedName = tableMapping.get(originTblName.toLowerCase()); + return caseSensitive ? mappedName : mappedName.toLowerCase(); + } + + String tblPrefix = prefix; + String tblSuffix = suffix; + + // second priority: prefix and postfix specified by db + if (dbPrefix.containsKey(originDbName.toLowerCase())) { + tblPrefix = dbPrefix.get(originDbName.toLowerCase()); + } + if (dbSuffix.containsKey(originDbName.toLowerCase())) { + tblSuffix = dbSuffix.get(originDbName.toLowerCase()); + } + + // third priority: normal prefix and suffix + String tableName = caseSensitive ? originTblName : originTblName.toLowerCase(); + return tblPrefix + tableName + tblSuffix; } public String convert(Identifier originIdentifier) { @@ -56,6 +104,20 @@ public String convert(Identifier originIdentifier) { : originIdentifier.getDatabaseName() + "_" + originIdentifier.getObjectName(); - return convert(rawName); + return convert(originIdentifier.getDatabaseName(), rawName); + } + + private Map lowerMapKey(Map map) { + int size = map == null ? 0 : map.size(); + Map lowerKeyMap = new HashMap<>(size); + if (size == 0) { + return lowerKeyMap; + } + + for (String key : map.keySet()) { + lowerKeyMap.put(key.toLowerCase(), map.get(key)); + } + + return lowerKeyMap; } } diff --git a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/format/aliyun/AliyunDataFormat.java b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/format/aliyun/AliyunDataFormat.java new file mode 100644 index 000000000000..ccbacdc2af5e --- /dev/null +++ b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/format/aliyun/AliyunDataFormat.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.flink.action.cdc.format.aliyun; + +import org.apache.paimon.flink.action.cdc.format.AbstractJsonDataFormat; +import org.apache.paimon.flink.action.cdc.format.RecordParserFactory; + +/** + * Supports the message queue's debezium json data format and provides definitions for the message + * queue's record json deserialization class and parsing class {@link AliyunRecordParser}. + */ +public class AliyunDataFormat extends AbstractJsonDataFormat { + + @Override + protected RecordParserFactory parser() { + return AliyunRecordParser::new; + } +} diff --git a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/format/aliyun/AliyunDataFormatFactory.java b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/format/aliyun/AliyunDataFormatFactory.java new file mode 100644 index 000000000000..a07e2f205c90 --- /dev/null +++ b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/format/aliyun/AliyunDataFormatFactory.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.flink.action.cdc.format.aliyun; + +import org.apache.paimon.flink.action.cdc.format.DataFormat; +import org.apache.paimon.flink.action.cdc.format.DataFormatFactory; + +/** Factory to create {@link AliyunDataFormat}. */ +public class AliyunDataFormatFactory implements DataFormatFactory { + + public static final String IDENTIFIER = "aliyun-json"; + + @Override + public String identifier() { + return IDENTIFIER; + } + + @Override + public DataFormat create() { + return new AliyunDataFormat(); + } +} diff --git a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/format/aliyun/AliyunFieldParser.java b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/format/aliyun/AliyunFieldParser.java new file mode 100644 index 000000000000..824ed9145943 --- /dev/null +++ b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/format/aliyun/AliyunFieldParser.java @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.flink.action.cdc.format.aliyun; + +/** Converts some special types such as enum、set、geometry. */ +public class AliyunFieldParser { + + protected static byte[] convertGeoType2WkbArray(byte[] mysqlGeomBytes) { + int sridLength = 4; + boolean hasSrid = false; + for (int i = 0; i < sridLength; ++i) { + if (mysqlGeomBytes[i] != 0) { + hasSrid = true; + break; + } + } + byte[] wkb; + if (hasSrid) { + wkb = new byte[mysqlGeomBytes.length]; + // byteOrder + geometry + System.arraycopy(mysqlGeomBytes, 4, wkb, 0, 5); + // srid + System.arraycopy(mysqlGeomBytes, 0, wkb, 5, 4); + // geometry + System.arraycopy(mysqlGeomBytes, 9, wkb, 9, wkb.length - 9); + + // set srid flag + if (wkb[0] == 0) { + // big endian + wkb[1] = (byte) (wkb[1] + 32); + } else { + wkb[4] = (byte) (wkb[4] + 32); + } + } else { + wkb = new byte[mysqlGeomBytes.length - 4]; + System.arraycopy(mysqlGeomBytes, 4, wkb, 0, wkb.length); + } + return wkb; + } + + protected static String convertSet(String value, String mysqlType) { + // mysql set type value can be filled with more than one, value is a bit string conversion + // from the long + int indexes = Integer.parseInt(value); + return getSetValuesByIndex(mysqlType, indexes); + } + + protected static String convertEnum(String value, String mysqlType) { + int elementIndex = Integer.parseInt(value); + // enum('a','b','c') + return getEnumValueByIndex(mysqlType, elementIndex); + } + + protected static String getEnumValueByIndex(String mysqlType, int elementIndex) { + String[] options = extractEnumValueByIndex(mysqlType); + + return options[elementIndex - 1]; + } + + protected static String getSetValuesByIndex(String mysqlType, int indexes) { + String[] options = extractSetValuesByIndex(mysqlType); + + StringBuilder sb = new StringBuilder(); + sb.append("["); + int index = 0; + boolean first = true; + int optionLen = options.length; + + while (indexes != 0L) { + if (indexes % 2L != 0) { + if (first) { + first = false; + } else { + sb.append(','); + } + if (index < optionLen) { + sb.append(options[index]); + } else { + throw new RuntimeException( + String.format( + "extractSetValues from mysqlType[%s],index:%d failed", + mysqlType, indexes)); + } + } + ++index; + indexes = indexes >>> 1; + } + sb.append("]"); + return sb.toString(); + } + + private static String[] extractSetValuesByIndex(String mysqlType) { + // set('x','y') + return mysqlType.substring(5, mysqlType.length() - 2).split("'\\s*,\\s*'"); + } + + private static String[] extractEnumValueByIndex(String mysqlType) { + // enum('x','y') + return mysqlType.substring(6, mysqlType.length() - 2).split("'\\s*,\\s*'"); + } +} diff --git a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/format/aliyun/AliyunRecordParser.java b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/format/aliyun/AliyunRecordParser.java new file mode 100644 index 000000000000..e31b282a76cb --- /dev/null +++ b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/format/aliyun/AliyunRecordParser.java @@ -0,0 +1,260 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.flink.action.cdc.format.aliyun; + +import org.apache.paimon.flink.action.cdc.ComputedColumn; +import org.apache.paimon.flink.action.cdc.TypeMapping; +import org.apache.paimon.flink.action.cdc.format.AbstractJsonRecordParser; +import org.apache.paimon.flink.action.cdc.mysql.MySqlTypeUtils; +import org.apache.paimon.flink.sink.cdc.RichCdcMultiplexRecord; +import org.apache.paimon.types.RowKind; +import org.apache.paimon.types.RowType; +import org.apache.paimon.utils.JsonSerdeUtil; + +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.core.type.TypeReference; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.databind.JsonNode; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.databind.node.ArrayNode; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.annotation.Nullable; + +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +import static org.apache.paimon.utils.JsonSerdeUtil.getNodeAs; +import static org.apache.paimon.utils.JsonSerdeUtil.isNull; + +/** + * The {@code CanalRecordParser} class is responsible for parsing records from the Canal-JSON + * format. Canal is a database binlog multi-platform consumer, which is used to synchronize data + * across databases. This parser extracts relevant information from the Canal-JSON format and + * transforms it into a list of {@link RichCdcMultiplexRecord} objects, which represent the changes + * captured in the database. + * + *

    The class handles different types of database operations such as INSERT, UPDATE, and DELETE, + * and generates corresponding {@link RichCdcMultiplexRecord} objects for each operation. + * + *

    Additionally, the parser supports schema extraction, which can be used to understand the + * structure of the incoming data and its corresponding field types. + */ +public class AliyunRecordParser extends AbstractJsonRecordParser { + + private static final Logger LOG = LoggerFactory.getLogger(AliyunRecordParser.class); + + private static final String FIELD_IS_DDL = "isDdl"; + private static final String FIELD_TYPE = "op"; + + private static final String OP_UPDATE_BEFORE = "UPDATE_BEFORE"; + private static final String OP_UPDATE_AFTER = "UPDATE_AFTER"; + private static final String OP_INSERT = "INSERT"; + private static final String OP_DELETE = "DELETE"; + + private static final String FIELD_PAYLOAD = "payload"; + private static final String FIELD_BEFORE = "before"; + private static final String FIELD_AFTER = "after"; + private static final String FIELD_COLUMN = "dataColumn"; + + private static final String FIELD_SCHEMA = "schema"; + private static final String FIELD_PK = "primaryKey"; + + @Override + protected boolean isDDL() { + JsonNode node = root.get(FIELD_IS_DDL); + return !isNull(node) && node.asBoolean(); + } + + public AliyunRecordParser(TypeMapping typeMapping, List computedColumns) { + super(typeMapping, computedColumns); + } + + @Override + protected String primaryField() { + return "schema.primaryKey"; + } + + @Override + protected String dataField() { + return "payload.dataColumn"; + } + + @Override + protected List extractPrimaryKeys() { + JsonNode schemaNode = root.get(FIELD_SCHEMA); + checkNotNull(schemaNode, FIELD_SCHEMA); + ArrayNode pkNode = getNodeAs(schemaNode, FIELD_PK, ArrayNode.class); + List pkFields = new ArrayList<>(); + pkNode.forEach( + pk -> { + if (isNull(pk)) { + throw new IllegalArgumentException( + String.format("Primary key cannot be null: %s", pk)); + } + + pkFields.add(pk.asText()); + }); + return pkFields; + } + + @Override + public List extractRecords() { + if (isDDL()) { + return Collections.emptyList(); + } + + List records = new ArrayList<>(); + + JsonNode payload = root.get(FIELD_PAYLOAD); + checkNotNull(payload, FIELD_PAYLOAD); + + String type = payload.get(FIELD_TYPE).asText(); + + RowKind rowKind = null; + String field = null; + switch (type) { + case OP_UPDATE_BEFORE: + rowKind = RowKind.UPDATE_BEFORE; + field = FIELD_BEFORE; + break; + case OP_UPDATE_AFTER: + rowKind = RowKind.UPDATE_AFTER; + field = FIELD_AFTER; + break; + case OP_INSERT: + rowKind = RowKind.INSERT; + field = FIELD_AFTER; + break; + case OP_DELETE: + rowKind = RowKind.DELETE; + field = FIELD_BEFORE; + break; + default: + throw new UnsupportedOperationException("Unknown record operation: " + type); + } + + JsonNode container = payload.get(field); + checkNotNull(container, String.format("%s.%s", FIELD_PAYLOAD, field)); + + JsonNode data = getNodeAs(container, FIELD_COLUMN, JsonNode.class); + checkNotNull(data, String.format("%s.%s.%s", FIELD_PAYLOAD, field, FIELD_COLUMN)); + + processRecord(data, rowKind, records); + + return records; + } + + @Override + protected Map extractRowData(JsonNode record, RowType.Builder rowTypeBuilder) { + + Map recordMap = + JsonSerdeUtil.convertValue(record, new TypeReference>() {}); + Map rowData = new HashMap<>(); + + fillDefaultTypes(record, rowTypeBuilder); + for (Map.Entry entry : recordMap.entrySet()) { + rowData.put(entry.getKey(), Objects.toString(entry.getValue(), null)); + } + + evalComputedColumns(rowData, rowTypeBuilder); + return rowData; + } + + @Override + protected String format() { + return "aliyun-json"; + } + + @Nullable + @Override + protected String getTableName() { + JsonNode schemaNode = root.get(FIELD_SCHEMA); + if (isNull(schemaNode)) { + return null; + } + JsonNode sourceNode = schemaNode.get("source"); + if (isNull(sourceNode)) { + return null; + } + + JsonNode tableNode = sourceNode.get("tableName"); + if (isNull(tableNode)) { + return null; + } + return tableNode.asText(); + } + + @Nullable + @Override + protected String getDatabaseName() { + JsonNode schemaNode = root.get(FIELD_SCHEMA); + if (isNull(schemaNode)) { + return null; + } + JsonNode sourceNode = schemaNode.get("source"); + if (isNull(sourceNode)) { + return null; + } + JsonNode databaseNode = sourceNode.get("dbName"); + if (isNull(databaseNode)) { + return null; + } + return databaseNode.asText(); + } + + private Map matchOldRecords(ArrayNode newData, ArrayNode oldData) { + return IntStream.range(0, newData.size()) + .boxed() + .collect(Collectors.toMap(newData::get, oldData::get)); + } + + private String transformValue(@Nullable String oldValue, String shortType, String mySqlType) { + if (oldValue == null) { + return null; + } + + if (MySqlTypeUtils.isSetType(shortType)) { + return AliyunFieldParser.convertSet(oldValue, mySqlType); + } + + if (MySqlTypeUtils.isEnumType(shortType)) { + return AliyunFieldParser.convertEnum(oldValue, mySqlType); + } + + if (MySqlTypeUtils.isGeoType(shortType)) { + try { + byte[] wkb = + AliyunFieldParser.convertGeoType2WkbArray( + oldValue.getBytes(StandardCharsets.ISO_8859_1)); + return MySqlTypeUtils.convertWkbArray(wkb); + } catch (Exception e) { + throw new IllegalArgumentException( + String.format("Failed to convert %s to geometry JSON.", oldValue), e); + } + } + return oldValue; + } +} diff --git a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/format/dms/DMSDataFormat.java b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/format/dms/DMSDataFormat.java new file mode 100644 index 000000000000..43228fca4554 --- /dev/null +++ b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/format/dms/DMSDataFormat.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.flink.action.cdc.format.dms; + +import org.apache.paimon.flink.action.cdc.format.AbstractJsonDataFormat; +import org.apache.paimon.flink.action.cdc.format.RecordParserFactory; + +/** + * Supports the message queue's AWS DMS json data format and provides definitions for the message + * queue's record json deserialization class and parsing class {@link DMSRecordParser}. + */ +public class DMSDataFormat extends AbstractJsonDataFormat { + + @Override + protected RecordParserFactory parser() { + return DMSRecordParser::new; + } +} diff --git a/paimon-format/src/main/java/org/apache/paimon/format/parquet/position/RowPosition.java b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/format/dms/DMSDataFormatFactory.java similarity index 61% rename from paimon-format/src/main/java/org/apache/paimon/format/parquet/position/RowPosition.java rename to paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/format/dms/DMSDataFormatFactory.java index fb6378349007..0be1270e8341 100644 --- a/paimon-format/src/main/java/org/apache/paimon/format/parquet/position/RowPosition.java +++ b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/format/dms/DMSDataFormatFactory.java @@ -16,25 +16,23 @@ * limitations under the License. */ -package org.apache.paimon.format.parquet.position; +package org.apache.paimon.flink.action.cdc.format.dms; -import javax.annotation.Nullable; +import org.apache.paimon.flink.action.cdc.format.DataFormat; +import org.apache.paimon.flink.action.cdc.format.DataFormatFactory; -/** To represent struct's position in repeated type. */ -public class RowPosition { - @Nullable private final boolean[] isNull; - private final int positionsCount; +/** Factory to create {@link DMSDataFormat}. */ +public class DMSDataFormatFactory implements DataFormatFactory { - public RowPosition(boolean[] isNull, int positionsCount) { - this.isNull = isNull; - this.positionsCount = positionsCount; - } + public static final String IDENTIFIER = "aws-dms-json"; - public boolean[] getIsNull() { - return isNull; + @Override + public DataFormat create() { + return new DMSDataFormat(); } - public int getPositionsCount() { - return positionsCount; + @Override + public String identifier() { + return IDENTIFIER; } } diff --git a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/format/dms/DMSRecordParser.java b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/format/dms/DMSRecordParser.java new file mode 100644 index 000000000000..8fc4808dd2d6 --- /dev/null +++ b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/format/dms/DMSRecordParser.java @@ -0,0 +1,159 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.flink.action.cdc.format.dms; + +import org.apache.paimon.flink.action.cdc.ComputedColumn; +import org.apache.paimon.flink.action.cdc.TypeMapping; +import org.apache.paimon.flink.action.cdc.format.AbstractJsonRecordParser; +import org.apache.paimon.flink.sink.cdc.RichCdcMultiplexRecord; +import org.apache.paimon.types.RowKind; +import org.apache.paimon.utils.Pair; + +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.databind.JsonNode; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.databind.node.ObjectNode; + +import javax.annotation.Nullable; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +/** + * The {@code DMSRecordParser} class extends the abstract {@link AbstractJsonRecordParser} and is + * designed to parse records from AWS DMS's JSON change data capture (CDC) format. AWS DMS is a CDC + * solution for RDMS that captures row-level changes to database tables and outputs them in JSON + * format. This parser extracts relevant information from the DMS-JSON format and converts it into a + * list of {@link RichCdcMultiplexRecord} objects. + * + *

    The class supports various database operations such as INSERT, UPDATE, and DELETE, and creates + * corresponding {@link RichCdcMultiplexRecord} objects to represent these changes. + * + *

    Validation is performed to ensure that the JSON records contain all necessary fields, and the + * class also supports schema extraction for the Kafka topic. + */ +public class DMSRecordParser extends AbstractJsonRecordParser { + + private static final String FIELD_DATA = "data"; + private static final String FIELD_METADATA = "metadata"; + private static final String FIELD_TYPE = "record-type"; + private static final String FIELD_OP = "operation"; + private static final String FIELD_DATABASE = "schema-name"; + private static final String FIELD_TABLE = "table-name"; + + private static final String OP_LOAD = "load"; + private static final String OP_INSERT = "insert"; + private static final String OP_UPDATE = "update"; + private static final String OP_DELETE = "delete"; + + private static final String BEFORE_PREFIX = "BI_"; + + public DMSRecordParser(TypeMapping typeMapping, List computedColumns) { + super(typeMapping, computedColumns); + } + + @Override + protected @Nullable String getTableName() { + JsonNode metaNode = getAndCheck(FIELD_METADATA); + return metaNode.get(FIELD_TABLE).asText(); + } + + @Override + protected List extractRecords() { + if (isDDL()) { + return Collections.emptyList(); + } + + JsonNode dataNode = getAndCheck(dataField()); + String operation = getAndCheck(FIELD_METADATA).get(FIELD_OP).asText(); + List records = new ArrayList<>(); + + switch (operation) { + case OP_LOAD: + case OP_INSERT: + processRecord(dataNode, RowKind.INSERT, records); + break; + case OP_UPDATE: + Pair dataAndBeforeNodes = splitBeforeAndData(dataNode); + processRecord(dataAndBeforeNodes.getRight(), RowKind.DELETE, records); + processRecord(dataAndBeforeNodes.getLeft(), RowKind.INSERT, records); + break; + case OP_DELETE: + processRecord(dataNode, RowKind.DELETE, records); + break; + default: + throw new UnsupportedOperationException("Unknown record operation: " + operation); + } + + return records; + } + + @Override + protected @Nullable String getDatabaseName() { + JsonNode metaNode = getAndCheck(FIELD_METADATA); + return metaNode.get(FIELD_DATABASE).asText(); + } + + @Override + protected String primaryField() { + return null; + } + + @Override + protected String dataField() { + return FIELD_DATA; + } + + @Override + protected String format() { + return "aws-dms-json"; + } + + @Override + protected boolean isDDL() { + String recordType = getAndCheck(FIELD_METADATA).get(FIELD_TYPE).asText(); + return !"data".equals(recordType); + } + + private Pair splitBeforeAndData(JsonNode dataNode) { + JsonNode newDataNode = dataNode.deepCopy(); + JsonNode beforeDataNode = dataNode.deepCopy(); + + Iterator> newDataFields = newDataNode.fields(); + while (newDataFields.hasNext()) { + Map.Entry next = newDataFields.next(); + if (next.getKey().startsWith(BEFORE_PREFIX)) { + newDataFields.remove(); + } + } + + Iterator> beforeDataFields = beforeDataNode.fields(); + while (beforeDataFields.hasNext()) { + Map.Entry next = beforeDataFields.next(); + if (next.getKey().startsWith(BEFORE_PREFIX)) { + String key = next.getKey().replaceFirst(BEFORE_PREFIX, ""); + ((ObjectNode) beforeDataNode).set(key, next.getValue()); + beforeDataFields.remove(); + } + } + + return Pair.of(newDataNode, beforeDataNode); + } +} diff --git a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/kafka/KafkaDebeziumAvroDeserializationSchema.java b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/kafka/KafkaDebeziumAvroDeserializationSchema.java index fc672b9dc0ab..eea364d460de 100644 --- a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/kafka/KafkaDebeziumAvroDeserializationSchema.java +++ b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/kafka/KafkaDebeziumAvroDeserializationSchema.java @@ -48,7 +48,7 @@ public class KafkaDebeziumAvroDeserializationSchema public KafkaDebeziumAvroDeserializationSchema(Configuration cdcSourceConfig) { this.topic = KafkaActionUtils.findOneTopic(cdcSourceConfig); - this.schemaRegistryUrl = cdcSourceConfig.getString(SCHEMA_REGISTRY_URL); + this.schemaRegistryUrl = cdcSourceConfig.get(SCHEMA_REGISTRY_URL); } @Override diff --git a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/mongodb/MongoDBSyncDatabaseAction.java b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/mongodb/MongoDBSyncDatabaseAction.java index 9f3ed1085600..3166a3c82ae5 100644 --- a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/mongodb/MongoDBSyncDatabaseAction.java +++ b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/mongodb/MongoDBSyncDatabaseAction.java @@ -66,6 +66,7 @@ protected CdcTimestampExtractor createCdcTimestampExtractor() { @Override protected MongoDBSource buildSource() { + validateRuntimeExecutionMode(); return MongoDBActionUtils.buildMongodbSource( cdcSourceConfig, CdcActionCommonUtils.combinedModeTableList( diff --git a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/mongodb/MongoDBSyncTableAction.java b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/mongodb/MongoDBSyncTableAction.java index 16dbbadfd776..34128a62fcde 100644 --- a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/mongodb/MongoDBSyncTableAction.java +++ b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/mongodb/MongoDBSyncTableAction.java @@ -77,6 +77,7 @@ protected CdcTimestampExtractor createCdcTimestampExtractor() { @Override protected MongoDBSource buildSource() { + validateRuntimeExecutionMode(); String tableList = cdcSourceConfig.get(MongoDBSourceOptions.DATABASE) + "\\." diff --git a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/mongodb/strategy/MongoVersionStrategy.java b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/mongodb/strategy/MongoVersionStrategy.java index 64f127571134..df288a4150e6 100644 --- a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/mongodb/strategy/MongoVersionStrategy.java +++ b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/mongodb/strategy/MongoVersionStrategy.java @@ -83,7 +83,7 @@ default Map getExtractRow( Configuration mongodbConfig) throws JsonProcessingException { SchemaAcquisitionMode mode = - SchemaAcquisitionMode.valueOf(mongodbConfig.getString(START_MODE).toUpperCase()); + SchemaAcquisitionMode.valueOf(mongodbConfig.get(START_MODE).toUpperCase()); ObjectNode objectNode = JsonSerdeUtil.asSpecificNodeType(jsonNode.asText(), ObjectNode.class); JsonNode idNode = objectNode.get(ID_FIELD); @@ -92,7 +92,7 @@ default Map getExtractRow( "The provided MongoDB JSON document does not contain an _id field."); } JsonNode document = - mongodbConfig.getBoolean(DEFAULT_ID_GENERATION) + mongodbConfig.get(DEFAULT_ID_GENERATION) ? objectNode.set( ID_FIELD, idNode.get(OID_FIELD) == null ? idNode : idNode.get(OID_FIELD)) @@ -101,8 +101,8 @@ default Map getExtractRow( case SPECIFIED: return parseFieldsFromJsonRecord( document.toString(), - mongodbConfig.getString(PARSER_PATH), - mongodbConfig.getString(FIELD_NAME), + mongodbConfig.get(PARSER_PATH), + mongodbConfig.get(FIELD_NAME), computedColumns, rowTypeBuilder); case DYNAMIC: diff --git a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/mysql/MySqlRecordParser.java b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/mysql/MySqlRecordParser.java index 502e6237a477..26579e718f56 100644 --- a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/mysql/MySqlRecordParser.java +++ b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/mysql/MySqlRecordParser.java @@ -45,6 +45,8 @@ import org.apache.flink.api.common.functions.FlatMapFunction; import org.apache.flink.cdc.connectors.mysql.source.config.MySqlSourceOptions; import org.apache.flink.cdc.debezium.table.DebeziumOptions; +import org.apache.flink.configuration.ConfigOption; +import org.apache.flink.configuration.ConfigOptions; import org.apache.flink.configuration.Configuration; import org.apache.flink.util.Collector; import org.slf4j.Logger; @@ -99,11 +101,14 @@ public MySqlRecordParser( .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); String stringifyServerTimeZone = mySqlConfig.get(MySqlSourceOptions.SERVER_TIME_ZONE); - this.isDebeziumSchemaCommentsEnabled = - mySqlConfig.getBoolean( - DebeziumOptions.DEBEZIUM_OPTIONS_PREFIX - + RelationalDatabaseConnectorConfig.INCLUDE_SCHEMA_COMMENTS.name(), - false); + ConfigOption includeSchemaCommentsConfig = + ConfigOptions.key( + DebeziumOptions.DEBEZIUM_OPTIONS_PREFIX + + RelationalDatabaseConnectorConfig.INCLUDE_SCHEMA_COMMENTS + .name()) + .booleanType() + .defaultValue(false); + this.isDebeziumSchemaCommentsEnabled = mySqlConfig.get(includeSchemaCommentsConfig); this.serverTimeZone = stringifyServerTimeZone == null ? ZoneId.systemDefault() diff --git a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/mysql/MySqlSyncDatabaseAction.java b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/mysql/MySqlSyncDatabaseAction.java index 01be020f7405..ce2e9124a664 100644 --- a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/mysql/MySqlSyncDatabaseAction.java +++ b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/mysql/MySqlSyncDatabaseAction.java @@ -138,11 +138,13 @@ protected void beforeBuildingSourceSink() throws Exception { + ", or MySQL database does not exist."); TableNameConverter tableNameConverter = - new TableNameConverter(allowUpperCase, mergeShards, tablePrefix, tableSuffix); + new TableNameConverter( + allowUpperCase, mergeShards, tablePrefix, tableSuffix, tableMapping); for (JdbcTableInfo tableInfo : jdbcTableInfos) { Identifier identifier = Identifier.create( - database, tableNameConverter.convert(tableInfo.toPaimonTableName())); + database, + tableNameConverter.convert("", tableInfo.toPaimonTableName())); FileStoreTable table; Schema fromMySql = CdcActionCommonUtils.buildPaimonSchema( @@ -188,6 +190,7 @@ protected CdcTimestampExtractor createCdcTimestampExtractor() { @Override protected MySqlSource buildSource() { + validateRuntimeExecutionMode(); return MySqlActionUtils.buildMySqlSource( cdcSourceConfig, tableList( diff --git a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/mysql/MySqlSyncTableAction.java b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/mysql/MySqlSyncTableAction.java index a05832b1d033..d73d9702f1e1 100644 --- a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/mysql/MySqlSyncTableAction.java +++ b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/mysql/MySqlSyncTableAction.java @@ -102,6 +102,7 @@ protected Schema retrieveSchema() throws Exception { @Override protected MySqlSource buildSource() { + validateRuntimeExecutionMode(); String tableList = String.format( "(%s)\\.(%s)", diff --git a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/postgres/PostgresSyncTableAction.java b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/postgres/PostgresSyncTableAction.java index f66c20dfa02a..7dc1b019cd02 100644 --- a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/postgres/PostgresSyncTableAction.java +++ b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/postgres/PostgresSyncTableAction.java @@ -103,6 +103,7 @@ protected Schema retrieveSchema() throws Exception { @Override protected JdbcIncrementalSource buildSource() { + validateRuntimeExecutionMode(); List pkTables = postgresSchemasInfo.pkTables(); Set schemaList = new HashSet<>(); String[] tableList = new String[pkTables.size()]; diff --git a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/pulsar/PulsarDebeziumAvroDeserializationSchema.java b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/pulsar/PulsarDebeziumAvroDeserializationSchema.java index b0d1d1bf620f..f45ee034bec8 100644 --- a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/pulsar/PulsarDebeziumAvroDeserializationSchema.java +++ b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/pulsar/PulsarDebeziumAvroDeserializationSchema.java @@ -46,7 +46,7 @@ public class PulsarDebeziumAvroDeserializationSchema public PulsarDebeziumAvroDeserializationSchema(Configuration cdcSourceConfig) { this.topic = PulsarActionUtils.findOneTopic(cdcSourceConfig); - this.schemaRegistryUrl = cdcSourceConfig.getString(SCHEMA_REGISTRY_URL); + this.schemaRegistryUrl = cdcSourceConfig.get(SCHEMA_REGISTRY_URL); } @Override diff --git a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/watermark/MessageQueueCdcTimestampExtractor.java b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/watermark/MessageQueueCdcTimestampExtractor.java index 8a9a28453bad..5bf2fefc1b78 100644 --- a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/watermark/MessageQueueCdcTimestampExtractor.java +++ b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/watermark/MessageQueueCdcTimestampExtractor.java @@ -54,6 +54,10 @@ public long extractTimestamp(CdcSourceRecord cdcSourceRecord) throws JsonProcess } else if (JsonSerdeUtil.isNodeExists(record, "source", "connector")) { // Dbz json return JsonSerdeUtil.extractValue(record, Long.class, "ts_ms"); + } else if (JsonSerdeUtil.isNodeExists(record, "payload", "timestamp")) { + // Aliyun json + return JsonSerdeUtil.extractValue( + record, Long.class, "payload", "timestamp", "systemTime"); } throw new RuntimeException( String.format( diff --git a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/kafka/KafkaSinkFunction.java b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/kafka/KafkaSinkFunction.java index 72a177adceaf..41e7141cf48a 100644 --- a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/kafka/KafkaSinkFunction.java +++ b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/kafka/KafkaSinkFunction.java @@ -21,6 +21,7 @@ import org.apache.paimon.flink.sink.LogSinkFunction; import org.apache.paimon.table.sink.SinkRecord; +import org.apache.flink.api.common.functions.OpenContext; import org.apache.flink.configuration.Configuration; import org.apache.flink.streaming.connectors.kafka.FlinkKafkaException; import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer; @@ -65,7 +66,16 @@ public void setWriteCallback(WriteCallback writeCallback) { this.writeCallback = writeCallback; } - @Override + /** + * Do not annotate with @override here to maintain compatibility with Flink 1.18-. + */ + public void open(OpenContext openContext) throws Exception { + open(new Configuration()); + } + + /** + * Do not annotate with @override here to maintain compatibility with Flink 2.0+. + */ public void open(Configuration configuration) throws Exception { super.open(configuration); Callback baseCallback = requireNonNull(callback); diff --git a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/CdcDynamicBucketSink.java b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/CdcDynamicBucketSink.java index 574ff685f3fa..6d9e3a4a7c82 100644 --- a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/CdcDynamicBucketSink.java +++ b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/CdcDynamicBucketSink.java @@ -25,7 +25,7 @@ import org.apache.paimon.table.sink.KeyAndBucketExtractor; import org.apache.flink.api.java.tuple.Tuple2; -import org.apache.flink.streaming.api.operators.OneInputStreamOperator; +import org.apache.flink.streaming.api.operators.OneInputStreamOperatorFactory; /** {@link CdcDynamicBucketSinkBase} for {@link CdcRecord}. */ public class CdcDynamicBucketSink extends CdcDynamicBucketSinkBase { @@ -42,8 +42,8 @@ protected KeyAndBucketExtractor createExtractor(TableSchema schema) { } @Override - protected OneInputStreamOperator, Committable> createWriteOperator( - StoreSinkWrite.Provider writeProvider, String commitUser) { - return new CdcDynamicBucketWriteOperator(table, writeProvider, commitUser); + protected OneInputStreamOperatorFactory, Committable> + createWriteOperatorFactory(StoreSinkWrite.Provider writeProvider, String commitUser) { + return new CdcDynamicBucketWriteOperator.Factory(table, writeProvider, commitUser); } } diff --git a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/CdcDynamicBucketWriteOperator.java b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/CdcDynamicBucketWriteOperator.java index 4276b7efcc0d..5637e8b12794 100644 --- a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/CdcDynamicBucketWriteOperator.java +++ b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/CdcDynamicBucketWriteOperator.java @@ -19,6 +19,7 @@ package org.apache.paimon.flink.sink.cdc; import org.apache.paimon.data.GenericRow; +import org.apache.paimon.flink.sink.Committable; import org.apache.paimon.flink.sink.PrepareCommitOperator; import org.apache.paimon.flink.sink.StoreSinkWrite; import org.apache.paimon.flink.sink.TableWriteOperator; @@ -26,6 +27,9 @@ import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.runtime.state.StateInitializationContext; +import org.apache.flink.streaming.api.operators.StreamOperator; +import org.apache.flink.streaming.api.operators.StreamOperatorFactory; +import org.apache.flink.streaming.api.operators.StreamOperatorParameters; import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; import java.io.IOException; @@ -49,11 +53,12 @@ public class CdcDynamicBucketWriteOperator extends TableWriteOperator parameters, FileStoreTable table, StoreSinkWrite.Provider storeSinkWriteProvider, String initialCommitUser) { - super(table, storeSinkWriteProvider, initialCommitUser); + super(parameters, table, storeSinkWriteProvider, initialCommitUser); this.retrySleepMillis = table.coreOptions().toConfiguration().get(RETRY_SLEEP_TIME).toMillis(); this.maxRetryNumTimes = table.coreOptions().toConfiguration().get(MAX_RETRY_NUM_TIMES); @@ -101,4 +106,30 @@ public void processElement(StreamRecord> element) thr } } } + + /** {@link StreamOperatorFactory} of {@link CdcDynamicBucketWriteOperator}. */ + public static class Factory extends TableWriteOperator.Factory> { + + public Factory( + FileStoreTable table, + StoreSinkWrite.Provider storeSinkWriteProvider, + String initialCommitUser) { + super(table, storeSinkWriteProvider, initialCommitUser); + } + + @Override + @SuppressWarnings("unchecked") + public > T createStreamOperator( + StreamOperatorParameters parameters) { + return (T) + new CdcDynamicBucketWriteOperator( + parameters, table, storeSinkWriteProvider, initialCommitUser); + } + + @Override + @SuppressWarnings("rawtypes") + public Class getStreamOperatorClass(ClassLoader classLoader) { + return CdcDynamicBucketWriteOperator.class; + } + } } diff --git a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/CdcDynamicTableParsingProcessFunction.java b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/CdcDynamicTableParsingProcessFunction.java index 0961ff160048..886e33e2046a 100644 --- a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/CdcDynamicTableParsingProcessFunction.java +++ b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/CdcDynamicTableParsingProcessFunction.java @@ -22,6 +22,7 @@ import org.apache.paimon.catalog.Identifier; import org.apache.paimon.types.DataField; +import org.apache.flink.api.common.functions.OpenContext; import org.apache.flink.api.common.typeinfo.TypeHint; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; @@ -74,7 +75,16 @@ public CdcDynamicTableParsingProcessFunction( this.parserFactory = parserFactory; } - @Override + /** + * Do not annotate with @override here to maintain compatibility with Flink 1.18-. + */ + public void open(OpenContext openContext) throws Exception { + open(new Configuration()); + } + + /** + * Do not annotate with @override here to maintain compatibility with Flink 2.0+. + */ public void open(Configuration parameters) throws Exception { parser = parserFactory.create(); catalog = catalogLoader.load(); diff --git a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/CdcFixedBucketSink.java b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/CdcFixedBucketSink.java index 59bdb192beea..bec9508888b4 100644 --- a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/CdcFixedBucketSink.java +++ b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/CdcFixedBucketSink.java @@ -24,7 +24,7 @@ import org.apache.paimon.flink.sink.StoreSinkWrite; import org.apache.paimon.table.FileStoreTable; -import org.apache.flink.streaming.api.operators.OneInputStreamOperator; +import org.apache.flink.streaming.api.operators.OneInputStreamOperatorFactory; /** * A {@link FlinkSink} for fixed-bucket table which accepts {@link CdcRecord} and waits for a schema @@ -39,8 +39,8 @@ public CdcFixedBucketSink(FileStoreTable table) { } @Override - protected OneInputStreamOperator createWriteOperator( + protected OneInputStreamOperatorFactory createWriteOperatorFactory( StoreSinkWrite.Provider writeProvider, String commitUser) { - return new CdcRecordStoreWriteOperator(table, writeProvider, commitUser); + return new CdcRecordStoreWriteOperator.Factory(table, writeProvider, commitUser); } } diff --git a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/CdcMultiTableParsingProcessFunction.java b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/CdcMultiTableParsingProcessFunction.java index b18a05c280cb..4c5e0600bb47 100644 --- a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/CdcMultiTableParsingProcessFunction.java +++ b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/CdcMultiTableParsingProcessFunction.java @@ -20,6 +20,7 @@ import org.apache.paimon.types.DataField; +import org.apache.flink.api.common.functions.OpenContext; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.typeutils.ListTypeInfo; import org.apache.flink.configuration.Configuration; @@ -51,7 +52,16 @@ public CdcMultiTableParsingProcessFunction(EventParser.Factory parserFactory) this.parserFactory = parserFactory; } - @Override + /** + * Do not annotate with @override here to maintain compatibility with Flink 1.18-. + */ + public void open(OpenContext openContext) throws Exception { + open(new Configuration()); + } + + /** + * Do not annotate with @override here to maintain compatibility with Flink 2.0+. + */ public void open(Configuration parameters) throws Exception { parser = parserFactory.create(); updatedDataFieldsOutputTags = new HashMap<>(); diff --git a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/CdcParsingProcessFunction.java b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/CdcParsingProcessFunction.java index 3456634942c8..eec228f3c09b 100644 --- a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/CdcParsingProcessFunction.java +++ b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/CdcParsingProcessFunction.java @@ -20,6 +20,7 @@ import org.apache.paimon.types.DataField; +import org.apache.flink.api.common.functions.OpenContext; import org.apache.flink.api.java.typeutils.ListTypeInfo; import org.apache.flink.configuration.Configuration; import org.apache.flink.streaming.api.functions.ProcessFunction; @@ -50,7 +51,16 @@ public CdcParsingProcessFunction(EventParser.Factory parserFactory) { this.parserFactory = parserFactory; } - @Override + /** + * Do not annotate with @override here to maintain compatibility with Flink 1.18-. + */ + public void open(OpenContext openContext) throws Exception { + open(new Configuration()); + } + + /** + * Do not annotate with @override here to maintain compatibility with Flink 2.0+. + */ public void open(Configuration parameters) throws Exception { parser = parserFactory.create(); } diff --git a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/CdcRecord.java b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/CdcRecord.java index 9adca753dc55..b23d0d6f06de 100644 --- a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/CdcRecord.java +++ b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/CdcRecord.java @@ -35,11 +35,12 @@ public class CdcRecord implements Serializable { private RowKind kind; - private final Map fields; + // field name -> value + private final Map data; - public CdcRecord(RowKind kind, Map fields) { + public CdcRecord(RowKind kind, Map data) { this.kind = kind; - this.fields = fields; + this.data = data; } public static CdcRecord emptyRecord() { @@ -50,16 +51,16 @@ public RowKind kind() { return kind; } - public Map fields() { - return fields; + public Map data() { + return data; } public CdcRecord fieldNameLowerCase() { - Map newFields = new HashMap<>(); - for (Map.Entry entry : fields.entrySet()) { - newFields.put(entry.getKey().toLowerCase(), entry.getValue()); + Map newData = new HashMap<>(); + for (Map.Entry entry : data.entrySet()) { + newData.put(entry.getKey().toLowerCase(), entry.getValue()); } - return new CdcRecord(kind, newFields); + return new CdcRecord(kind, newData); } @Override @@ -69,16 +70,16 @@ public boolean equals(Object o) { } CdcRecord that = (CdcRecord) o; - return Objects.equals(kind, that.kind) && Objects.equals(fields, that.fields); + return Objects.equals(kind, that.kind) && Objects.equals(data, that.data); } @Override public int hashCode() { - return Objects.hash(kind, fields); + return Objects.hash(kind, data); } @Override public String toString() { - return kind.shortString() + " " + fields; + return kind.shortString() + " " + data; } } diff --git a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/CdcRecordStoreMultiWriteOperator.java b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/CdcRecordStoreMultiWriteOperator.java index abd86cb8099b..9f1b68949f6d 100644 --- a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/CdcRecordStoreMultiWriteOperator.java +++ b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/CdcRecordStoreMultiWriteOperator.java @@ -38,6 +38,9 @@ import org.apache.flink.runtime.state.StateInitializationContext; import org.apache.flink.runtime.state.StateSnapshotContext; +import org.apache.flink.streaming.api.operators.StreamOperator; +import org.apache.flink.streaming.api.operators.StreamOperatorFactory; +import org.apache.flink.streaming.api.operators.StreamOperatorParameters; import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; import java.io.IOException; @@ -76,12 +79,13 @@ public class CdcRecordStoreMultiWriteOperator private String commitUser; private ExecutorService compactExecutor; - public CdcRecordStoreMultiWriteOperator( + private CdcRecordStoreMultiWriteOperator( + StreamOperatorParameters parameters, Catalog.Loader catalogLoader, StoreSinkWrite.WithWriteBufferProvider storeSinkWriteProvider, String initialCommitUser, Options options) { - super(options); + super(parameters, options); this.catalogLoader = catalogLoader; this.storeSinkWriteProvider = storeSinkWriteProvider; this.initialCommitUser = initialCommitUser; @@ -267,4 +271,42 @@ public Map writes() { public String commitUser() { return commitUser; } + + /** {@link StreamOperatorFactory} of {@link CdcRecordStoreMultiWriteOperator}. */ + public static class Factory + extends PrepareCommitOperator.Factory { + private final StoreSinkWrite.WithWriteBufferProvider storeSinkWriteProvider; + private final String initialCommitUser; + private final Catalog.Loader catalogLoader; + + public Factory( + Catalog.Loader catalogLoader, + StoreSinkWrite.WithWriteBufferProvider storeSinkWriteProvider, + String initialCommitUser, + Options options) { + super(options); + this.catalogLoader = catalogLoader; + this.storeSinkWriteProvider = storeSinkWriteProvider; + this.initialCommitUser = initialCommitUser; + } + + @Override + @SuppressWarnings("unchecked") + public > T createStreamOperator( + StreamOperatorParameters parameters) { + return (T) + new CdcRecordStoreMultiWriteOperator( + parameters, + catalogLoader, + storeSinkWriteProvider, + initialCommitUser, + options); + } + + @Override + @SuppressWarnings("rawtypes") + public Class getStreamOperatorClass(ClassLoader classLoader) { + return CdcRecordStoreMultiWriteOperator.class; + } + } } diff --git a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/CdcRecordStoreWriteOperator.java b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/CdcRecordStoreWriteOperator.java index fb2ec3a17110..8a8233842df7 100644 --- a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/CdcRecordStoreWriteOperator.java +++ b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/CdcRecordStoreWriteOperator.java @@ -19,6 +19,7 @@ package org.apache.paimon.flink.sink.cdc; import org.apache.paimon.data.GenericRow; +import org.apache.paimon.flink.sink.Committable; import org.apache.paimon.flink.sink.PrepareCommitOperator; import org.apache.paimon.flink.sink.StoreSinkWrite; import org.apache.paimon.flink.sink.TableWriteOperator; @@ -27,6 +28,9 @@ import org.apache.paimon.table.FileStoreTable; import org.apache.flink.runtime.state.StateInitializationContext; +import org.apache.flink.streaming.api.operators.StreamOperator; +import org.apache.flink.streaming.api.operators.StreamOperatorFactory; +import org.apache.flink.streaming.api.operators.StreamOperatorParameters; import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; import java.io.IOException; @@ -66,11 +70,12 @@ public class CdcRecordStoreWriteOperator extends TableWriteOperator { private final boolean skipCorruptRecord; - public CdcRecordStoreWriteOperator( + protected CdcRecordStoreWriteOperator( + StreamOperatorParameters parameters, FileStoreTable table, StoreSinkWrite.Provider storeSinkWriteProvider, String initialCommitUser) { - super(table, storeSinkWriteProvider, initialCommitUser); + super(parameters, table, storeSinkWriteProvider, initialCommitUser); this.retrySleepMillis = table.coreOptions().toConfiguration().get(RETRY_SLEEP_TIME).toMillis(); this.maxRetryNumTimes = table.coreOptions().toConfiguration().get(MAX_RETRY_NUM_TIMES); @@ -118,4 +123,30 @@ public void processElement(StreamRecord element) throws Exception { } } } + + /** {@link StreamOperatorFactory} of {@link CdcRecordStoreWriteOperator}. */ + public static class Factory extends TableWriteOperator.Factory { + + public Factory( + FileStoreTable table, + StoreSinkWrite.Provider storeSinkWriteProvider, + String initialCommitUser) { + super(table, storeSinkWriteProvider, initialCommitUser); + } + + @Override + @SuppressWarnings("unchecked") + public > T createStreamOperator( + StreamOperatorParameters parameters) { + return (T) + new CdcRecordStoreWriteOperator( + parameters, table, storeSinkWriteProvider, initialCommitUser); + } + + @Override + @SuppressWarnings("rawtypes") + public Class getStreamOperatorClass(ClassLoader classLoader) { + return CdcRecordStoreWriteOperator.class; + } + } } diff --git a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/CdcRecordUtils.java b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/CdcRecordUtils.java index 0d192dd538d6..91979a2c99b8 100644 --- a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/CdcRecordUtils.java +++ b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/CdcRecordUtils.java @@ -54,7 +54,7 @@ public static GenericRow projectAsInsert(CdcRecord record, List dataF GenericRow genericRow = new GenericRow(dataFields.size()); for (int i = 0; i < dataFields.size(); i++) { DataField dataField = dataFields.get(i); - String fieldValue = record.fields().get(dataField.name()); + String fieldValue = record.data().get(dataField.name()); if (fieldValue != null) { genericRow.setField( i, TypeUtils.castFromCdcValueString(fieldValue, dataField.type())); @@ -83,7 +83,7 @@ public static Optional toGenericRow(CdcRecord record, List fieldNames = dataFields.stream().map(DataField::name).collect(Collectors.toList()); - for (Map.Entry field : record.fields().entrySet()) { + for (Map.Entry field : record.data().entrySet()) { String key = field.getKey(); String value = field.getValue(); @@ -117,14 +117,14 @@ public static Optional toGenericRow(CdcRecord record, List fieldNames) { - Map fields = new HashMap<>(); + Map data = new HashMap<>(); for (int i = 0; i < row.getFieldCount(); i++) { Object field = row.getField(i); if (field != null) { - fields.put(fieldNames.get(i), field.toString()); + data.put(fieldNames.get(i), field.toString()); } } - return new CdcRecord(row.getRowKind(), fields); + return new CdcRecord(row.getRowKind(), data); } } diff --git a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/CdcUnawareBucketSink.java b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/CdcUnawareBucketSink.java index 313f4d013ef8..820ef7728f8c 100644 --- a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/CdcUnawareBucketSink.java +++ b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/CdcUnawareBucketSink.java @@ -24,7 +24,7 @@ import org.apache.paimon.table.FileStoreTable; import org.apache.flink.streaming.api.datastream.DataStream; -import org.apache.flink.streaming.api.operators.OneInputStreamOperator; +import org.apache.flink.streaming.api.operators.OneInputStreamOperatorFactory; import javax.annotation.Nullable; @@ -42,9 +42,9 @@ public CdcUnawareBucketSink(FileStoreTable table, Integer parallelism) { } @Override - protected OneInputStreamOperator createWriteOperator( + protected OneInputStreamOperatorFactory createWriteOperatorFactory( StoreSinkWrite.Provider writeProvider, String commitUser) { - return new CdcUnawareBucketWriteOperator(table, writeProvider, commitUser); + return new CdcUnawareBucketWriteOperator.Factory(table, writeProvider, commitUser); } @Override diff --git a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/CdcUnawareBucketWriteOperator.java b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/CdcUnawareBucketWriteOperator.java index c57a40f3f71d..26f65fdd09ce 100644 --- a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/CdcUnawareBucketWriteOperator.java +++ b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/CdcUnawareBucketWriteOperator.java @@ -18,21 +18,26 @@ package org.apache.paimon.flink.sink.cdc; +import org.apache.paimon.flink.sink.Committable; import org.apache.paimon.flink.sink.PrepareCommitOperator; import org.apache.paimon.flink.sink.StoreSinkWrite; import org.apache.paimon.table.FileStoreTable; import org.apache.paimon.types.RowKind; +import org.apache.flink.streaming.api.operators.StreamOperator; +import org.apache.flink.streaming.api.operators.StreamOperatorFactory; +import org.apache.flink.streaming.api.operators.StreamOperatorParameters; import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; /** A {@link PrepareCommitOperator} to write {@link CdcRecord} to unaware-bucket mode table. */ public class CdcUnawareBucketWriteOperator extends CdcRecordStoreWriteOperator { - public CdcUnawareBucketWriteOperator( + private CdcUnawareBucketWriteOperator( + StreamOperatorParameters parameters, FileStoreTable table, StoreSinkWrite.Provider storeSinkWriteProvider, String initialCommitUser) { - super(table, storeSinkWriteProvider, initialCommitUser); + super(parameters, table, storeSinkWriteProvider, initialCommitUser); } @Override @@ -42,4 +47,30 @@ public void processElement(StreamRecord element) throws Exception { super.processElement(element); } } + + /** {@link StreamOperatorFactory} of {@link CdcUnawareBucketWriteOperator}. */ + public static class Factory extends CdcRecordStoreWriteOperator.Factory { + + public Factory( + FileStoreTable table, + StoreSinkWrite.Provider storeSinkWriteProvider, + String initialCommitUser) { + super(table, storeSinkWriteProvider, initialCommitUser); + } + + @Override + @SuppressWarnings("unchecked") + public > T createStreamOperator( + StreamOperatorParameters parameters) { + return (T) + new CdcUnawareBucketWriteOperator( + parameters, table, storeSinkWriteProvider, initialCommitUser); + } + + @Override + @SuppressWarnings("rawtypes") + public Class getStreamOperatorClass(ClassLoader classLoader) { + return CdcUnawareBucketWriteOperator.class; + } + } } diff --git a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/FlinkCdcMultiTableSink.java b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/FlinkCdcMultiTableSink.java index 55e987c6055f..1688d4deb088 100644 --- a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/FlinkCdcMultiTableSink.java +++ b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/FlinkCdcMultiTableSink.java @@ -21,7 +21,7 @@ import org.apache.paimon.catalog.Catalog; import org.apache.paimon.flink.sink.CommittableStateManager; import org.apache.paimon.flink.sink.Committer; -import org.apache.paimon.flink.sink.CommitterOperator; +import org.apache.paimon.flink.sink.CommitterOperatorFactory; import org.apache.paimon.flink.sink.FlinkSink; import org.apache.paimon.flink.sink.FlinkStreamPartitioner; import org.apache.paimon.flink.sink.MultiTableCommittable; @@ -40,8 +40,8 @@ import org.apache.flink.streaming.api.datastream.DataStreamSink; import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; -import org.apache.flink.streaming.api.functions.sink.DiscardingSink; -import org.apache.flink.streaming.api.operators.OneInputStreamOperator; +import org.apache.flink.streaming.api.functions.sink.v2.DiscardingSink; +import org.apache.flink.streaming.api.operators.OneInputStreamOperatorFactory; import javax.annotation.Nullable; @@ -63,19 +63,16 @@ public class FlinkCdcMultiTableSink implements Serializable { private final Catalog.Loader catalogLoader; private final double commitCpuCores; @Nullable private final MemorySize commitHeapMemory; - private final boolean commitChaining; private final String commitUser; public FlinkCdcMultiTableSink( Catalog.Loader catalogLoader, double commitCpuCores, @Nullable MemorySize commitHeapMemory, - boolean commitChaining, String commitUser) { this.catalogLoader = catalogLoader; this.commitCpuCores = commitCpuCores; this.commitHeapMemory = commitHeapMemory; - this.commitChaining = commitChaining; this.commitUser = commitUser; } @@ -129,21 +126,21 @@ public DataStreamSink sinkFrom( .transform( GLOBAL_COMMITTER_NAME, typeInfo, - new CommitterOperator<>( + new CommitterOperatorFactory<>( true, false, - commitChaining, commitUser, createCommitterFactory(), createCommittableStateManager())) .setParallelism(input.getParallelism()); configureGlobalCommitter(committed, commitCpuCores, commitHeapMemory); - return committed.addSink(new DiscardingSink<>()).name("end").setParallelism(1); + return committed.sinkTo(new DiscardingSink<>()).name("end").setParallelism(1); } - protected OneInputStreamOperator createWriteOperator( - StoreSinkWrite.WithWriteBufferProvider writeProvider, String commitUser) { - return new CdcRecordStoreMultiWriteOperator( + protected OneInputStreamOperatorFactory + createWriteOperator( + StoreSinkWrite.WithWriteBufferProvider writeProvider, String commitUser) { + return new CdcRecordStoreMultiWriteOperator.Factory( catalogLoader, writeProvider, commitUser, new Options()); } diff --git a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/FlinkCdcSyncDatabaseSinkBuilder.java b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/FlinkCdcSyncDatabaseSinkBuilder.java index ed8fdd113389..a9ad66847b4b 100644 --- a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/FlinkCdcSyncDatabaseSinkBuilder.java +++ b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/FlinkCdcSyncDatabaseSinkBuilder.java @@ -66,7 +66,6 @@ public class FlinkCdcSyncDatabaseSinkBuilder { @Nullable private Integer parallelism; private double committerCpu; @Nullable private MemorySize committerMemory; - private boolean commitChaining; // Paimon catalog used to check and create tables. There will be two // places where this catalog is used. 1) in processing function, @@ -103,7 +102,6 @@ public FlinkCdcSyncDatabaseSinkBuilder withTableOptions(Options options) { this.parallelism = options.get(FlinkConnectorOptions.SINK_PARALLELISM); this.committerCpu = options.get(FlinkConnectorOptions.SINK_COMMITTER_CPU); this.committerMemory = options.get(FlinkConnectorOptions.SINK_COMMITTER_MEMORY); - this.commitChaining = options.get(FlinkConnectorOptions.SINK_COMMITTER_OPERATOR_CHAINING); this.commitUser = createCommitUser(options); return this; } @@ -169,7 +167,7 @@ private void buildCombinedCdcSink() { FlinkCdcMultiTableSink sink = new FlinkCdcMultiTableSink( - catalogLoader, committerCpu, committerMemory, commitChaining, commitUser); + catalogLoader, committerCpu, committerMemory, commitUser); sink.sinkFrom(partitioned); } diff --git a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/RichCdcRecord.java b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/RichCdcRecord.java index 7fc0c3ff7b09..04b86fea568f 100644 --- a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/RichCdcRecord.java +++ b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/RichCdcRecord.java @@ -48,7 +48,7 @@ public RichCdcRecord(CdcRecord cdcRecord, List fields) { } public boolean hasPayload() { - return !cdcRecord.fields().isEmpty(); + return !cdcRecord.data().isEmpty(); } public RowKind kind() { @@ -95,7 +95,7 @@ public static class Builder { private final RowKind kind; private final AtomicInteger fieldId; private final List fields = new ArrayList<>(); - private final Map fieldValues = new HashMap<>(); + private final Map data = new HashMap<>(); public Builder(RowKind kind, AtomicInteger fieldId) { this.kind = kind; @@ -109,12 +109,12 @@ public Builder field(String name, DataType type, String value) { public Builder field( String name, DataType type, String value, @Nullable String description) { fields.add(new DataField(fieldId.incrementAndGet(), name, type, description)); - fieldValues.put(name, value); + data.put(name, value); return this; } public RichCdcRecord build() { - return new RichCdcRecord(new CdcRecord(kind, fieldValues), fields); + return new RichCdcRecord(new CdcRecord(kind, data), fields); } } } diff --git a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/UpdatedDataFieldsProcessFunction.java b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/UpdatedDataFieldsProcessFunction.java index 4a33eb1b7ec9..64f00d96b0f5 100644 --- a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/UpdatedDataFieldsProcessFunction.java +++ b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/UpdatedDataFieldsProcessFunction.java @@ -23,11 +23,18 @@ import org.apache.paimon.schema.SchemaChange; import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.types.DataField; +import org.apache.paimon.types.FieldIdentifier; +import org.apache.paimon.types.RowType; +import org.apache.commons.collections.CollectionUtils; import org.apache.flink.streaming.api.functions.ProcessFunction; import org.apache.flink.util.Collector; +import java.util.HashSet; import java.util.List; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; /** * A {@link ProcessFunction} to handle schema changes. New schema is represented by a list of {@link @@ -43,19 +50,51 @@ public class UpdatedDataFieldsProcessFunction private final Identifier identifier; + private Set latestFields; + public UpdatedDataFieldsProcessFunction( SchemaManager schemaManager, Identifier identifier, Catalog.Loader catalogLoader) { super(catalogLoader); this.schemaManager = schemaManager; this.identifier = identifier; + this.latestFields = new HashSet<>(); } @Override public void processElement( List updatedDataFields, Context context, Collector collector) throws Exception { - for (SchemaChange schemaChange : extractSchemaChanges(schemaManager, updatedDataFields)) { + List actualUpdatedDataFields = + updatedDataFields.stream() + .filter( + dataField -> + !latestDataFieldContain(new FieldIdentifier(dataField))) + .collect(Collectors.toList()); + if (CollectionUtils.isEmpty(actualUpdatedDataFields)) { + return; + } + for (SchemaChange schemaChange : + extractSchemaChanges(schemaManager, actualUpdatedDataFields)) { applySchemaChange(schemaManager, schemaChange, identifier); } + /** + * Here, actualUpdatedDataFields cannot be used to update latestFields because there is a + * non-SchemaChange.AddColumn scenario. Otherwise, the previously existing fields cannot be + * modified again. + */ + updateLatestFields(); + } + + private boolean latestDataFieldContain(FieldIdentifier dataField) { + return latestFields.stream().anyMatch(previous -> Objects.equals(previous, dataField)); + } + + private void updateLatestFields() { + RowType oldRowType = schemaManager.latest().get().logicalRowType(); + Set fieldIdentifiers = + oldRowType.getFields().stream() + .map(item -> new FieldIdentifier(item)) + .collect(Collectors.toSet()); + latestFields = fieldIdentifiers; } } diff --git a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/UpdatedDataFieldsProcessFunctionBase.java b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/UpdatedDataFieldsProcessFunctionBase.java index 77c49e8f3da2..4f02b784c2ba 100644 --- a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/UpdatedDataFieldsProcessFunctionBase.java +++ b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/UpdatedDataFieldsProcessFunctionBase.java @@ -31,6 +31,7 @@ import org.apache.paimon.utils.Preconditions; import org.apache.paimon.utils.StringUtils; +import org.apache.flink.api.common.functions.OpenContext; import org.apache.flink.configuration.Configuration; import org.apache.flink.streaming.api.functions.ProcessFunction; import org.slf4j.Logger; @@ -73,7 +74,16 @@ protected UpdatedDataFieldsProcessFunctionBase(Catalog.Loader catalogLoader) { this.catalogLoader = catalogLoader; } - @Override + /** + * Do not annotate with @override here to maintain compatibility with Flink 1.18-. + */ + public void open(OpenContext openContext) throws Exception { + open(new Configuration()); + } + + /** + * Do not annotate with @override here to maintain compatibility with Flink 2.0+. + */ public void open(Configuration parameters) { this.catalog = catalogLoader.load(); this.allowUpperCase = this.catalog.allowUpperCase(); @@ -100,6 +110,9 @@ protected void applySchemaChange( } else if (schemaChange instanceof SchemaChange.UpdateColumnType) { SchemaChange.UpdateColumnType updateColumnType = (SchemaChange.UpdateColumnType) schemaChange; + Preconditions.checkState( + updateColumnType.fieldNames().length == 1, + "Paimon CDC currently does not support nested type schema evolution."); TableSchema schema = schemaManager .latest() @@ -107,11 +120,11 @@ protected void applySchemaChange( () -> new RuntimeException( "Table does not exist. This is unexpected.")); - int idx = schema.fieldNames().indexOf(updateColumnType.fieldName()); + int idx = schema.fieldNames().indexOf(updateColumnType.fieldNames()[0]); Preconditions.checkState( idx >= 0, "Field name " - + updateColumnType.fieldName() + + updateColumnType.fieldNames()[0] + " does not exist in table. This is unexpected."); DataType oldType = schema.fields().get(idx).type(); DataType newType = updateColumnType.newDataType(); @@ -123,7 +136,7 @@ protected void applySchemaChange( throw new UnsupportedOperationException( String.format( "Cannot convert field %s from type %s to %s of Paimon table %s.", - updateColumnType.fieldName(), + updateColumnType.fieldNames()[0], oldType, newType, identifier.getFullName())); diff --git a/paimon-flink/paimon-flink-cdc/src/main/resources/META-INF/services/org.apache.paimon.factories.Factory b/paimon-flink/paimon-flink-cdc/src/main/resources/META-INF/services/org.apache.paimon.factories.Factory index 9c4c4d0ac3a2..1b30c7ab6396 100644 --- a/paimon-flink/paimon-flink-cdc/src/main/resources/META-INF/services/org.apache.paimon.factories.Factory +++ b/paimon-flink/paimon-flink-cdc/src/main/resources/META-INF/services/org.apache.paimon.factories.Factory @@ -27,9 +27,11 @@ org.apache.paimon.flink.action.cdc.mongodb.MongoDBSyncDatabaseActionFactory org.apache.paimon.flink.action.cdc.postgres.PostgresSyncTableActionFactory ### message queue data format factories +org.apache.paimon.flink.action.cdc.format.aliyun.AliyunDataFormatFactory org.apache.paimon.flink.action.cdc.format.canal.CanalDataFormatFactory org.apache.paimon.flink.action.cdc.format.debezium.DebeziumAvroDataFormatFactory org.apache.paimon.flink.action.cdc.format.debezium.DebeziumJsonDataFormatFactory org.apache.paimon.flink.action.cdc.format.json.JsonDataFormatFactory org.apache.paimon.flink.action.cdc.format.maxwell.MaxwellDataFormatFactory org.apache.paimon.flink.action.cdc.format.ogg.OggDataFormatFactory +org.apache.paimon.flink.action.cdc.format.dms.DMSDataFormatFactory diff --git a/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/action/cdc/CdcActionITCaseBase.java b/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/action/cdc/CdcActionITCaseBase.java index 08289569086a..00a8b236173b 100644 --- a/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/action/cdc/CdcActionITCaseBase.java +++ b/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/action/cdc/CdcActionITCaseBase.java @@ -38,6 +38,7 @@ import org.apache.paimon.types.RowType; import org.apache.flink.api.common.JobStatus; +import org.apache.flink.api.common.RuntimeExecutionMode; import org.apache.flink.core.execution.JobClient; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.junit.jupiter.api.AfterEach; @@ -214,6 +215,15 @@ protected List nullableToArgs(String argKey, @Nullable T nullable) { } public JobClient runActionWithDefaultEnv(ActionBase action) throws Exception { + env.setRuntimeMode(RuntimeExecutionMode.STREAMING); + action.withStreamExecutionEnvironment(env).build(); + JobClient client = env.executeAsync(); + waitJobRunning(client); + return client; + } + + public JobClient runActionWithBatchEnv(ActionBase action) throws Exception { + env.setRuntimeMode(RuntimeExecutionMode.BATCH); action.withStreamExecutionEnvironment(env).build(); JobClient client = env.executeAsync(); waitJobRunning(client); diff --git a/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/action/cdc/SchemaEvolutionTest.java b/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/action/cdc/SchemaEvolutionTest.java new file mode 100644 index 000000000000..9ba18376867f --- /dev/null +++ b/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/action/cdc/SchemaEvolutionTest.java @@ -0,0 +1,219 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.flink.action.cdc; + +import org.apache.paimon.CoreOptions; +import org.apache.paimon.catalog.Catalog; +import org.apache.paimon.catalog.Identifier; +import org.apache.paimon.flink.FlinkCatalogFactory; +import org.apache.paimon.flink.sink.cdc.UpdatedDataFieldsProcessFunction; +import org.apache.paimon.fs.FileIO; +import org.apache.paimon.fs.Path; +import org.apache.paimon.fs.local.LocalFileIO; +import org.apache.paimon.options.Options; +import org.apache.paimon.schema.Schema; +import org.apache.paimon.schema.SchemaManager; +import org.apache.paimon.schema.SchemaUtils; +import org.apache.paimon.schema.TableSchema; +import org.apache.paimon.table.FileStoreTable; +import org.apache.paimon.table.FileStoreTableFactory; +import org.apache.paimon.table.TableTestBase; +import org.apache.paimon.types.BigIntType; +import org.apache.paimon.types.DataField; +import org.apache.paimon.types.DataTypes; +import org.apache.paimon.types.DecimalType; +import org.apache.paimon.types.DoubleType; +import org.apache.paimon.types.IntType; +import org.apache.paimon.types.VarCharType; + +import org.apache.flink.streaming.api.datastream.DataStream; +import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.util.Arrays; +import java.util.List; + +/** Used to test schema evolution related logic. */ +public class SchemaEvolutionTest extends TableTestBase { + + private static List> prepareData() { + List upField1 = + Arrays.asList( + new DataField(0, "col_0", new VarCharType(), "test description."), + new DataField(1, "col_1", new IntType(), "test description."), + new DataField(2, "col_2", new IntType(), "test description."), + new DataField(3, "col_3", new VarCharType(), "Someone's desc."), + new DataField(4, "col_4", new VarCharType(), "Someone's desc."), + new DataField(5, "col_5", new VarCharType(), "Someone's desc."), + new DataField(6, "col_6", new DecimalType(), "Someone's desc."), + new DataField(7, "col_7", new VarCharType(), "Someone's desc."), + new DataField(8, "col_8", new VarCharType(), "Someone's desc."), + new DataField(9, "col_9", new VarCharType(), "Someone's desc."), + new DataField(10, "col_10", new VarCharType(), "Someone's desc."), + new DataField(11, "col_11", new VarCharType(), "Someone's desc."), + new DataField(12, "col_12", new DoubleType(), "Someone's desc."), + new DataField(13, "col_13", new VarCharType(), "Someone's desc."), + new DataField(14, "col_14", new VarCharType(), "Someone's desc."), + new DataField(15, "col_15", new VarCharType(), "Someone's desc."), + new DataField(16, "col_16", new VarCharType(), "Someone's desc."), + new DataField(17, "col_17", new VarCharType(), "Someone's desc."), + new DataField(18, "col_18", new VarCharType(), "Someone's desc."), + new DataField(19, "col_19", new VarCharType(), "Someone's desc."), + new DataField(20, "col_20", new VarCharType(), "Someone's desc.")); + List upField2 = + Arrays.asList( + new DataField(0, "col_0", new VarCharType(), "test description."), + new DataField(1, "col_1", new BigIntType(), "test description."), + new DataField(2, "col_2", new IntType(), "test description."), + new DataField(3, "col_3", new VarCharType(), "Someone's desc."), + new DataField(4, "col_4", new VarCharType(), "Someone's desc."), + new DataField(5, "col_5", new VarCharType(), "Someone's desc."), + new DataField(6, "col_6", new DecimalType(), "Someone's desc."), + new DataField(7, "col_7", new VarCharType(), "Someone's desc."), + new DataField(8, "col_8", new VarCharType(), "Someone's desc."), + new DataField(9, "col_9", new VarCharType(), "Someone's desc."), + new DataField(10, "col_10", new VarCharType(), "Someone's desc."), + new DataField(11, "col_11", new VarCharType(), "Someone's desc."), + new DataField(12, "col_12", new DoubleType(), "Someone's desc."), + new DataField(13, "col_13", new VarCharType(), "Someone's desc."), + new DataField(14, "col_14", new VarCharType(), "Someone's desc."), + new DataField(15, "col_15", new VarCharType(), "Someone's desc."), + new DataField(16, "col_16", new VarCharType(), "Someone's desc."), + new DataField(17, "col_17", new VarCharType(), "Someone's desc."), + new DataField(18, "col_18", new VarCharType(), "Someone's desc."), + new DataField(19, "col_19", new VarCharType(), "Someone's desc."), + new DataField(20, "col_20", new VarCharType(), "Someone's desc.")); + List upField3 = + Arrays.asList( + new DataField(0, "col_0", new VarCharType(), "test description."), + new DataField(1, "col_1", new BigIntType(), "test description."), + new DataField(2, "col_2", new IntType(), "test description 2."), + new DataField(3, "col_3", new VarCharType(), "Someone's desc."), + new DataField(4, "col_4", new VarCharType(), "Someone's desc."), + new DataField(5, "col_5", new VarCharType(), "Someone's desc."), + new DataField(6, "col_6", new DecimalType(), "Someone's desc."), + new DataField(7, "col_7", new VarCharType(), "Someone's desc."), + new DataField(8, "col_8", new VarCharType(), "Someone's desc."), + new DataField(9, "col_9", new VarCharType(), "Someone's desc."), + new DataField(10, "col_10", new VarCharType(), "Someone's desc."), + new DataField(11, "col_11", new VarCharType(), "Someone's desc."), + new DataField(12, "col_12", new DoubleType(), "Someone's desc."), + new DataField(13, "col_13", new VarCharType(), "Someone's desc."), + new DataField(14, "col_14", new VarCharType(), "Someone's desc."), + new DataField(15, "col_15", new VarCharType(), "Someone's desc."), + new DataField(16, "col_16", new VarCharType(), "Someone's desc."), + new DataField(17, "col_17", new VarCharType(), "Someone's desc."), + new DataField(18, "col_18", new VarCharType(), "Someone's desc."), + new DataField(19, "col_19", new VarCharType(), "Someone's desc."), + new DataField(20, "col_20", new VarCharType(), "Someone's desc.")); + List upField4 = + Arrays.asList( + new DataField(0, "col_0", new VarCharType(), "test description."), + new DataField(1, "col_1", new BigIntType(), "test description."), + new DataField(2, "col_2", new IntType(), "test description."), + new DataField(3, "col_3_1", new VarCharType(), "Someone's desc."), + new DataField(4, "col_4", new VarCharType(), "Someone's desc."), + new DataField(5, "col_5", new VarCharType(), "Someone's desc."), + new DataField(6, "col_6", new DecimalType(), "Someone's desc."), + new DataField(7, "col_7", new VarCharType(), "Someone's desc."), + new DataField(8, "col_8", new VarCharType(), "Someone's desc."), + new DataField(9, "col_9", new VarCharType(), "Someone's desc."), + new DataField(10, "col_10", new VarCharType(), "Someone's desc."), + new DataField(11, "col_11", new VarCharType(), "Someone's desc."), + new DataField(12, "col_12", new DoubleType(), "Someone's desc."), + new DataField(13, "col_13", new VarCharType(), "Someone's desc."), + new DataField(14, "col_14", new VarCharType(), "Someone's desc."), + new DataField(15, "col_15", new VarCharType(), "Someone's desc."), + new DataField(16, "col_16", new VarCharType(), "Someone's desc."), + new DataField(17, "col_17", new VarCharType(), "Someone's desc."), + new DataField(18, "col_18", new VarCharType(), "Someone's desc."), + new DataField(19, "col_19", new VarCharType(), "Someone's desc."), + new DataField(20, "col_20", new VarCharType(), "Someone's desc.")); + List upField5 = + Arrays.asList( + new DataField(0, "col_0", new VarCharType(), "test description."), + new DataField(1, "col_1", new BigIntType(), "test description."), + new DataField(2, "col_2_1", new BigIntType(), "test description 2."), + new DataField(3, "col_3", new VarCharType(), "Someone's desc."), + new DataField(4, "col_4", new VarCharType(), "Someone's desc."), + new DataField(5, "col_5", new VarCharType(), "Someone's desc."), + new DataField(6, "col_6", new DecimalType(), "Someone's desc."), + new DataField(7, "col_7", new VarCharType(), "Someone's desc."), + new DataField(8, "col_8", new VarCharType(), "Someone's desc."), + new DataField(9, "col_9", new VarCharType(), "Someone's desc."), + new DataField(10, "col_10", new VarCharType(), "Someone's desc."), + new DataField(11, "col_11", new VarCharType(), "Someone's desc."), + new DataField(12, "col_12", new DoubleType(), "Someone's desc."), + new DataField(13, "col_13", new VarCharType(), "Someone's desc."), + new DataField(14, "col_14", new VarCharType(), "Someone's desc."), + new DataField(15, "col_15", new VarCharType(), "Someone's desc."), + new DataField(16, "col_16", new VarCharType(), "Someone's desc."), + new DataField(17, "col_17", new VarCharType(), "Someone's desc."), + new DataField(18, "col_18", new VarCharType(), "Someone's desc."), + new DataField(19, "col_19", new VarCharType(), "Someone's desc."), + new DataField(20, "col_20", new VarCharType(), "Someone's desc.")); + return Arrays.asList(upField1, upField2, upField3, upField4, upField5); + } + + private FileStoreTable table; + private String tableName = "MyTable"; + + @BeforeEach + public void before() throws Exception { + FileIO fileIO = LocalFileIO.create(); + Path tablePath = new Path(String.format("%s/%s.db/%s", warehouse, database, tableName)); + Schema schema = + Schema.newBuilder() + .column("pk", DataTypes.INT()) + .column("pt1", DataTypes.INT()) + .column("pt2", DataTypes.INT()) + .column("col1", DataTypes.INT()) + .partitionKeys("pt1", "pt2") + .primaryKey("pk", "pt1", "pt2") + .option(CoreOptions.CHANGELOG_PRODUCER.key(), "input") + .option(CoreOptions.BUCKET.key(), "2") + .option(CoreOptions.SEQUENCE_FIELD.key(), "col1") + .build(); + TableSchema tableSchema = + SchemaUtils.forceCommit(new SchemaManager(fileIO, tablePath), schema); + table = FileStoreTableFactory.create(LocalFileIO.create(), tablePath, tableSchema); + } + + @Test + public void testSchemaEvolution() throws Exception { + final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); + DataStream> upDataFieldStream = env.fromCollection(prepareData()); + Options options = new Options(); + options.set("warehouse", tempPath.toString()); + final Catalog.Loader catalogLoader = () -> FlinkCatalogFactory.createPaimonCatalog(options); + Identifier identifier = Identifier.create(database, tableName); + DataStream schemaChangeProcessFunction = + upDataFieldStream + .process( + new UpdatedDataFieldsProcessFunction( + new SchemaManager(table.fileIO(), table.location()), + identifier, + catalogLoader)) + .name("Schema Evolution"); + schemaChangeProcessFunction.getTransformation().setParallelism(1); + schemaChangeProcessFunction.getTransformation().setMaxParallelism(1); + env.execute(); + } +} diff --git a/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/action/cdc/TableNameConverterTest.java b/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/action/cdc/TableNameConverterTest.java new file mode 100644 index 000000000000..89bbadfeb8c8 --- /dev/null +++ b/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/action/cdc/TableNameConverterTest.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.flink.action.cdc; + +import org.junit.Assert; +import org.junit.Test; + +import java.util.HashMap; +import java.util.Map; + +/** Tests for {@link TableNameConverter}. */ +public class TableNameConverterTest { + + @Test + public void testConvertTableName() { + Map tableMapping = new HashMap<>(1); + tableMapping.put("mapped_src", "mapped_TGT"); + TableNameConverter caseConverter = + new TableNameConverter(true, true, "pre_", "_pos", tableMapping); + Assert.assertEquals(caseConverter.convert("", "mapped_SRC"), "mapped_TGT"); + + Assert.assertEquals(caseConverter.convert("", "unmapped_src"), "pre_unmapped_src_pos"); + + TableNameConverter noCaseConverter = + new TableNameConverter(false, true, "pre_", "_pos", tableMapping); + Assert.assertEquals(noCaseConverter.convert("", "mapped_src"), "mapped_tgt"); + Assert.assertEquals(noCaseConverter.convert("", "unmapped_src"), "pre_unmapped_src_pos"); + } + + @Test + public void testConvertTableNameByDBPrefix_Suffix() { + Map dbPrefix = new HashMap<>(2); + dbPrefix.put("db_with_prefix", "db_pref_"); + dbPrefix.put("db_with_prefix_suffix", "db_pref_"); + + Map dbSuffix = new HashMap<>(2); + dbSuffix.put("db_with_suffix", "_db_suff"); + dbSuffix.put("db_with_prefix_suffix", "_db_suff"); + + TableNameConverter tblNameConverter = + new TableNameConverter(false, true, dbPrefix, dbSuffix, "pre_", "_suf", null); + + // Tables in the specified db should have the specified prefix and suffix. + + // db prefix + normal suffix + Assert.assertEquals( + "db_pref_table_name_suf", tblNameConverter.convert("db_with_prefix", "table_name")); + + // normal prefix + db suffix + Assert.assertEquals( + "pre_table_name_db_suff", tblNameConverter.convert("db_with_suffix", "table_name")); + + // db prefix + db suffix + Assert.assertEquals( + "db_pref_table_name_db_suff", + tblNameConverter.convert("db_with_prefix_suffix", "table_name")); + + // only normal prefix and suffix + Assert.assertEquals( + "pre_table_name_suf", + tblNameConverter.convert("db_without_prefix_suffix", "table_name")); + } +} diff --git a/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/action/cdc/format/aliyun/AliyunJsonRecordParserTest.java b/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/action/cdc/format/aliyun/AliyunJsonRecordParserTest.java new file mode 100644 index 000000000000..f06268d700e5 --- /dev/null +++ b/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/action/cdc/format/aliyun/AliyunJsonRecordParserTest.java @@ -0,0 +1,167 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.flink.action.cdc.format.aliyun; + +import org.apache.paimon.flink.action.cdc.CdcSourceRecord; +import org.apache.paimon.flink.action.cdc.TypeMapping; +import org.apache.paimon.flink.action.cdc.kafka.KafkaActionITCaseBase; +import org.apache.paimon.flink.action.cdc.watermark.MessageQueueCdcTimestampExtractor; +import org.apache.paimon.flink.sink.cdc.CdcRecord; +import org.apache.paimon.flink.sink.cdc.RichCdcMultiplexRecord; +import org.apache.paimon.schema.Schema; +import org.apache.paimon.types.RowKind; + +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.databind.JsonNode; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.databind.ObjectMapper; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +/** Test for AliyunJsonRecordParser. */ +public class AliyunJsonRecordParserTest extends KafkaActionITCaseBase { + + private static final Logger log = LoggerFactory.getLogger(AliyunJsonRecordParserTest.class); + private static List insertList = new ArrayList<>(); + private static List updateList = new ArrayList<>(); + private static List deleteList = new ArrayList<>(); + + private static ObjectMapper objMapper = new ObjectMapper(); + + @Before + public void setup() { + String insertRes = "kafka/aliyun/table/event/event-insert.txt"; + String updateRes = "kafka/aliyun/table/event/event-update-in-one.txt"; + String deleteRes = "kafka/aliyun/table/event/event-delete.txt"; + URL url; + try { + url = AliyunJsonRecordParserTest.class.getClassLoader().getResource(insertRes); + Files.readAllLines(Paths.get(url.toURI())).stream() + .filter(this::isRecordLine) + .forEach(e -> insertList.add(e)); + + url = AliyunJsonRecordParserTest.class.getClassLoader().getResource(updateRes); + Files.readAllLines(Paths.get(url.toURI())).stream() + .filter(this::isRecordLine) + .forEach(e -> updateList.add(e)); + + url = AliyunJsonRecordParserTest.class.getClassLoader().getResource(deleteRes); + Files.readAllLines(Paths.get(url.toURI())).stream() + .filter(this::isRecordLine) + .forEach(e -> deleteList.add(e)); + + } catch (Exception e) { + log.error("Fail to init aliyun-json cases", e); + } + } + + @Test + public void extractInsertRecord() throws Exception { + AliyunRecordParser parser = + new AliyunRecordParser(TypeMapping.defaultMapping(), Collections.emptyList()); + for (String json : insertList) { + // 将json解析为JsonNode对象 + JsonNode rootNode = objMapper.readValue(json, JsonNode.class); + CdcSourceRecord cdcRecord = new CdcSourceRecord(rootNode); + Schema schema = parser.buildSchema(cdcRecord); + Assert.assertEquals(schema.primaryKeys(), Arrays.asList("id")); + + List records = parser.extractRecords(); + Assert.assertEquals(records.size(), 1); + + CdcRecord result = records.get(0).toRichCdcRecord().toCdcRecord(); + Assert.assertEquals(result.kind(), RowKind.INSERT); + + String dbName = parser.getDatabaseName(); + Assert.assertEquals(dbName, "bigdata_test"); + + String tableName = parser.getTableName(); + Assert.assertEquals(tableName, "sync_test_table"); + + MessageQueueCdcTimestampExtractor extractor = new MessageQueueCdcTimestampExtractor(); + Assert.assertTrue(extractor.extractTimestamp(cdcRecord) > 0); + } + } + + @Test + public void extractUpdateRecord() throws Exception { + AliyunRecordParser parser = + new AliyunRecordParser(TypeMapping.defaultMapping(), Collections.emptyList()); + for (String json : updateList) { + // 将json解析为JsonNode对象 + JsonNode jsonNode = objMapper.readValue(json, JsonNode.class); + CdcSourceRecord cdcRecord = new CdcSourceRecord(jsonNode); + Schema schema = parser.buildSchema(cdcRecord); + Assert.assertEquals(schema.primaryKeys(), Arrays.asList("id")); + + List records = parser.extractRecords(); + Assert.assertEquals(records.size(), 1); + + CdcRecord result = records.get(0).toRichCdcRecord().toCdcRecord(); + Assert.assertEquals(result.kind(), RowKind.UPDATE_AFTER); + + String dbName = parser.getDatabaseName(); + Assert.assertEquals(dbName, "bigdata_test"); + + String tableName = parser.getTableName(); + Assert.assertEquals(tableName, "sync_test_table"); + + MessageQueueCdcTimestampExtractor extractor = new MessageQueueCdcTimestampExtractor(); + Assert.assertTrue(extractor.extractTimestamp(cdcRecord) > 0); + } + } + + @Test + public void extractDeleteRecord() throws Exception { + AliyunRecordParser parser = + new AliyunRecordParser(TypeMapping.defaultMapping(), Collections.emptyList()); + for (String json : deleteList) { + // 将json解析为JsonNode对象 + JsonNode jsonNode = objMapper.readValue(json, JsonNode.class); + CdcSourceRecord cdcRecord = new CdcSourceRecord(jsonNode); + Schema schema = parser.buildSchema(cdcRecord); + Assert.assertEquals(schema.primaryKeys(), Arrays.asList("id")); + + List records = parser.extractRecords(); + Assert.assertEquals(records.size(), 1); + + CdcRecord result = records.get(0).toRichCdcRecord().toCdcRecord(); + Assert.assertEquals(result.kind(), RowKind.DELETE); + + String dbName = parser.getDatabaseName(); + Assert.assertEquals(dbName, "bigdata_test"); + + String tableName = parser.getTableName(); + Assert.assertEquals(tableName, "sync_test_table"); + + MessageQueueCdcTimestampExtractor extractor = new MessageQueueCdcTimestampExtractor(); + Assert.assertTrue(extractor.extractTimestamp(cdcRecord) > 0); + } + } +} diff --git a/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/action/cdc/kafka/KafkaAWSDMSSyncDatabaseActionITCase.java b/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/action/cdc/kafka/KafkaAWSDMSSyncDatabaseActionITCase.java new file mode 100644 index 000000000000..da9f863dc07b --- /dev/null +++ b/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/action/cdc/kafka/KafkaAWSDMSSyncDatabaseActionITCase.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.flink.action.cdc.kafka; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; + +import java.util.Map; + +/** IT cases for {@link KafkaSyncDatabaseAction}. */ +public class KafkaAWSDMSSyncDatabaseActionITCase extends KafkaSyncDatabaseActionITCase { + + private static final String AWSDMS = "aws-dms"; + + @Override + protected KafkaSyncDatabaseActionBuilder syncDatabaseActionBuilder( + Map kafkaConfig) { + KafkaSyncDatabaseActionBuilder builder = new KafkaSyncDatabaseActionBuilder(kafkaConfig); + builder.withPrimaryKeys("id"); + return builder; + } + + @Test + @Timeout(60) + public void testSchemaEvolutionMultiTopic() throws Exception { + testSchemaEvolutionMultiTopic(AWSDMS); + } + + @Test + @Timeout(60) + public void testSchemaEvolutionOneTopic() throws Exception { + testSchemaEvolutionOneTopic(AWSDMS); + } + + @Test + public void testTopicIsEmpty() { + testTopicIsEmpty(AWSDMS); + } + + @Test + @Timeout(60) + public void testTableAffixMultiTopic() throws Exception { + testTableAffixMultiTopic(AWSDMS); + } + + @Test + @Timeout(60) + public void testTableAffixOneTopic() throws Exception { + testTableAffixOneTopic(AWSDMS); + } + + @Test + @Timeout(60) + public void testIncludingTables() throws Exception { + testIncludingTables(AWSDMS); + } + + @Test + @Timeout(60) + public void testExcludingTables() throws Exception { + testExcludingTables(AWSDMS); + } + + @Test + @Timeout(60) + public void testIncludingAndExcludingTables() throws Exception { + testIncludingAndExcludingTables(AWSDMS); + } +} diff --git a/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/action/cdc/kafka/KafkaAWSDMSSyncTableActionITCase.java b/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/action/cdc/kafka/KafkaAWSDMSSyncTableActionITCase.java new file mode 100644 index 000000000000..02ac86cdab69 --- /dev/null +++ b/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/action/cdc/kafka/KafkaAWSDMSSyncTableActionITCase.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.flink.action.cdc.kafka; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; + +/** IT cases for {@link KafkaSyncTableAction}. */ +public class KafkaAWSDMSSyncTableActionITCase extends KafkaSyncTableActionITCase { + + private static final String AWSDMS = "aws-dms"; + + @Test + @Timeout(60) + public void testSchemaEvolution() throws Exception { + runSingleTableSchemaEvolution("schemaevolution", AWSDMS); + } + + @Test + @Timeout(60) + public void testAssertSchemaCompatible() throws Exception { + testAssertSchemaCompatible(AWSDMS); + } + + @Test + @Timeout(60) + public void testStarUpOptionSpecific() throws Exception { + testStarUpOptionSpecific(AWSDMS); + } + + @Test + @Timeout(60) + public void testStarUpOptionLatest() throws Exception { + testStarUpOptionLatest(AWSDMS); + } + + @Test + @Timeout(60) + public void testStarUpOptionTimestamp() throws Exception { + testStarUpOptionTimestamp(AWSDMS); + } + + @Test + @Timeout(60) + public void testStarUpOptionEarliest() throws Exception { + testStarUpOptionEarliest(AWSDMS); + } + + @Test + @Timeout(60) + public void testStarUpOptionGroup() throws Exception { + testStarUpOptionGroup(AWSDMS); + } + + @Test + @Timeout(60) + public void testComputedColumn() throws Exception { + testComputedColumn(AWSDMS); + } + + @Test + @Timeout(60) + public void testFieldValNullSyncTable() throws Exception { + testTableFiledValNull(AWSDMS); + } +} diff --git a/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/action/cdc/mongodb/MongoDBSyncTableActionITCase.java b/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/action/cdc/mongodb/MongoDBSyncTableActionITCase.java index 48c6cd481103..b4f31f2d6d3d 100644 --- a/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/action/cdc/mongodb/MongoDBSyncTableActionITCase.java +++ b/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/action/cdc/mongodb/MongoDBSyncTableActionITCase.java @@ -34,7 +34,9 @@ import java.util.List; import java.util.Map; +import static org.apache.paimon.testutils.assertj.PaimonAssertions.anyCauseMatches; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; /** IT cases for {@link MongoDBSyncTableAction}. */ public class MongoDBSyncTableActionITCase extends MongoDBActionITCaseBase { @@ -398,4 +400,26 @@ public void testComputedColumnWithCaseInsensitive() throws Exception { rowType, Collections.singletonList("_id")); } + + @Test + @Timeout(60) + public void testRuntimeExecutionModeCheckForCdcSync() { + Map mongodbConfig = getBasicMongoDBConfig(); + mongodbConfig.put("database", database); + mongodbConfig.put("collection", "products"); + mongodbConfig.put("field.name", "_id,name,description"); + mongodbConfig.put("parser.path", "$._id,$.name,$.description"); + mongodbConfig.put("schema.start.mode", "specified"); + + MongoDBSyncTableAction action = + syncTableActionBuilder(mongodbConfig) + .withTableConfig(getBasicTableConfig()) + .build(); + + assertThatThrownBy(() -> runActionWithBatchEnv(action)) + .satisfies( + anyCauseMatches( + IllegalArgumentException.class, + "It's only support STREAMING mode for flink-cdc sync table action")); + } } diff --git a/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/action/cdc/mongodb/MongodbSchemaITCase.java b/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/action/cdc/mongodb/MongodbSchemaITCase.java index 394cdd1f149b..f0328b566324 100644 --- a/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/action/cdc/mongodb/MongodbSchemaITCase.java +++ b/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/action/cdc/mongodb/MongodbSchemaITCase.java @@ -76,13 +76,12 @@ public static void initMongoDB() { @Test public void testCreateSchemaFromValidConfig() { Configuration mongodbConfig = new Configuration(); - mongodbConfig.setString(MongoDBSourceOptions.HOSTS, MONGODB_CONTAINER.getHostAndPort()); - mongodbConfig.setString(MongoDBSourceOptions.USERNAME, MongoDBContainer.PAIMON_USER); - mongodbConfig.setString( - MongoDBSourceOptions.PASSWORD, MongoDBContainer.PAIMON_USER_PASSWORD); - mongodbConfig.setString(MongoDBSourceOptions.CONNECTION_OPTIONS, "authSource=admin"); - mongodbConfig.setString(MongoDBSourceOptions.DATABASE, "testDatabase"); - mongodbConfig.setString(MongoDBSourceOptions.COLLECTION, "testCollection"); + mongodbConfig.set(MongoDBSourceOptions.HOSTS, MONGODB_CONTAINER.getHostAndPort()); + mongodbConfig.set(MongoDBSourceOptions.USERNAME, MongoDBContainer.PAIMON_USER); + mongodbConfig.set(MongoDBSourceOptions.PASSWORD, MongoDBContainer.PAIMON_USER_PASSWORD); + mongodbConfig.set(MongoDBSourceOptions.CONNECTION_OPTIONS, "authSource=admin"); + mongodbConfig.set(MongoDBSourceOptions.DATABASE, "testDatabase"); + mongodbConfig.set(MongoDBSourceOptions.COLLECTION, "testCollection"); Schema schema = MongodbSchemaUtils.getMongodbSchema(mongodbConfig); assertNotNull(schema); } @@ -90,13 +89,12 @@ public void testCreateSchemaFromValidConfig() { @Test public void testCreateSchemaFromInvalidHost() { Configuration mongodbConfig = new Configuration(); - mongodbConfig.setString(MongoDBSourceOptions.HOSTS, "127.0.0.1:12345"); - mongodbConfig.setString(MongoDBSourceOptions.USERNAME, MongoDBContainer.PAIMON_USER); - mongodbConfig.setString( - MongoDBSourceOptions.PASSWORD, MongoDBContainer.PAIMON_USER_PASSWORD); - mongodbConfig.setString(MongoDBSourceOptions.CONNECTION_OPTIONS, "authSource=admin"); - mongodbConfig.setString(MongoDBSourceOptions.DATABASE, "testDatabase"); - mongodbConfig.setString(MongoDBSourceOptions.COLLECTION, "testCollection"); + mongodbConfig.set(MongoDBSourceOptions.HOSTS, "127.0.0.1:12345"); + mongodbConfig.set(MongoDBSourceOptions.USERNAME, MongoDBContainer.PAIMON_USER); + mongodbConfig.set(MongoDBSourceOptions.PASSWORD, MongoDBContainer.PAIMON_USER_PASSWORD); + mongodbConfig.set(MongoDBSourceOptions.CONNECTION_OPTIONS, "authSource=admin"); + mongodbConfig.set(MongoDBSourceOptions.DATABASE, "testDatabase"); + mongodbConfig.set(MongoDBSourceOptions.COLLECTION, "testCollection"); assertThrows( RuntimeException.class, () -> MongodbSchemaUtils.getMongodbSchema(mongodbConfig)); @@ -106,7 +104,7 @@ public void testCreateSchemaFromInvalidHost() { public void testCreateSchemaFromIncompleteConfig() { // Create a Configuration object with missing necessary settings Configuration mongodbConfig = new Configuration(); - mongodbConfig.setString(MongoDBSourceOptions.HOSTS, MONGODB_CONTAINER.getHostAndPort()); + mongodbConfig.set(MongoDBSourceOptions.HOSTS, MONGODB_CONTAINER.getHostAndPort()); // Expect an exception to be thrown due to missing necessary settings assertThrows( NullPointerException.class, @@ -117,13 +115,12 @@ public void testCreateSchemaFromIncompleteConfig() { public void testCreateSchemaFromDynamicConfig() { // Create a Configuration object with the necessary settings Configuration mongodbConfig = new Configuration(); - mongodbConfig.setString(MongoDBSourceOptions.HOSTS, MONGODB_CONTAINER.getHostAndPort()); - mongodbConfig.setString(MongoDBSourceOptions.USERNAME, MongoDBContainer.PAIMON_USER); - mongodbConfig.setString( - MongoDBSourceOptions.PASSWORD, MongoDBContainer.PAIMON_USER_PASSWORD); - mongodbConfig.setString(MongoDBSourceOptions.CONNECTION_OPTIONS, "authSource=admin"); - mongodbConfig.setString(MongoDBSourceOptions.DATABASE, "testDatabase"); - mongodbConfig.setString(MongoDBSourceOptions.COLLECTION, "testCollection"); + mongodbConfig.set(MongoDBSourceOptions.HOSTS, MONGODB_CONTAINER.getHostAndPort()); + mongodbConfig.set(MongoDBSourceOptions.USERNAME, MongoDBContainer.PAIMON_USER); + mongodbConfig.set(MongoDBSourceOptions.PASSWORD, MongoDBContainer.PAIMON_USER_PASSWORD); + mongodbConfig.set(MongoDBSourceOptions.CONNECTION_OPTIONS, "authSource=admin"); + mongodbConfig.set(MongoDBSourceOptions.DATABASE, "testDatabase"); + mongodbConfig.set(MongoDBSourceOptions.COLLECTION, "testCollection"); // Call the method and check the results Schema schema = MongodbSchemaUtils.getMongodbSchema(mongodbConfig); @@ -142,13 +139,12 @@ public void testCreateSchemaFromDynamicConfig() { @Test public void testCreateSchemaFromInvalidDatabase() { Configuration mongodbConfig = new Configuration(); - mongodbConfig.setString(MongoDBSourceOptions.HOSTS, MONGODB_CONTAINER.getHostAndPort()); - mongodbConfig.setString(MongoDBSourceOptions.USERNAME, MongoDBContainer.PAIMON_USER); - mongodbConfig.setString( - MongoDBSourceOptions.PASSWORD, MongoDBContainer.PAIMON_USER_PASSWORD); - mongodbConfig.setString(MongoDBSourceOptions.CONNECTION_OPTIONS, "authSource=admin"); - mongodbConfig.setString(MongoDBSourceOptions.DATABASE, "invalidDatabase"); - mongodbConfig.setString(MongoDBSourceOptions.COLLECTION, "testCollection"); + mongodbConfig.set(MongoDBSourceOptions.HOSTS, MONGODB_CONTAINER.getHostAndPort()); + mongodbConfig.set(MongoDBSourceOptions.USERNAME, MongoDBContainer.PAIMON_USER); + mongodbConfig.set(MongoDBSourceOptions.PASSWORD, MongoDBContainer.PAIMON_USER_PASSWORD); + mongodbConfig.set(MongoDBSourceOptions.CONNECTION_OPTIONS, "authSource=admin"); + mongodbConfig.set(MongoDBSourceOptions.DATABASE, "invalidDatabase"); + mongodbConfig.set(MongoDBSourceOptions.COLLECTION, "testCollection"); assertThrows( RuntimeException.class, () -> MongodbSchemaUtils.getMongodbSchema(mongodbConfig)); @@ -157,13 +153,12 @@ public void testCreateSchemaFromInvalidDatabase() { @Test public void testCreateSchemaFromInvalidCollection() { Configuration mongodbConfig = new Configuration(); - mongodbConfig.setString(MongoDBSourceOptions.HOSTS, MONGODB_CONTAINER.getHostAndPort()); - mongodbConfig.setString(MongoDBSourceOptions.USERNAME, MongoDBContainer.PAIMON_USER); - mongodbConfig.setString( - MongoDBSourceOptions.PASSWORD, MongoDBContainer.PAIMON_USER_PASSWORD); - mongodbConfig.setString(MongoDBSourceOptions.CONNECTION_OPTIONS, "authSource=admin"); - mongodbConfig.setString(MongoDBSourceOptions.DATABASE, "testDatabase"); - mongodbConfig.setString(MongoDBSourceOptions.COLLECTION, "invalidCollection"); + mongodbConfig.set(MongoDBSourceOptions.HOSTS, MONGODB_CONTAINER.getHostAndPort()); + mongodbConfig.set(MongoDBSourceOptions.USERNAME, MongoDBContainer.PAIMON_USER); + mongodbConfig.set(MongoDBSourceOptions.PASSWORD, MongoDBContainer.PAIMON_USER_PASSWORD); + mongodbConfig.set(MongoDBSourceOptions.CONNECTION_OPTIONS, "authSource=admin"); + mongodbConfig.set(MongoDBSourceOptions.DATABASE, "testDatabase"); + mongodbConfig.set(MongoDBSourceOptions.COLLECTION, "invalidCollection"); assertThrows( RuntimeException.class, () -> MongodbSchemaUtils.getMongodbSchema(mongodbConfig)); diff --git a/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/action/cdc/mysql/MySqlSyncTableActionITCase.java b/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/action/cdc/mysql/MySqlSyncTableActionITCase.java index ce18d0b1f0e8..febbe4e1deaa 100644 --- a/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/action/cdc/mysql/MySqlSyncTableActionITCase.java +++ b/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/action/cdc/mysql/MySqlSyncTableActionITCase.java @@ -31,7 +31,8 @@ import org.apache.paimon.utils.CommonTestUtils; import org.apache.paimon.utils.JsonSerdeUtil; -import org.apache.flink.api.common.restartstrategy.RestartStrategies; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.configuration.RestartStrategyOptions; import org.apache.flink.core.execution.JobClient; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.junit.jupiter.api.BeforeAll; @@ -1285,8 +1286,11 @@ public void testDefaultCheckpointInterval() throws Exception { mySqlConfig.put("database-name", "default_checkpoint"); mySqlConfig.put("table-name", "t"); - StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); - env.setRestartStrategy(RestartStrategies.noRestart()); + // Using `none` to avoid compatibility issues with Flink 1.18-. + Configuration configuration = new Configuration(); + configuration.set(RestartStrategyOptions.RESTART_STRATEGY, "none"); + StreamExecutionEnvironment env = + StreamExecutionEnvironment.getExecutionEnvironment(configuration); MySqlSyncTableAction action = syncTableActionBuilder(mySqlConfig).build(); action.withStreamExecutionEnvironment(env); @@ -1497,4 +1501,39 @@ public void testUnknowMysqlScanStartupMode() { + scanStartupMode + "'. Valid scan.startup.mode for MySQL CDC are [initial, earliest-offset, latest-offset, specific-offset, timestamp, snapshot]")); } + + @Test + @Timeout(1000) + public void testRuntimeExecutionModeCheckForCdcSync() throws Exception { + Map mySqlConfig = getBasicMySqlConfig(); + mySqlConfig.put("database-name", "check_cdc_sync_runtime_execution_mode"); + mySqlConfig.put("table-name", "t"); + + Map tableConfig = getBasicTableConfig(); + tableConfig.put(CoreOptions.WRITE_ONLY.key(), "true"); + + MySqlSyncTableAction action = syncTableActionBuilder(mySqlConfig).build(); + + assertThatThrownBy(() -> runActionWithBatchEnv(action)) + .satisfies( + anyCauseMatches( + IllegalArgumentException.class, + "It's only support STREAMING mode for flink-cdc sync table action")); + + runActionWithDefaultEnv(action); + + FileStoreTable table = getFileStoreTable(); + + try (Statement statement = getStatement()) { + statement.executeUpdate("USE check_cdc_sync_runtime_execution_mode"); + statement.executeUpdate("INSERT INTO t VALUES (1, 'one'), (2, 'two')"); + RowType rowType = + RowType.of( + new DataType[] {DataTypes.INT().notNull(), DataTypes.VARCHAR(10)}, + new String[] {"k", "v1"}); + List primaryKeys = Collections.singletonList("k"); + List expected = Arrays.asList("+I[1, one]", "+I[2, two]"); + waitForResult(expected, table, rowType, primaryKeys); + } + } } diff --git a/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/action/cdc/postgres/PostgresSyncTableActionITCase.java b/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/action/cdc/postgres/PostgresSyncTableActionITCase.java index 10f14ca732d5..58d122b3c1ab 100644 --- a/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/action/cdc/postgres/PostgresSyncTableActionITCase.java +++ b/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/action/cdc/postgres/PostgresSyncTableActionITCase.java @@ -789,4 +789,29 @@ public void testCatalogAndTableConfig() { private FileStoreTable getFileStoreTable() throws Exception { return getFileStoreTable(tableName); } + + @Test + @Timeout(60) + public void testRuntimeExecutionModeCheckForCdcSync() { + Map postgresConfig = getBasicPostgresConfig(); + postgresConfig.put(PostgresSourceOptions.DATABASE_NAME.key(), DATABASE_NAME); + postgresConfig.put(PostgresSourceOptions.SCHEMA_NAME.key(), SCHEMA_NAME); + postgresConfig.put(PostgresSourceOptions.TABLE_NAME.key(), "schema_evolution_\\d+"); + + PostgresSyncTableAction action = + syncTableActionBuilder(postgresConfig) + .withCatalogConfig( + Collections.singletonMap( + CatalogOptions.METASTORE.key(), "test-alter-table")) + .withTableConfig(getBasicTableConfig()) + .withPartitionKeys("pt") + .withPrimaryKeys("pt", "_id") + .build(); + + assertThatThrownBy(() -> runActionWithBatchEnv(action)) + .satisfies( + anyCauseMatches( + IllegalArgumentException.class, + "It's only support STREAMING mode for flink-cdc sync table action")); + } } diff --git a/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/sink/cdc/CdcMultiplexRecordChannelComputerTest.java b/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/sink/cdc/CdcMultiplexRecordChannelComputerTest.java index ce0d484f4ce4..867cbdbae002 100644 --- a/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/sink/cdc/CdcMultiplexRecordChannelComputerTest.java +++ b/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/sink/cdc/CdcMultiplexRecordChannelComputerTest.java @@ -163,9 +163,9 @@ private void testImpl(Identifier tableId, List> input) { // assert that insert and delete records are routed into same channel - for (Map fields : input) { - CdcRecord insertRecord = new CdcRecord(RowKind.INSERT, fields); - CdcRecord deleteRecord = new CdcRecord(RowKind.DELETE, fields); + for (Map data : input) { + CdcRecord insertRecord = new CdcRecord(RowKind.INSERT, data); + CdcRecord deleteRecord = new CdcRecord(RowKind.DELETE, data); assertThat( channelComputer.channel( @@ -184,8 +184,8 @@ private void testImpl(Identifier tableId, List> input) { // assert that channel >= 0 int numTests = random.nextInt(10) + 1; for (int test = 0; test < numTests; test++) { - Map fields = input.get(random.nextInt(input.size())); - CdcRecord record = new CdcRecord(RowKind.INSERT, fields); + Map data = input.get(random.nextInt(input.size())); + CdcRecord record = new CdcRecord(RowKind.INSERT, data); int numBuckets = random.nextInt(numChannels * 4) + 1; for (int i = 0; i < numBuckets; i++) { diff --git a/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/sink/cdc/CdcRecordChannelComputerTest.java b/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/sink/cdc/CdcRecordChannelComputerTest.java index 9a19013e2983..8271ad18751c 100644 --- a/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/sink/cdc/CdcRecordChannelComputerTest.java +++ b/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/sink/cdc/CdcRecordChannelComputerTest.java @@ -128,9 +128,9 @@ private void testImpl(TableSchema schema, List> input) { // assert that channel(record) and channel(partition, bucket) gives the same result - for (Map fields : input) { - CdcRecord insertRecord = new CdcRecord(RowKind.INSERT, fields); - CdcRecord deleteRecord = new CdcRecord(RowKind.DELETE, fields); + for (Map data : input) { + CdcRecord insertRecord = new CdcRecord(RowKind.INSERT, data); + CdcRecord deleteRecord = new CdcRecord(RowKind.DELETE, data); extractor.setRecord(random.nextBoolean() ? insertRecord : deleteRecord); BinaryRow partition = extractor.partition(); @@ -151,8 +151,8 @@ private void testImpl(TableSchema schema, List> input) { bucketsPerChannel.put(i, 0); } - Map fields = input.get(random.nextInt(input.size())); - extractor.setRecord(new CdcRecord(RowKind.INSERT, fields)); + Map data = input.get(random.nextInt(input.size())); + extractor.setRecord(new CdcRecord(RowKind.INSERT, data)); BinaryRow partition = extractor.partition(); int numBuckets = random.nextInt(numChannels * 4) + 1; diff --git a/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/sink/cdc/CdcRecordKeyAndBucketExtractorTest.java b/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/sink/cdc/CdcRecordKeyAndBucketExtractorTest.java index 8384b7155a0e..802a3ea9d4cf 100644 --- a/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/sink/cdc/CdcRecordKeyAndBucketExtractorTest.java +++ b/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/sink/cdc/CdcRecordKeyAndBucketExtractorTest.java @@ -87,19 +87,19 @@ public void testExtract() throws Exception { StringData.fromString(v2)); expected.setRecord(rowData); - Map fields = new HashMap<>(); - fields.put("pt1", pt1); - fields.put("pt2", String.valueOf(pt2)); - fields.put("k1", String.valueOf(k1)); - fields.put("v1", String.valueOf(v1)); - fields.put("k2", k2); - fields.put("v2", v2); - - actual.setRecord(new CdcRecord(RowKind.INSERT, fields)); + Map data = new HashMap<>(); + data.put("pt1", pt1); + data.put("pt2", String.valueOf(pt2)); + data.put("k1", String.valueOf(k1)); + data.put("v1", String.valueOf(v1)); + data.put("k2", k2); + data.put("v2", v2); + + actual.setRecord(new CdcRecord(RowKind.INSERT, data)); assertThat(actual.partition()).isEqualTo(expected.partition()); assertThat(actual.bucket()).isEqualTo(expected.bucket()); - actual.setRecord(new CdcRecord(RowKind.DELETE, fields)); + actual.setRecord(new CdcRecord(RowKind.DELETE, data)); assertThat(actual.partition()).isEqualTo(expected.partition()); assertThat(actual.bucket()).isEqualTo(expected.bucket()); } @@ -122,19 +122,19 @@ public void testNullPartition() throws Exception { null, null, k1, v1, StringData.fromString(k2), StringData.fromString(v2)); expected.setRecord(rowData); - Map fields = new HashMap<>(); - fields.put("pt1", null); - fields.put("pt2", null); - fields.put("k1", String.valueOf(k1)); - fields.put("v1", String.valueOf(v1)); - fields.put("k2", k2); - fields.put("v2", v2); + Map data = new HashMap<>(); + data.put("pt1", null); + data.put("pt2", null); + data.put("k1", String.valueOf(k1)); + data.put("v1", String.valueOf(v1)); + data.put("k2", k2); + data.put("v2", v2); - actual.setRecord(new CdcRecord(RowKind.INSERT, fields)); + actual.setRecord(new CdcRecord(RowKind.INSERT, data)); assertThat(actual.partition()).isEqualTo(expected.partition()); assertThat(actual.bucket()).isEqualTo(expected.bucket()); - actual.setRecord(new CdcRecord(RowKind.DELETE, fields)); + actual.setRecord(new CdcRecord(RowKind.DELETE, data)); assertThat(actual.partition()).isEqualTo(expected.partition()); assertThat(actual.bucket()).isEqualTo(expected.bucket()); } @@ -161,19 +161,19 @@ public void testEmptyPartition() throws Exception { StringData.fromString(v2)); expected.setRecord(rowData); - Map fields = new HashMap<>(); - fields.put("pt1", ""); - fields.put("pt2", null); - fields.put("k1", String.valueOf(k1)); - fields.put("v1", String.valueOf(v1)); - fields.put("k2", k2); - fields.put("v2", v2); + Map data = new HashMap<>(); + data.put("pt1", ""); + data.put("pt2", null); + data.put("k1", String.valueOf(k1)); + data.put("v1", String.valueOf(v1)); + data.put("k2", k2); + data.put("v2", v2); - actual.setRecord(new CdcRecord(RowKind.INSERT, fields)); + actual.setRecord(new CdcRecord(RowKind.INSERT, data)); assertThat(actual.partition()).isEqualTo(expected.partition()); assertThat(actual.bucket()).isEqualTo(expected.bucket()); - actual.setRecord(new CdcRecord(RowKind.DELETE, fields)); + actual.setRecord(new CdcRecord(RowKind.DELETE, data)); assertThat(actual.partition()).isEqualTo(expected.partition()); assertThat(actual.bucket()).isEqualTo(expected.bucket()); } diff --git a/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/sink/cdc/CdcRecordSerializeITCase.java b/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/sink/cdc/CdcRecordSerializeITCase.java index 698900436e8d..b202ca53c9cc 100644 --- a/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/sink/cdc/CdcRecordSerializeITCase.java +++ b/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/sink/cdc/CdcRecordSerializeITCase.java @@ -25,6 +25,8 @@ import org.apache.paimon.types.VarCharType; import org.apache.flink.api.common.ExecutionConfig; +import org.apache.flink.api.common.serialization.SerializerConfig; +import org.apache.flink.api.common.serialization.SerializerConfigImpl; import org.apache.flink.api.java.typeutils.runtime.kryo.KryoSerializer; import org.apache.flink.core.memory.DataInputView; import org.apache.flink.core.memory.DataOutputView; @@ -35,6 +37,8 @@ import java.io.DataInputStream; import java.io.DataOutputStream; import java.io.IOException; +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -49,7 +53,7 @@ public class CdcRecordSerializeITCase { @Test - public void testCdcRecordKryoSerialize() throws IOException { + public void testCdcRecordKryoSerialize() throws Exception { KryoSerializer kr = createFlinkKryoSerializer(RichCdcMultiplexRecord.class); RowType.Builder rowType = RowType.builder(); @@ -78,7 +82,7 @@ public void testCdcRecordKryoSerialize() throws IOException { } @Test - public void testUnmodifiableListKryoSerialize() throws IOException { + public void testUnmodifiableListKryoSerialize() throws Exception { KryoSerializer kryoSerializer = createFlinkKryoSerializer(List.class); RowType.Builder rowType = RowType.builder(); rowType.field("id", new BigIntType()); @@ -101,8 +105,24 @@ public void testUnmodifiableListKryoSerialize() throws IOException { assertThat(deserializeRecord).isEqualTo(fields); } - public static KryoSerializer createFlinkKryoSerializer(Class type) { - return new KryoSerializer<>(type, new ExecutionConfig()); + @SuppressWarnings({"unchecked", "rawtypes"}) + public static KryoSerializer createFlinkKryoSerializer(Class type) + throws NoSuchMethodException, InvocationTargetException, InstantiationException, + IllegalAccessException { + try { + Constructor constructor = + KryoSerializer.class.getConstructor(Class.class, SerializerConfig.class); + return (KryoSerializer) constructor.newInstance(type, new SerializerConfigImpl()); + } catch (NoSuchMethodException + | InvocationTargetException + | IllegalAccessException + | InstantiationException e) { + // to stay compatible with Flink 1.18- + } + + Constructor constructor = + KryoSerializer.class.getConstructor(Class.class, ExecutionConfig.class); + return (KryoSerializer) constructor.newInstance(type, new ExecutionConfig()); } private static final class TestOutputView extends DataOutputStream implements DataOutputView { diff --git a/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/sink/cdc/CdcRecordStoreMultiWriteOperatorTest.java b/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/sink/cdc/CdcRecordStoreMultiWriteOperatorTest.java index 2a1bb4004306..9f35b25026bb 100644 --- a/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/sink/cdc/CdcRecordStoreMultiWriteOperatorTest.java +++ b/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/sink/cdc/CdcRecordStoreMultiWriteOperatorTest.java @@ -172,16 +172,14 @@ public void testAsyncTableCreate() throws Exception { t.start(); // check that records should be processed after table is created - Map fields = new HashMap<>(); - fields.put("pt", "0"); - fields.put("k", "1"); - fields.put("v", "10"); + Map data = new HashMap<>(); + data.put("pt", "0"); + data.put("k", "1"); + data.put("v", "10"); CdcMultiplexRecord expected = CdcMultiplexRecord.fromCdcRecord( - databaseName, - tableId.getObjectName(), - new CdcRecord(RowKind.INSERT, fields)); + databaseName, tableId.getObjectName(), new CdcRecord(RowKind.INSERT, data)); runner.offer(expected); CdcMultiplexRecord actual = runner.poll(1); @@ -192,15 +190,13 @@ public void testAsyncTableCreate() throws Exception { assertThat(actual).isEqualTo(expected); // after table is created, record should be processed immediately - fields = new HashMap<>(); - fields.put("pt", "0"); - fields.put("k", "3"); - fields.put("v", "30"); + data = new HashMap<>(); + data.put("pt", "0"); + data.put("k", "3"); + data.put("v", "30"); expected = CdcMultiplexRecord.fromCdcRecord( - databaseName, - tableId.getObjectName(), - new CdcRecord(RowKind.INSERT, fields)); + databaseName, tableId.getObjectName(), new CdcRecord(RowKind.INSERT, data)); runner.offer(expected); actual = runner.take(); assertThat(actual).isEqualTo(expected); @@ -227,16 +223,14 @@ public void testInitializeState() throws Exception { t.start(); // check that records should be processed after table is created - Map fields = new HashMap<>(); - fields.put("pt", "0"); - fields.put("k", "1"); - fields.put("v", "10"); + Map data = new HashMap<>(); + data.put("pt", "0"); + data.put("k", "1"); + data.put("v", "10"); CdcMultiplexRecord expected = CdcMultiplexRecord.fromCdcRecord( - databaseName, - tableId.getObjectName(), - new CdcRecord(RowKind.INSERT, fields)); + databaseName, tableId.getObjectName(), new CdcRecord(RowKind.INSERT, data)); runner.offer(expected); CdcMultiplexRecord actual = runner.poll(1); @@ -254,15 +248,13 @@ public void testInitializeState() throws Exception { assertThat(operator.writes().size()).isEqualTo(1); // after table is created, record should be processed immediately - fields = new HashMap<>(); - fields.put("pt", "0"); - fields.put("k", "3"); - fields.put("v", "30"); + data = new HashMap<>(); + data.put("pt", "0"); + data.put("k", "3"); + data.put("v", "30"); expected = CdcMultiplexRecord.fromCdcRecord( - databaseName, - tableId.getObjectName(), - new CdcRecord(RowKind.INSERT, fields)); + databaseName, tableId.getObjectName(), new CdcRecord(RowKind.INSERT, data)); runner.offer(expected); actual = runner.take(); assertThat(actual).isEqualTo(expected); @@ -302,44 +294,38 @@ public void testSingleTableAddColumn() throws Exception { // check that records with compatible schema can be processed immediately - Map fields = new HashMap<>(); - fields.put("pt", "0"); - fields.put("k", "1"); - fields.put("v", "10"); + Map data = new HashMap<>(); + data.put("pt", "0"); + data.put("k", "1"); + data.put("v", "10"); CdcMultiplexRecord expected = CdcMultiplexRecord.fromCdcRecord( - databaseName, - tableId.getObjectName(), - new CdcRecord(RowKind.INSERT, fields)); + databaseName, tableId.getObjectName(), new CdcRecord(RowKind.INSERT, data)); runner.offer(expected); CdcMultiplexRecord actual = runner.take(); assertThat(actual).isEqualTo(expected); - fields = new HashMap<>(); - fields.put("pt", "0"); - fields.put("k", "2"); + data = new HashMap<>(); + data.put("pt", "0"); + data.put("k", "2"); expected = CdcMultiplexRecord.fromCdcRecord( - databaseName, - tableId.getObjectName(), - new CdcRecord(RowKind.INSERT, fields)); + databaseName, tableId.getObjectName(), new CdcRecord(RowKind.INSERT, data)); runner.offer(expected); actual = runner.take(); assertThat(actual).isEqualTo(expected); - // check that records with new fields should be processed after schema is updated + // check that records with new data should be processed after schema is updated - fields = new HashMap<>(); - fields.put("pt", "0"); - fields.put("k", "3"); - fields.put("v", "30"); - fields.put("v2", "300"); + data = new HashMap<>(); + data.put("pt", "0"); + data.put("k", "3"); + data.put("v", "30"); + data.put("v2", "300"); expected = CdcMultiplexRecord.fromCdcRecord( - databaseName, - tableId.getObjectName(), - new CdcRecord(RowKind.INSERT, fields)); + databaseName, tableId.getObjectName(), new CdcRecord(RowKind.INSERT, data)); runner.offer(expected); actual = runner.poll(1); assertThat(actual).isNull(); @@ -383,34 +369,30 @@ public void testSingleTableUpdateColumnType() throws Exception { // check that records with compatible schema can be processed immediately - Map fields = new HashMap<>(); - fields.put("k", "1"); - fields.put("v1", "10"); - fields.put("v2", "0.625"); - fields.put("v3", "one"); - fields.put("v4", "b_one"); + Map data = new HashMap<>(); + data.put("k", "1"); + data.put("v1", "10"); + data.put("v2", "0.625"); + data.put("v3", "one"); + data.put("v4", "b_one"); CdcMultiplexRecord expected = CdcMultiplexRecord.fromCdcRecord( - databaseName, - tableId.getObjectName(), - new CdcRecord(RowKind.INSERT, fields)); + databaseName, tableId.getObjectName(), new CdcRecord(RowKind.INSERT, data)); runner.offer(expected); CdcMultiplexRecord actual = runner.take(); assertThat(actual).isEqualTo(expected); - // check that records with new fields should be processed after schema is updated + // check that records with new data should be processed after schema is updated // int -> bigint - fields = new HashMap<>(); - fields.put("k", "2"); - fields.put("v1", "12345678987654321"); - fields.put("v2", "0.25"); + data = new HashMap<>(); + data.put("k", "2"); + data.put("v1", "12345678987654321"); + data.put("v2", "0.25"); expected = CdcMultiplexRecord.fromCdcRecord( - databaseName, - tableId.getObjectName(), - new CdcRecord(RowKind.INSERT, fields)); + databaseName, tableId.getObjectName(), new CdcRecord(RowKind.INSERT, data)); runner.offer(expected); actual = runner.poll(1); assertThat(actual).isNull(); @@ -422,15 +404,13 @@ public void testSingleTableUpdateColumnType() throws Exception { // float -> double - fields = new HashMap<>(); - fields.put("k", "3"); - fields.put("v1", "100"); - fields.put("v2", "1.0000000000009095"); + data = new HashMap<>(); + data.put("k", "3"); + data.put("v1", "100"); + data.put("v2", "1.0000000000009095"); expected = CdcMultiplexRecord.fromCdcRecord( - databaseName, - tableId.getObjectName(), - new CdcRecord(RowKind.INSERT, fields)); + databaseName, tableId.getObjectName(), new CdcRecord(RowKind.INSERT, data)); runner.offer(expected); actual = runner.poll(1); assertThat(actual).isNull(); @@ -441,15 +421,13 @@ public void testSingleTableUpdateColumnType() throws Exception { // varchar(5) -> varchar(10) - fields = new HashMap<>(); - fields.put("k", "4"); - fields.put("v1", "40"); - fields.put("v3", "long four"); + data = new HashMap<>(); + data.put("k", "4"); + data.put("v1", "40"); + data.put("v3", "long four"); expected = CdcMultiplexRecord.fromCdcRecord( - databaseName, - tableId.getObjectName(), - new CdcRecord(RowKind.INSERT, fields)); + databaseName, tableId.getObjectName(), new CdcRecord(RowKind.INSERT, data)); runner.offer(expected); actual = runner.poll(1); assertThat(actual).isNull(); @@ -460,15 +438,13 @@ public void testSingleTableUpdateColumnType() throws Exception { // varbinary(5) -> varbinary(10) - fields = new HashMap<>(); - fields.put("k", "5"); - fields.put("v1", "50"); - fields.put("v4", "long five~"); + data = new HashMap<>(); + data.put("k", "5"); + data.put("v1", "50"); + data.put("v4", "long five~"); expected = CdcMultiplexRecord.fromCdcRecord( - databaseName, - tableId.getObjectName(), - new CdcRecord(RowKind.INSERT, fields)); + databaseName, tableId.getObjectName(), new CdcRecord(RowKind.INSERT, data)); runner.offer(expected); actual = runner.poll(1); assertThat(actual).isNull(); @@ -499,53 +475,53 @@ public void testMultiTableUpdateColumnType() throws Exception { // check that records with compatible schema from different tables // can be processed immediately - Map fields; + Map data; // first table record - fields = new HashMap<>(); - fields.put("pt", "0"); - fields.put("k", "1"); - fields.put("v", "10"); + data = new HashMap<>(); + data.put("pt", "0"); + data.put("k", "1"); + data.put("v", "10"); CdcMultiplexRecord expected = CdcMultiplexRecord.fromCdcRecord( databaseName, firstTable.getObjectName(), - new CdcRecord(RowKind.INSERT, fields)); + new CdcRecord(RowKind.INSERT, data)); runner.offer(expected); CdcMultiplexRecord actual = runner.take(); assertThat(actual).isEqualTo(expected); // second table record - fields = new HashMap<>(); - fields.put("k", "1"); - fields.put("v1", "10"); - fields.put("v2", "0.625"); - fields.put("v3", "one"); - fields.put("v4", "b_one"); + data = new HashMap<>(); + data.put("k", "1"); + data.put("v1", "10"); + data.put("v2", "0.625"); + data.put("v3", "one"); + data.put("v4", "b_one"); expected = CdcMultiplexRecord.fromCdcRecord( databaseName, secondTable.getObjectName(), - new CdcRecord(RowKind.INSERT, fields)); + new CdcRecord(RowKind.INSERT, data)); runner.offer(expected); actual = runner.take(); assertThat(actual).isEqualTo(expected); - // check that records with new fields should be processed after schema is updated + // check that records with new data should be processed after schema is updated // int -> bigint SchemaManager schemaManager; // first table - fields = new HashMap<>(); - fields.put("pt", "1"); - fields.put("k", "123456789876543211"); - fields.put("v", "varchar"); + data = new HashMap<>(); + data.put("pt", "1"); + data.put("k", "123456789876543211"); + data.put("v", "varchar"); expected = CdcMultiplexRecord.fromCdcRecord( databaseName, firstTable.getObjectName(), - new CdcRecord(RowKind.INSERT, fields)); + new CdcRecord(RowKind.INSERT, data)); runner.offer(expected); actual = runner.poll(1); assertThat(actual).isNull(); @@ -556,15 +532,15 @@ public void testMultiTableUpdateColumnType() throws Exception { assertThat(actual).isEqualTo(expected); // second table - fields = new HashMap<>(); - fields.put("k", "2"); - fields.put("v1", "12345678987654321"); - fields.put("v2", "0.25"); + data = new HashMap<>(); + data.put("k", "2"); + data.put("v1", "12345678987654321"); + data.put("v2", "0.25"); expected = CdcMultiplexRecord.fromCdcRecord( databaseName, secondTable.getObjectName(), - new CdcRecord(RowKind.INSERT, fields)); + new CdcRecord(RowKind.INSERT, data)); runner.offer(expected); actual = runner.poll(1); assertThat(actual).isNull(); @@ -577,15 +553,15 @@ public void testMultiTableUpdateColumnType() throws Exception { // below are schema changes only from the second table // float -> double - fields = new HashMap<>(); - fields.put("k", "3"); - fields.put("v1", "100"); - fields.put("v2", "1.0000000000009095"); + data = new HashMap<>(); + data.put("k", "3"); + data.put("v1", "100"); + data.put("v2", "1.0000000000009095"); expected = CdcMultiplexRecord.fromCdcRecord( databaseName, secondTable.getObjectName(), - new CdcRecord(RowKind.INSERT, fields)); + new CdcRecord(RowKind.INSERT, data)); runner.offer(expected); actual = runner.poll(1); assertThat(actual).isNull(); @@ -597,15 +573,15 @@ public void testMultiTableUpdateColumnType() throws Exception { // varchar(5) -> varchar(10) - fields = new HashMap<>(); - fields.put("k", "4"); - fields.put("v1", "40"); - fields.put("v3", "long four"); + data = new HashMap<>(); + data.put("k", "4"); + data.put("v1", "40"); + data.put("v3", "long four"); expected = CdcMultiplexRecord.fromCdcRecord( databaseName, secondTable.getObjectName(), - new CdcRecord(RowKind.INSERT, fields)); + new CdcRecord(RowKind.INSERT, data)); runner.offer(expected); actual = runner.poll(1); assertThat(actual).isNull(); @@ -617,15 +593,15 @@ public void testMultiTableUpdateColumnType() throws Exception { // varbinary(5) -> varbinary(10) - fields = new HashMap<>(); - fields.put("k", "5"); - fields.put("v1", "50"); - fields.put("v4", "long five~"); + data = new HashMap<>(); + data.put("k", "5"); + data.put("v1", "50"); + data.put("v4", "long five~"); expected = CdcMultiplexRecord.fromCdcRecord( databaseName, secondTable.getObjectName(), - new CdcRecord(RowKind.INSERT, fields)); + new CdcRecord(RowKind.INSERT, data)); runner.offer(expected); actual = runner.poll(1); assertThat(actual).isNull(); @@ -651,33 +627,33 @@ public void testUsingTheSameCompactExecutor() throws Exception { t.start(); // write records to two tables thus two FileStoreWrite will be created - Map fields; + Map data; // first table record - fields = new HashMap<>(); - fields.put("pt", "0"); - fields.put("k", "1"); - fields.put("v", "10"); + data = new HashMap<>(); + data.put("pt", "0"); + data.put("k", "1"); + data.put("v", "10"); CdcMultiplexRecord expected = CdcMultiplexRecord.fromCdcRecord( databaseName, firstTable.getObjectName(), - new CdcRecord(RowKind.INSERT, fields)); + new CdcRecord(RowKind.INSERT, data)); runner.offer(expected); // second table record - fields = new HashMap<>(); - fields.put("k", "1"); - fields.put("v1", "10"); - fields.put("v2", "0.625"); - fields.put("v3", "one"); - fields.put("v4", "b_one"); + data = new HashMap<>(); + data.put("k", "1"); + data.put("v1", "10"); + data.put("v2", "0.625"); + data.put("v3", "one"); + data.put("v4", "b_one"); expected = CdcMultiplexRecord.fromCdcRecord( databaseName, secondTable.getObjectName(), - new CdcRecord(RowKind.INSERT, fields)); + new CdcRecord(RowKind.INSERT, data)); runner.offer(expected); // get and check compactExecutor from two FileStoreWrite @@ -713,8 +689,8 @@ public void testUsingTheSameCompactExecutor() throws Exception { private OneInputStreamOperatorTestHarness createTestHarness(Catalog.Loader catalogLoader) throws Exception { - CdcRecordStoreMultiWriteOperator operator = - new CdcRecordStoreMultiWriteOperator( + CdcRecordStoreMultiWriteOperator.Factory operatorFactory = + new CdcRecordStoreMultiWriteOperator.Factory( catalogLoader, (t, commitUser, state, ioManager, memoryPoolFactory, metricGroup) -> new StoreSinkWriteImpl( @@ -733,7 +709,7 @@ public void testUsingTheSameCompactExecutor() throws Exception { TypeSerializer outputSerializer = new MultiTableCommittableTypeInfo().createSerializer(new ExecutionConfig()); OneInputStreamOperatorTestHarness harness = - new OneInputStreamOperatorTestHarness<>(operator, inputSerializer); + new OneInputStreamOperatorTestHarness<>(operatorFactory, inputSerializer); harness.setup(outputSerializer); return harness; } diff --git a/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/sink/cdc/CdcRecordStoreWriteOperatorTest.java b/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/sink/cdc/CdcRecordStoreWriteOperatorTest.java index 9af7eabdaaad..f00229d99890 100644 --- a/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/sink/cdc/CdcRecordStoreWriteOperatorTest.java +++ b/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/sink/cdc/CdcRecordStoreWriteOperatorTest.java @@ -106,31 +106,31 @@ public void testAddColumn() throws Exception { // check that records with compatible schema can be processed immediately - Map fields = new HashMap<>(); - fields.put("pt", "0"); - fields.put("k", "1"); - fields.put("v", "10"); - CdcRecord expected = new CdcRecord(RowKind.INSERT, fields); + Map data = new HashMap<>(); + data.put("pt", "0"); + data.put("k", "1"); + data.put("v", "10"); + CdcRecord expected = new CdcRecord(RowKind.INSERT, data); runner.offer(expected); CdcRecord actual = runner.take(); assertThat(actual).isEqualTo(expected); - fields = new HashMap<>(); - fields.put("pt", "0"); - fields.put("k", "2"); - expected = new CdcRecord(RowKind.INSERT, fields); + data = new HashMap<>(); + data.put("pt", "0"); + data.put("k", "2"); + expected = new CdcRecord(RowKind.INSERT, data); runner.offer(expected); actual = runner.take(); assertThat(actual).isEqualTo(expected); - // check that records with new fields should be processed after schema is updated + // check that records with new data should be processed after schema is updated - fields = new HashMap<>(); - fields.put("pt", "0"); - fields.put("k", "3"); - fields.put("v", "30"); - fields.put("v2", "300"); - expected = new CdcRecord(RowKind.INSERT, fields); + data = new HashMap<>(); + data.put("pt", "0"); + data.put("k", "3"); + data.put("v", "30"); + data.put("v2", "300"); + expected = new CdcRecord(RowKind.INSERT, data); runner.offer(expected); actual = runner.poll(1); assertThat(actual).isNull(); @@ -172,26 +172,26 @@ public void testUpdateColumnType() throws Exception { // check that records with compatible schema can be processed immediately - Map fields = new HashMap<>(); - fields.put("k", "1"); - fields.put("v1", "10"); - fields.put("v2", "0.625"); - fields.put("v3", "one"); - fields.put("v4", "b_one"); - CdcRecord expected = new CdcRecord(RowKind.INSERT, fields); + Map data = new HashMap<>(); + data.put("k", "1"); + data.put("v1", "10"); + data.put("v2", "0.625"); + data.put("v3", "one"); + data.put("v4", "b_one"); + CdcRecord expected = new CdcRecord(RowKind.INSERT, data); runner.offer(expected); CdcRecord actual = runner.take(); assertThat(actual).isEqualTo(expected); - // check that records with new fields should be processed after schema is updated + // check that records with new data should be processed after schema is updated // int -> bigint - fields = new HashMap<>(); - fields.put("k", "2"); - fields.put("v1", "12345678987654321"); - fields.put("v2", "0.25"); - expected = new CdcRecord(RowKind.INSERT, fields); + data = new HashMap<>(); + data.put("k", "2"); + data.put("v1", "12345678987654321"); + data.put("v2", "0.25"); + expected = new CdcRecord(RowKind.INSERT, data); runner.offer(expected); actual = runner.poll(1); assertThat(actual).isNull(); @@ -203,11 +203,11 @@ public void testUpdateColumnType() throws Exception { // float -> double - fields = new HashMap<>(); - fields.put("k", "3"); - fields.put("v1", "100"); - fields.put("v2", "1.0000000000009095"); - expected = new CdcRecord(RowKind.INSERT, fields); + data = new HashMap<>(); + data.put("k", "3"); + data.put("v1", "100"); + data.put("v2", "1.0000000000009095"); + expected = new CdcRecord(RowKind.INSERT, data); runner.offer(expected); actual = runner.poll(1); assertThat(actual).isNull(); @@ -218,11 +218,11 @@ public void testUpdateColumnType() throws Exception { // varchar(5) -> varchar(10) - fields = new HashMap<>(); - fields.put("k", "4"); - fields.put("v1", "40"); - fields.put("v3", "long four"); - expected = new CdcRecord(RowKind.INSERT, fields); + data = new HashMap<>(); + data.put("k", "4"); + data.put("v1", "40"); + data.put("v3", "long four"); + expected = new CdcRecord(RowKind.INSERT, data); runner.offer(expected); actual = runner.poll(1); assertThat(actual).isNull(); @@ -233,11 +233,11 @@ public void testUpdateColumnType() throws Exception { // varbinary(5) -> varbinary(10) - fields = new HashMap<>(); - fields.put("k", "5"); - fields.put("v1", "50"); - fields.put("v4", "long five~"); - expected = new CdcRecord(RowKind.INSERT, fields); + data = new HashMap<>(); + data.put("k", "5"); + data.put("v1", "50"); + data.put("v4", "long five~"); + expected = new CdcRecord(RowKind.INSERT, data); runner.offer(expected); actual = runner.poll(1); assertThat(actual).isNull(); @@ -253,8 +253,8 @@ public void testUpdateColumnType() throws Exception { private OneInputStreamOperatorTestHarness createTestHarness( FileStoreTable table) throws Exception { - CdcRecordStoreWriteOperator operator = - new CdcRecordStoreWriteOperator( + CdcRecordStoreWriteOperator.Factory operatorFactory = + new CdcRecordStoreWriteOperator.Factory( table, (t, commitUser, state, ioManager, memoryPool, metricGroup) -> new StoreSinkWriteImpl( @@ -272,7 +272,7 @@ private OneInputStreamOperatorTestHarness createTestHarn TypeSerializer outputSerializer = new CommittableTypeInfo().createSerializer(new ExecutionConfig()); OneInputStreamOperatorTestHarness harness = - new OneInputStreamOperatorTestHarness<>(operator, inputSerializer); + new OneInputStreamOperatorTestHarness<>(operatorFactory, inputSerializer); harness.setup(outputSerializer); return harness; } diff --git a/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/sink/cdc/FlinkCdcMultiTableSinkTest.java b/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/sink/cdc/FlinkCdcMultiTableSinkTest.java index fd23e500d5e5..ab81e37c7d04 100644 --- a/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/sink/cdc/FlinkCdcMultiTableSinkTest.java +++ b/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/sink/cdc/FlinkCdcMultiTableSinkTest.java @@ -22,11 +22,10 @@ import org.apache.paimon.flink.FlinkConnectorOptions; import org.apache.paimon.options.Options; +import org.apache.flink.api.dag.Transformation; import org.apache.flink.streaming.api.datastream.DataStreamSink; import org.apache.flink.streaming.api.datastream.DataStreamSource; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; -import org.apache.flink.streaming.api.functions.source.ParallelSourceFunction; -import org.apache.flink.streaming.api.transformations.LegacySinkTransformation; import org.apache.flink.streaming.api.transformations.OneInputTransformation; import org.apache.flink.streaming.api.transformations.PartitionTransformation; import org.junit.jupiter.api.Test; @@ -45,14 +44,7 @@ public void testTransformationParallelism() { env.setParallelism(8); int inputParallelism = ThreadLocalRandom.current().nextInt(8) + 1; DataStreamSource input = - env.addSource( - new ParallelSourceFunction() { - @Override - public void run(SourceContext ctx) {} - - @Override - public void cancel() {} - }) + env.fromData(CdcMultiplexRecord.class, new CdcMultiplexRecord("", "", null)) .setParallelism(inputParallelism); FlinkCdcMultiTableSink sink = @@ -60,13 +52,11 @@ public void cancel() {} () -> FlinkCatalogFactory.createPaimonCatalog(new Options()), FlinkConnectorOptions.SINK_COMMITTER_CPU.defaultValue(), null, - true, UUID.randomUUID().toString()); DataStreamSink dataStreamSink = sink.sinkFrom(input); // check the transformation graph - LegacySinkTransformation end = - (LegacySinkTransformation) dataStreamSink.getTransformation(); + Transformation end = dataStreamSink.getTransformation(); assertThat(end.getName()).isEqualTo("end"); OneInputTransformation committer = diff --git a/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/sink/cdc/FlinkCdcSyncDatabaseSinkITCase.java b/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/sink/cdc/FlinkCdcSyncDatabaseSinkITCase.java index a7c6b2cb6323..28b137a93ed9 100644 --- a/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/sink/cdc/FlinkCdcSyncDatabaseSinkITCase.java +++ b/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/sink/cdc/FlinkCdcSyncDatabaseSinkITCase.java @@ -42,6 +42,7 @@ import org.apache.paimon.utils.FailingFileIO; import org.apache.paimon.utils.TraceableFileIO; +import org.apache.flink.api.common.eventtime.WatermarkStrategy; import org.apache.flink.streaming.api.datastream.DataStreamSource; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.junit.jupiter.api.Test; @@ -154,8 +155,9 @@ private void innerTestRandomCdcEvents(Supplier bucket, boolean unawareB .allowRestart(enableFailure) .build(); - TestCdcSourceFunction sourceFunction = new TestCdcSourceFunction(events); - DataStreamSource source = env.addSource(sourceFunction); + TestCdcSource testCdcSource = new TestCdcSource(events); + DataStreamSource source = + env.fromSource(testCdcSource, WatermarkStrategy.noWatermarks(), "TestCdcSource"); source.setParallelism(2); Options catalogOptions = new Options(); diff --git a/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/sink/cdc/FlinkCdcSyncTableSinkITCase.java b/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/sink/cdc/FlinkCdcSyncTableSinkITCase.java index 081bd7d073d7..8b19391f3eda 100644 --- a/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/sink/cdc/FlinkCdcSyncTableSinkITCase.java +++ b/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/sink/cdc/FlinkCdcSyncTableSinkITCase.java @@ -43,6 +43,7 @@ import org.apache.paimon.utils.FailingFileIO; import org.apache.paimon.utils.TraceableFileIO; +import org.apache.flink.api.common.eventtime.WatermarkStrategy; import org.apache.flink.streaming.api.datastream.DataStreamSource; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.junit.jupiter.api.Disabled; @@ -151,8 +152,9 @@ private void innerTestRandomCdcEvents( .allowRestart(enableFailure) .build(); - TestCdcSourceFunction sourceFunction = new TestCdcSourceFunction(testTable.events()); - DataStreamSource source = env.addSource(sourceFunction); + TestCdcSource testCdcSource = new TestCdcSource(testTable.events()); + DataStreamSource source = + env.fromSource(testCdcSource, WatermarkStrategy.noWatermarks(), "TestCdcSource"); source.setParallelism(2); Options catalogOptions = new Options(); diff --git a/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/sink/cdc/TestCdcSource.java b/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/sink/cdc/TestCdcSource.java new file mode 100644 index 000000000000..b45983000a23 --- /dev/null +++ b/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/sink/cdc/TestCdcSource.java @@ -0,0 +1,126 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.flink.sink.cdc; + +import org.apache.paimon.flink.source.AbstractNonCoordinatedSource; +import org.apache.paimon.flink.source.AbstractNonCoordinatedSourceReader; +import org.apache.paimon.flink.source.SimpleSourceSplit; +import org.apache.paimon.flink.source.SplitListState; + +import org.apache.flink.api.connector.source.Boundedness; +import org.apache.flink.api.connector.source.ReaderOutput; +import org.apache.flink.api.connector.source.SourceReader; +import org.apache.flink.api.connector.source.SourceReaderContext; +import org.apache.flink.core.io.InputStatus; + +import java.util.Collection; +import java.util.LinkedList; +import java.util.List; +import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * Testing parallel {@link org.apache.flink.api.connector.source.Source} to produce {@link + * TestCdcEvent}. {@link TestCdcEvent}s with the same key will be produced by the same parallelism. + */ +public class TestCdcSource extends AbstractNonCoordinatedSource { + + private static final long serialVersionUID = 1L; + private final LinkedList events; + + public TestCdcSource(Collection events) { + this.events = new LinkedList<>(events); + } + + @Override + public Boundedness getBoundedness() { + return Boundedness.CONTINUOUS_UNBOUNDED; + } + + @Override + public SourceReader createReader(SourceReaderContext context) { + return new Reader( + context.getIndexOfSubtask(), + context.currentParallelism(), + new LinkedList<>(events)); + } + + private static class Reader extends AbstractNonCoordinatedSourceReader { + private final int subtaskId; + private final int totalSubtasks; + + private final LinkedList events; + private final SplitListState remainingEventsCount = + new SplitListState<>("events", x -> Integer.toString(x), Integer::parseInt); + + private final int numRecordsPerCheckpoint; + private final AtomicInteger recordsThisCheckpoint; + + private Reader(int subtaskId, int totalSubtasks, LinkedList events) { + this.subtaskId = subtaskId; + this.totalSubtasks = totalSubtasks; + this.events = events; + numRecordsPerCheckpoint = + events.size() / ThreadLocalRandom.current().nextInt(10, 20) + 1; + recordsThisCheckpoint = new AtomicInteger(0); + } + + @Override + public InputStatus pollNext(ReaderOutput readerOutput) throws Exception { + if (events.isEmpty()) { + return InputStatus.END_OF_INPUT; + } + + if (recordsThisCheckpoint.get() >= numRecordsPerCheckpoint) { + Thread.sleep(10); + return InputStatus.MORE_AVAILABLE; + } + + TestCdcEvent event = events.poll(); + if (event.records() != null) { + if (Math.abs(event.hashCode()) % totalSubtasks != subtaskId) { + return InputStatus.MORE_AVAILABLE; + } + } + readerOutput.collect(event); + recordsThisCheckpoint.incrementAndGet(); + return InputStatus.MORE_AVAILABLE; + } + + @Override + public List snapshotState(long l) { + recordsThisCheckpoint.set(0); + remainingEventsCount.clear(); + remainingEventsCount.add(events.size()); + return remainingEventsCount.snapshotState(); + } + + @Override + public void addSplits(List list) { + remainingEventsCount.restoreState(list); + int count = 0; + for (int c : remainingEventsCount.get()) { + count += c; + } + while (events.size() > count) { + events.poll(); + } + } + } +} diff --git a/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/sink/cdc/TestCdcSourceFunction.java b/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/sink/cdc/TestCdcSourceFunction.java deleted file mode 100644 index 4e03256a5253..000000000000 --- a/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/sink/cdc/TestCdcSourceFunction.java +++ /dev/null @@ -1,107 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.paimon.flink.sink.cdc; - -import org.apache.flink.api.common.state.ListState; -import org.apache.flink.api.common.state.ListStateDescriptor; -import org.apache.flink.runtime.state.FunctionInitializationContext; -import org.apache.flink.runtime.state.FunctionSnapshotContext; -import org.apache.flink.streaming.api.checkpoint.CheckpointedFunction; -import org.apache.flink.streaming.api.functions.source.RichParallelSourceFunction; - -import java.util.Collection; -import java.util.LinkedList; -import java.util.concurrent.ThreadLocalRandom; -import java.util.concurrent.atomic.AtomicInteger; - -/** - * Testing {@link RichParallelSourceFunction} to produce {@link TestCdcEvent}. {@link TestCdcEvent}s - * with the same key will be produced by the same parallelism. - */ -public class TestCdcSourceFunction extends RichParallelSourceFunction - implements CheckpointedFunction { - - private static final long serialVersionUID = 1L; - - private final LinkedList events; - - private volatile boolean isRunning = true; - private transient int numRecordsPerCheckpoint; - private transient AtomicInteger recordsThisCheckpoint; - private transient ListState remainingEventsCount; - - public TestCdcSourceFunction(Collection events) { - this.events = new LinkedList<>(events); - } - - @Override - public void initializeState(FunctionInitializationContext context) throws Exception { - numRecordsPerCheckpoint = events.size() / ThreadLocalRandom.current().nextInt(10, 20) + 1; - recordsThisCheckpoint = new AtomicInteger(0); - - remainingEventsCount = - context.getOperatorStateStore() - .getListState(new ListStateDescriptor<>("count", Integer.class)); - - if (context.isRestored()) { - int count = 0; - for (int c : remainingEventsCount.get()) { - count += c; - } - while (events.size() > count) { - events.poll(); - } - } - } - - @Override - public void snapshotState(FunctionSnapshotContext context) throws Exception { - recordsThisCheckpoint.set(0); - remainingEventsCount.clear(); - remainingEventsCount.add(events.size()); - } - - @Override - public void run(SourceContext ctx) throws Exception { - while (isRunning && !events.isEmpty()) { - if (recordsThisCheckpoint.get() >= numRecordsPerCheckpoint) { - Thread.sleep(10); - continue; - } - - synchronized (ctx.getCheckpointLock()) { - TestCdcEvent event = events.poll(); - if (event.records() != null) { - int subtaskId = getRuntimeContext().getIndexOfThisSubtask(); - int totalSubtasks = getRuntimeContext().getNumberOfParallelSubtasks(); - if (Math.abs(event.hashCode()) % totalSubtasks != subtaskId) { - continue; - } - } - ctx.collect(event); - recordsThisCheckpoint.incrementAndGet(); - } - } - } - - @Override - public void cancel() { - isRunning = false; - } -} diff --git a/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/sink/cdc/TestTable.java b/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/sink/cdc/TestTable.java index 525a05096942..6a38c1c2659d 100644 --- a/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/sink/cdc/TestTable.java +++ b/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/sink/cdc/TestTable.java @@ -114,18 +114,18 @@ public TestTable( } events.add(new TestCdcEvent(tableName, currentDataFieldList(fieldNames, isBigInt))); } else { - Map fields = new HashMap<>(); + Map data = new HashMap<>(); int key = random.nextInt(numKeys); - fields.put("k", String.valueOf(key)); + data.put("k", String.valueOf(key)); int pt = key % numPartitions; - fields.put("pt", String.valueOf(pt)); + data.put("pt", String.valueOf(pt)); for (int j = 0; j < fieldNames.size(); j++) { String fieldName = fieldNames.get(j); if (isBigInt.get(j)) { - fields.put(fieldName, String.valueOf(random.nextLong())); + data.put(fieldName, String.valueOf(random.nextLong())); } else { - fields.put(fieldName, String.valueOf(random.nextInt())); + data.put(fieldName, String.valueOf(random.nextInt())); } } @@ -140,8 +140,8 @@ public TestTable( shouldInsert = random.nextInt(5) > 0; } if (shouldInsert) { - records.add(new CdcRecord(RowKind.INSERT, fields)); - expected.put(key, fields); + records.add(new CdcRecord(RowKind.INSERT, data)); + expected.put(key, data); } } // Generate test data for append table @@ -149,8 +149,8 @@ public TestTable( if (expected.containsKey(key)) { records.add(new CdcRecord(RowKind.DELETE, expected.get(key))); } else { - records.add(new CdcRecord(RowKind.INSERT, fields)); - expected.put(key, fields); + records.add(new CdcRecord(RowKind.INSERT, data)); + expected.put(key, data); } } events.add(new TestCdcEvent(tableName, records, Objects.hash(tableName, key))); diff --git a/paimon-flink/paimon-flink-cdc/src/test/resources/kafka/aliyun/table/event/event-delete.txt b/paimon-flink/paimon-flink-cdc/src/test/resources/kafka/aliyun/table/event/event-delete.txt new file mode 100644 index 000000000000..ebae6608a755 --- /dev/null +++ b/paimon-flink/paimon-flink-cdc/src/test/resources/kafka/aliyun/table/event/event-delete.txt @@ -0,0 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +{"schema":{"dataColumn":[{"name":"id","type":"LONG"},{"name":"val","type":"DOUBLE"},{"name":"name","type":"STRING"},{"name":"create_time","type":"DATE"}],"primaryKey":["id"],"source":{"dbType":"MySQL","dbName":"bigdata_test","tableName":"sync_test_table"}},"payload":{"before":{"dataColumn":{"id":1,"val":"1.100000","name":"a","create_time":1731661114000}},"after":null,"sequenceId":"1731663842292000000","timestamp":{"eventTime":1731662085000,"systemTime":1731663848953,"checkpointTime":1731662085000},"op":"DELETE","ddl":null},"version":"0.0.1"} \ No newline at end of file diff --git a/paimon-flink/paimon-flink-cdc/src/test/resources/kafka/aliyun/table/event/event-insert.txt b/paimon-flink/paimon-flink-cdc/src/test/resources/kafka/aliyun/table/event/event-insert.txt new file mode 100644 index 000000000000..d1cd34e5e6ac --- /dev/null +++ b/paimon-flink/paimon-flink-cdc/src/test/resources/kafka/aliyun/table/event/event-insert.txt @@ -0,0 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +{"payload":{"after":{"dataColumn":{"create_time":1731661114000,"id":2,"name":"a","val":"1.100000"}},"before":null,"ddl":null,"op":"INSERT","sequenceId":"-1","timestamp":{"checkpointTime":-1,"eventTime":-1,"systemTime":1731661820245}},"schema":{"dataColumn":[{"name":"id","type":"LONG"},{"name":"val","type":"DOUBLE"},{"name":"name","type":"STRING"},{"name":"create_time","type":"DATE"}],"primaryKey":["id"],"source":{"dbName":"bigdata_test","dbType":"MySQL","tableName":"sync_test_table"}},"version":"0.0.1"} \ No newline at end of file diff --git a/paimon-flink/paimon-flink-cdc/src/test/resources/kafka/aliyun/table/event/event-update-in-one.txt b/paimon-flink/paimon-flink-cdc/src/test/resources/kafka/aliyun/table/event/event-update-in-one.txt new file mode 100644 index 000000000000..9acf6309cc48 --- /dev/null +++ b/paimon-flink/paimon-flink-cdc/src/test/resources/kafka/aliyun/table/event/event-update-in-one.txt @@ -0,0 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + {"schema":{"dataColumn":[{"name":"id","type":"LONG"},{"name":"val","type":"DOUBLE"},{"name":"name","type":"STRING"},{"name":"create_time","type":"DATE"}],"primaryKey":["id"],"source":{"dbType":"MySQL","dbName":"bigdata_test","tableName":"sync_test_table"}},"payload":{"before":{"dataColumn":{"id":2,"val":"1.100000","name":"a","create_time":1731661114000}},"after":{"dataColumn":{"id":2,"val":"2.200000","name":"a","create_time":1731661114000}},"sequenceId":"1731663842292000001","timestamp":{"eventTime":1731662097000,"systemTime":1731663848979,"checkpointTime":1731662097000},"op":"UPDATE_AFTER","ddl":null},"version":"0.0.1"} \ No newline at end of file diff --git a/paimon-flink/paimon-flink-cdc/src/test/resources/kafka/aws-dms/database/include/topic0/aws-dms-data-1.txt b/paimon-flink/paimon-flink-cdc/src/test/resources/kafka/aws-dms/database/include/topic0/aws-dms-data-1.txt new file mode 100644 index 000000000000..d779b9fb4ad5 --- /dev/null +++ b/paimon-flink/paimon-flink-cdc/src/test/resources/kafka/aws-dms/database/include/topic0/aws-dms-data-1.txt @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +{"data":{"id":101,"name":"scooter","description":"Small 2-wheel scooter","weight":3.14},"metadata":{"timestamp":"2024-10-27T12:39:03.210671Z","record-type":"data","operation":"insert","partition-key-type":"schema-table","schema-name":"paimon_sync_database_affix","table-name":"paimon_1","transaction-id":670014899490}} +{"data":{"id":102,"name":"car battery","description":"12V car battery","weight":8.1},"metadata":{"timestamp":"2024-10-27T12:39:03.210671Z","record-type":"data","operation":"insert","partition-key-type":"schema-table","schema-name":"paimon_sync_database_affix","table-name":"paimon_2","transaction-id":670014899490}} +{"data":{"id":103,"name":"12-pack drill bits","description":"12-pack of drill bits with sizes ranging from #40 to #3","weight":0.8},"metadata":{"timestamp":"2024-10-27T12:39:03.210671Z","record-type":"data","operation":"insert","partition-key-type":"schema-table","schema-name":"paimon_sync_database_affix","table-name":"ignore","transaction-id":670014899490}} +{"data":{"id":104,"name":"hammer","description":"12oz carpenter's hammer","weight":0.75},"metadata":{"timestamp":"2024-10-27T12:39:03.210671Z","record-type":"data","operation":"insert","partition-key-type":"schema-table","schema-name":"paimon_sync_database_affix","table-name":"flink","transaction-id":670014899490}} \ No newline at end of file diff --git a/paimon-flink/paimon-flink-cdc/src/test/resources/kafka/aws-dms/database/prefixsuffix/topic0/aws-dms-data-1.txt b/paimon-flink/paimon-flink-cdc/src/test/resources/kafka/aws-dms/database/prefixsuffix/topic0/aws-dms-data-1.txt new file mode 100644 index 000000000000..5ac7c5dbecef --- /dev/null +++ b/paimon-flink/paimon-flink-cdc/src/test/resources/kafka/aws-dms/database/prefixsuffix/topic0/aws-dms-data-1.txt @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +{"data":{"id":101,"name":"scooter","description":"Small 2-wheel scooter","weight":3.14},"metadata":{"timestamp":"2024-10-27T12:39:03.210671Z","record-type":"data","operation":"insert","partition-key-type":"schema-table","schema-name":"paimon_sync_database","table-name":"t1","transaction-id":670014899490}} +{"data":{"id":102,"name":"car battery","description":"12V car battery","weight":8.1},"metadata":{"timestamp":"2024-10-27T12:39:03.210671Z","record-type":"data","operation":"insert","partition-key-type":"schema-table","schema-name":"paimon_sync_database","table-name":"t1","transaction-id":670014899490}} \ No newline at end of file diff --git a/paimon-flink/paimon-flink-cdc/src/test/resources/kafka/aws-dms/database/prefixsuffix/topic0/aws-dms-data-2.txt b/paimon-flink/paimon-flink-cdc/src/test/resources/kafka/aws-dms/database/prefixsuffix/topic0/aws-dms-data-2.txt new file mode 100644 index 000000000000..56e1b53c1017 --- /dev/null +++ b/paimon-flink/paimon-flink-cdc/src/test/resources/kafka/aws-dms/database/prefixsuffix/topic0/aws-dms-data-2.txt @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +{"data":{"id":101,"name":"scooter","description":"Small 2-wheel scooter","weight":3.14,"address":"Beijing"},"metadata":{"timestamp":"2024-10-27T12:39:03.210671Z","record-type":"data","operation":"insert","partition-key-type":"schema-table","schema-name":"paimon_sync_database","table-name":"t1","transaction-id":670014899490}} +{"data":{"id":102,"name":"car battery","description":"12V car battery","weight":8.1,"address":"Shanghai"},"metadata":{"timestamp":"2024-10-27T12:39:03.210671Z","record-type":"data","operation":"insert","partition-key-type":"schema-table","schema-name":"paimon_sync_database","table-name":"t1","transaction-id":670014899490}} \ No newline at end of file diff --git a/paimon-flink/paimon-flink-cdc/src/test/resources/kafka/aws-dms/database/prefixsuffix/topic1/aws-dms-data-1.txt b/paimon-flink/paimon-flink-cdc/src/test/resources/kafka/aws-dms/database/prefixsuffix/topic1/aws-dms-data-1.txt new file mode 100644 index 000000000000..a0351adb7fd6 --- /dev/null +++ b/paimon-flink/paimon-flink-cdc/src/test/resources/kafka/aws-dms/database/prefixsuffix/topic1/aws-dms-data-1.txt @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +{"data":{"id":103,"name":"12-pack drill bits","description":"12-pack of drill bits with sizes ranging from #40 to #3","weight":0.8},"metadata":{"timestamp":"2024-10-27T12:39:03.210671Z","record-type":"data","operation":"insert","partition-key-type":"schema-table","schema-name":"paimon_sync_database","table-name":"t2","transaction-id":670014899490}} +{"data":{"id":104,"name":"hammer","description":"12oz carpenter's hammer","weight":0.75},"metadata":{"timestamp":"2024-10-27T12:39:03.210671Z","record-type":"data","operation":"insert","partition-key-type":"schema-table","schema-name":"paimon_sync_database","table-name":"t2","transaction-id":670014899490}} \ No newline at end of file diff --git a/paimon-flink/paimon-flink-cdc/src/test/resources/kafka/aws-dms/database/prefixsuffix/topic1/aws-dms-data-2.txt b/paimon-flink/paimon-flink-cdc/src/test/resources/kafka/aws-dms/database/prefixsuffix/topic1/aws-dms-data-2.txt new file mode 100644 index 000000000000..e59ef1c9a479 --- /dev/null +++ b/paimon-flink/paimon-flink-cdc/src/test/resources/kafka/aws-dms/database/prefixsuffix/topic1/aws-dms-data-2.txt @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +{"data":{"id":103,"name":"12-pack drill bits","description":"12-pack of drill bits with sizes ranging from #40 to #3","weight":0.8,"age":19},"metadata":{"timestamp":"2024-10-27T12:39:03.210671Z","record-type":"data","operation":"insert","partition-key-type":"schema-table","schema-name":"paimon_sync_database","table-name":"t2","transaction-id":670014899490}} +{"data":{"id":104,"name":"hammer","description":"12oz carpenter's hammer","weight":0.75,"age":25},"metadata":{"timestamp":"2024-10-27T12:39:03.210671Z","record-type":"data","operation":"insert","partition-key-type":"schema-table","schema-name":"paimon_sync_database","table-name":"t2","transaction-id":670014899490}} \ No newline at end of file diff --git a/paimon-flink/paimon-flink-cdc/src/test/resources/kafka/aws-dms/database/schemaevolution/topic0/aws-dms-data-1.txt b/paimon-flink/paimon-flink-cdc/src/test/resources/kafka/aws-dms/database/schemaevolution/topic0/aws-dms-data-1.txt new file mode 100644 index 000000000000..5ac7c5dbecef --- /dev/null +++ b/paimon-flink/paimon-flink-cdc/src/test/resources/kafka/aws-dms/database/schemaevolution/topic0/aws-dms-data-1.txt @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +{"data":{"id":101,"name":"scooter","description":"Small 2-wheel scooter","weight":3.14},"metadata":{"timestamp":"2024-10-27T12:39:03.210671Z","record-type":"data","operation":"insert","partition-key-type":"schema-table","schema-name":"paimon_sync_database","table-name":"t1","transaction-id":670014899490}} +{"data":{"id":102,"name":"car battery","description":"12V car battery","weight":8.1},"metadata":{"timestamp":"2024-10-27T12:39:03.210671Z","record-type":"data","operation":"insert","partition-key-type":"schema-table","schema-name":"paimon_sync_database","table-name":"t1","transaction-id":670014899490}} \ No newline at end of file diff --git a/paimon-flink/paimon-flink-cdc/src/test/resources/kafka/aws-dms/database/schemaevolution/topic0/aws-dms-data-2.txt b/paimon-flink/paimon-flink-cdc/src/test/resources/kafka/aws-dms/database/schemaevolution/topic0/aws-dms-data-2.txt new file mode 100644 index 000000000000..eeb254d71c4e --- /dev/null +++ b/paimon-flink/paimon-flink-cdc/src/test/resources/kafka/aws-dms/database/schemaevolution/topic0/aws-dms-data-2.txt @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +{"data":{"id":103,"name":"12-pack drill bits","description":"12-pack of drill bits with sizes ranging from #40 to #3","weight":0.8,"age":19},"metadata":{"timestamp":"2024-10-27T12:39:03.210671Z","record-type":"data","operation":"insert","partition-key-type":"schema-table","schema-name":"paimon_sync_database","table-name":"t1","transaction-id":670014899490}} +{"data":{"id":104,"name":"hammer","description":"12oz carpenter's hammer","weight":0.75,"age":25},"metadata":{"timestamp":"2024-10-27T12:39:03.210671Z","record-type":"data","operation":"insert","partition-key-type":"schema-table","schema-name":"paimon_sync_database","table-name":"t1","transaction-id":670014899490}} \ No newline at end of file diff --git a/paimon-flink/paimon-flink-cdc/src/test/resources/kafka/aws-dms/database/schemaevolution/topic1/aws-dms-data-1.txt b/paimon-flink/paimon-flink-cdc/src/test/resources/kafka/aws-dms/database/schemaevolution/topic1/aws-dms-data-1.txt new file mode 100644 index 000000000000..a0351adb7fd6 --- /dev/null +++ b/paimon-flink/paimon-flink-cdc/src/test/resources/kafka/aws-dms/database/schemaevolution/topic1/aws-dms-data-1.txt @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +{"data":{"id":103,"name":"12-pack drill bits","description":"12-pack of drill bits with sizes ranging from #40 to #3","weight":0.8},"metadata":{"timestamp":"2024-10-27T12:39:03.210671Z","record-type":"data","operation":"insert","partition-key-type":"schema-table","schema-name":"paimon_sync_database","table-name":"t2","transaction-id":670014899490}} +{"data":{"id":104,"name":"hammer","description":"12oz carpenter's hammer","weight":0.75},"metadata":{"timestamp":"2024-10-27T12:39:03.210671Z","record-type":"data","operation":"insert","partition-key-type":"schema-table","schema-name":"paimon_sync_database","table-name":"t2","transaction-id":670014899490}} \ No newline at end of file diff --git a/paimon-flink/paimon-flink-cdc/src/test/resources/kafka/aws-dms/database/schemaevolution/topic1/aws-dms-data-2.txt b/paimon-flink/paimon-flink-cdc/src/test/resources/kafka/aws-dms/database/schemaevolution/topic1/aws-dms-data-2.txt new file mode 100644 index 000000000000..a189a9d85df7 --- /dev/null +++ b/paimon-flink/paimon-flink-cdc/src/test/resources/kafka/aws-dms/database/schemaevolution/topic1/aws-dms-data-2.txt @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +{"data":{"id":103,"name":"12-pack drill bits","description":"12-pack of drill bits with sizes ranging from #40 to #3","weight":0.8,"address":"Beijing"},"metadata":{"timestamp":"2024-10-27T12:39:03.210671Z","record-type":"data","operation":"insert","partition-key-type":"schema-table","schema-name":"paimon_sync_database","table-name":"t2","transaction-id":670014899490}} +{"data":{"id":104,"name":"hammer","description":"12oz carpenter's hammer","weight":0.75,"address":"Shanghai"},"metadata":{"timestamp":"2024-10-27T12:39:03.210671Z","record-type":"data","operation":"insert","partition-key-type":"schema-table","schema-name":"paimon_sync_database","table-name":"t2","transaction-id":670014899490}} \ No newline at end of file diff --git a/paimon-flink/paimon-flink-cdc/src/test/resources/kafka/aws-dms/table/computedcolumn/aws-dms-data-1.txt b/paimon-flink/paimon-flink-cdc/src/test/resources/kafka/aws-dms/table/computedcolumn/aws-dms-data-1.txt new file mode 100644 index 000000000000..cf9112abc4bf --- /dev/null +++ b/paimon-flink/paimon-flink-cdc/src/test/resources/kafka/aws-dms/table/computedcolumn/aws-dms-data-1.txt @@ -0,0 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +{"data":{"_id":101,"_date":"2023-03-23"},"metadata":{"timestamp":"2024-10-27T12:39:03.210671Z","record-type":"data","operation":"insert","partition-key-type":"schema-table","schema-name":"test","table-name":"product","transaction-id":670014899490}} diff --git a/paimon-flink/paimon-flink-cdc/src/test/resources/kafka/aws-dms/table/schemaevolution/aws-dms-data-1.txt b/paimon-flink/paimon-flink-cdc/src/test/resources/kafka/aws-dms/table/schemaevolution/aws-dms-data-1.txt new file mode 100644 index 000000000000..42a00afe5b5e --- /dev/null +++ b/paimon-flink/paimon-flink-cdc/src/test/resources/kafka/aws-dms/table/schemaevolution/aws-dms-data-1.txt @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +{"data":{"id":101,"name":"scooter","description":"Small 2-wheel scooter","weight":3.14},"metadata":{"timestamp":"2024-10-27T12:39:03.210671Z","record-type":"data","operation":"insert","partition-key-type":"schema-table","schema-name":"test","table-name":"product","transaction-id":670014899490}} +{"data":{"id":102,"name":"car battery","description":"12V car battery","weight":8.1},"metadata":{"timestamp":"2024-10-27T12:39:03.210671Z","record-type":"data","operation":"insert","partition-key-type":"schema-table","schema-name":"test","table-name":"product","transaction-id":670014899490}} \ No newline at end of file diff --git a/paimon-flink/paimon-flink-cdc/src/test/resources/kafka/aws-dms/table/schemaevolution/aws-dms-data-2.txt b/paimon-flink/paimon-flink-cdc/src/test/resources/kafka/aws-dms/table/schemaevolution/aws-dms-data-2.txt new file mode 100644 index 000000000000..3ecfdab8b1a4 --- /dev/null +++ b/paimon-flink/paimon-flink-cdc/src/test/resources/kafka/aws-dms/table/schemaevolution/aws-dms-data-2.txt @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +{"data":{"id":103,"name":"12-pack drill bits","description":"12-pack of drill bits with sizes ranging from #40 to #3","weight":0.8,"age":18},"metadata":{"timestamp":"2024-10-27T12:39:03.210671Z","record-type":"data","operation":"insert","partition-key-type":"schema-table","schema-name":"test","table-name":"product","transaction-id":670014899490}} +{"data":{"id":104,"name":"hammer","description":"12oz carpenter's hammer","weight":0.75,"age":24},"metadata":{"timestamp":"2024-10-27T12:39:03.210671Z","record-type":"data","operation":"insert","partition-key-type":"schema-table","schema-name":"test","table-name":"product","transaction-id":670014899490}} \ No newline at end of file diff --git a/paimon-flink/paimon-flink-cdc/src/test/resources/kafka/aws-dms/table/schemaevolution/aws-dms-data-3.txt b/paimon-flink/paimon-flink-cdc/src/test/resources/kafka/aws-dms/table/schemaevolution/aws-dms-data-3.txt new file mode 100644 index 000000000000..04e18e1db548 --- /dev/null +++ b/paimon-flink/paimon-flink-cdc/src/test/resources/kafka/aws-dms/table/schemaevolution/aws-dms-data-3.txt @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +{"data":{"id":105,"name":"hammer","description":"14oz carpenter's hammer","weight":0.875,"address":"Shanghai"},"metadata":{"timestamp":"2024-10-27T12:39:03.210671Z","record-type":"data","operation":"insert","partition-key-type":"schema-table","schema-name":"test","table-name":"product","transaction-id":670014899490}} +{"data":{"id":101,"name":"scooter","description":"Small 2-wheel scooter","weight":3.14},"metadata":{"timestamp":"2024-10-27T12:39:03.210671Z","record-type":"data","operation":"delete","partition-key-type":"schema-table","schema-name":"test","table-name":"product","transaction-id":670014899490}} +{"data":{"id":105,"name":"hammer","description":"14oz carpenter's hammer","weight":0.875,"address":"Beijing","BI_id":105,"BI_name":"hammer","BI_description":"14oz carpenter's hammer","BI_weight":0.875,"BI_address":"Shanghai"},"metadata":{"timestamp":"2024-10-27T12:39:03.210671Z","record-type":"data","operation":"update","partition-key-type":"schema-table","schema-name":"test","table-name":"product","transaction-id":670014899490}} +{"data":{"id":107,"name":"rocks","description":"box of assorted rocks","weight":5.3},"metadata":{"timestamp":"2024-10-27T12:39:03.210671Z","record-type":"data","operation":"insert","partition-key-type":"schema-table","schema-name":"test","table-name":"product","transaction-id":670014899490}} \ No newline at end of file diff --git a/paimon-flink/paimon-flink-cdc/src/test/resources/kafka/aws-dms/table/schemaevolution/aws-dms-data-4.txt b/paimon-flink/paimon-flink-cdc/src/test/resources/kafka/aws-dms/table/schemaevolution/aws-dms-data-4.txt new file mode 100644 index 000000000000..e93607aed68d --- /dev/null +++ b/paimon-flink/paimon-flink-cdc/src/test/resources/kafka/aws-dms/table/schemaevolution/aws-dms-data-4.txt @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +{"data":{"id":103,"name":"12-pack drill bits","description":"12-pack of drill bits with sizes ranging from #40 to #3","weight":null},"metadata":{"timestamp":"2024-10-27T12:39:03.210671Z","record-type":"data","operation":"insert","partition-key-type":"schema-table","schema-name":"test","table-name":"product","transaction-id":670014899490}} +{"data":{"id":104,"name":"hammer","description":"12oz carpenter's hammer","weight":0.75},"metadata":{"timestamp":"2024-10-27T12:39:03.210671Z","record-type":"data","operation":"insert","partition-key-type":"schema-table","schema-name":"test","table-name":"product","transaction-id":670014899490}} \ No newline at end of file diff --git a/paimon-flink/paimon-flink-cdc/src/test/resources/kafka/aws-dms/table/startupmode/aws-dms-data-1.txt b/paimon-flink/paimon-flink-cdc/src/test/resources/kafka/aws-dms/table/startupmode/aws-dms-data-1.txt new file mode 100644 index 000000000000..42a00afe5b5e --- /dev/null +++ b/paimon-flink/paimon-flink-cdc/src/test/resources/kafka/aws-dms/table/startupmode/aws-dms-data-1.txt @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +{"data":{"id":101,"name":"scooter","description":"Small 2-wheel scooter","weight":3.14},"metadata":{"timestamp":"2024-10-27T12:39:03.210671Z","record-type":"data","operation":"insert","partition-key-type":"schema-table","schema-name":"test","table-name":"product","transaction-id":670014899490}} +{"data":{"id":102,"name":"car battery","description":"12V car battery","weight":8.1},"metadata":{"timestamp":"2024-10-27T12:39:03.210671Z","record-type":"data","operation":"insert","partition-key-type":"schema-table","schema-name":"test","table-name":"product","transaction-id":670014899490}} \ No newline at end of file diff --git a/paimon-flink/paimon-flink-cdc/src/test/resources/kafka/aws-dms/table/startupmode/aws-dms-data-2.txt b/paimon-flink/paimon-flink-cdc/src/test/resources/kafka/aws-dms/table/startupmode/aws-dms-data-2.txt new file mode 100644 index 000000000000..70c0fb1675ea --- /dev/null +++ b/paimon-flink/paimon-flink-cdc/src/test/resources/kafka/aws-dms/table/startupmode/aws-dms-data-2.txt @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +{"data":{"id":103,"name":"12-pack drill bits","description":"12-pack of drill bits with sizes ranging from #40 to #3","weight":0.8},"metadata":{"timestamp":"2024-10-27T12:39:03.210671Z","record-type":"data","operation":"insert","partition-key-type":"schema-table","schema-name":"test","table-name":"product","transaction-id":670014899490}} +{"data":{"id":104,"name":"hammer","description":"12oz carpenter's hammer","weight":0.75},"metadata":{"timestamp":"2024-10-27T12:39:03.210671Z","record-type":"data","operation":"insert","partition-key-type":"schema-table","schema-name":"test","table-name":"product","transaction-id":670014899490}} \ No newline at end of file diff --git a/paimon-flink/paimon-flink-cdc/src/test/resources/kafka/aws-dms/table/watermark/aws-dms-data-1.txt b/paimon-flink/paimon-flink-cdc/src/test/resources/kafka/aws-dms/table/watermark/aws-dms-data-1.txt new file mode 100644 index 000000000000..42a00afe5b5e --- /dev/null +++ b/paimon-flink/paimon-flink-cdc/src/test/resources/kafka/aws-dms/table/watermark/aws-dms-data-1.txt @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +{"data":{"id":101,"name":"scooter","description":"Small 2-wheel scooter","weight":3.14},"metadata":{"timestamp":"2024-10-27T12:39:03.210671Z","record-type":"data","operation":"insert","partition-key-type":"schema-table","schema-name":"test","table-name":"product","transaction-id":670014899490}} +{"data":{"id":102,"name":"car battery","description":"12V car battery","weight":8.1},"metadata":{"timestamp":"2024-10-27T12:39:03.210671Z","record-type":"data","operation":"insert","partition-key-type":"schema-table","schema-name":"test","table-name":"product","transaction-id":670014899490}} \ No newline at end of file diff --git a/paimon-flink/paimon-flink-cdc/src/test/resources/mysql/sync_table_setup.sql b/paimon-flink/paimon-flink-cdc/src/test/resources/mysql/sync_table_setup.sql index 965f884ec680..10a0f20d45aa 100644 --- a/paimon-flink/paimon-flink-cdc/src/test/resources/mysql/sync_table_setup.sql +++ b/paimon-flink/paimon-flink-cdc/src/test/resources/mysql/sync_table_setup.sql @@ -445,3 +445,14 @@ CREATE TABLE t ( k INT PRIMARY KEY, v1 VARCHAR(10) ); + +-- ################################################################################ +-- testRuntimeExecutionModeCheckForCdcSync +-- ################################################################################ + +CREATE DATABASE check_cdc_sync_runtime_execution_mode; +USE check_cdc_sync_runtime_execution_mode; +CREATE TABLE t ( + k INT PRIMARY KEY, + v1 VARCHAR(10) +); \ No newline at end of file diff --git a/paimon-flink/paimon-flink-common/pom.xml b/paimon-flink/paimon-flink-common/pom.xml index ef0f7fe1776a..91222983bf6b 100644 --- a/paimon-flink/paimon-flink-common/pom.xml +++ b/paimon-flink/paimon-flink-common/pom.xml @@ -122,7 +122,7 @@ under the License. org.apache.flink - flink-table-planner_${scala.binary.version} + flink-table-planner_${flink.scala.binary.version} ${flink.version} test test-jar @@ -130,7 +130,7 @@ under the License. org.apache.flink - flink-table-planner_${scala.binary.version} + flink-table-planner_${flink.scala.binary.version} ${flink.version} test @@ -150,6 +150,25 @@ under the License. test + + org.apache.iceberg + iceberg-core + ${iceberg.version} + test + + + + org.apache.iceberg + iceberg-data + ${iceberg.version} + test + + + parquet-avro + org.apache.parquet + + + diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/AbstractFlinkTableFactory.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/AbstractFlinkTableFactory.java index e469044f5f4f..6b10dbb84bf4 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/AbstractFlinkTableFactory.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/AbstractFlinkTableFactory.java @@ -22,17 +22,15 @@ import org.apache.paimon.CoreOptions.LogConsistency; import org.apache.paimon.CoreOptions.StreamingReadMode; import org.apache.paimon.annotation.VisibleForTesting; +import org.apache.paimon.catalog.Catalog; import org.apache.paimon.catalog.CatalogContext; -import org.apache.paimon.data.Timestamp; +import org.apache.paimon.catalog.Identifier; import org.apache.paimon.flink.log.LogStoreTableFactory; import org.apache.paimon.flink.sink.FlinkTableSink; import org.apache.paimon.flink.source.DataTableSource; import org.apache.paimon.flink.source.SystemTableSource; -import org.apache.paimon.lineage.LineageMeta; -import org.apache.paimon.lineage.LineageMetaFactory; -import org.apache.paimon.lineage.TableLineageEntity; -import org.apache.paimon.lineage.TableLineageEntityImpl; import org.apache.paimon.options.Options; +import org.apache.paimon.options.OptionsUtils; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.table.FileStoreTable; @@ -44,7 +42,6 @@ import org.apache.flink.configuration.ConfigOption; import org.apache.flink.configuration.Configuration; import org.apache.flink.configuration.ExecutionOptions; -import org.apache.flink.configuration.PipelineOptions; import org.apache.flink.configuration.ReadableConfig; import org.apache.flink.table.api.TableConfig; import org.apache.flink.table.api.ValidationException; @@ -68,8 +65,6 @@ import java.util.Objects; import java.util.Optional; import java.util.Set; -import java.util.function.BiConsumer; -import java.util.regex.Matcher; import java.util.regex.Pattern; import static org.apache.paimon.CoreOptions.LOG_CHANGELOG_MODE; @@ -89,6 +84,12 @@ public abstract class AbstractFlinkTableFactory private static final Logger LOG = LoggerFactory.getLogger(AbstractFlinkTableFactory.class); + @Nullable private final FlinkCatalog flinkCatalog; + + public AbstractFlinkTableFactory(@Nullable FlinkCatalog flinkCatalog) { + this.flinkCatalog = flinkCatalog; + } + @Override public DynamicTableSource createDynamicTableSource(Context context) { CatalogTable origin = context.getCatalogTable().getOrigin(); @@ -101,23 +102,9 @@ public DynamicTableSource createDynamicTableSource(Context context) { isStreamingMode, context.getObjectIdentifier()); } else { - Table table = buildPaimonTable(context); - if (table instanceof FileStoreTable) { - storeTableLineage( - ((FileStoreTable) table).catalogEnvironment().lineageMetaFactory(), - context, - (entity, lineageFactory) -> { - try (LineageMeta lineage = - lineageFactory.create(() -> Options.fromMap(table.options()))) { - lineage.saveSourceTableLineage(entity); - } catch (Exception e) { - throw new RuntimeException(e); - } - }); - } return new DataTableSource( context.getObjectIdentifier(), - table, + buildPaimonTable(context), isStreamingMode, context, createOptionalLogStoreFactory(context).orElse(null)); @@ -126,46 +113,13 @@ public DynamicTableSource createDynamicTableSource(Context context) { @Override public DynamicTableSink createDynamicTableSink(Context context) { - Table table = buildPaimonTable(context); - if (table instanceof FileStoreTable) { - storeTableLineage( - ((FileStoreTable) table).catalogEnvironment().lineageMetaFactory(), - context, - (entity, lineageFactory) -> { - try (LineageMeta lineage = - lineageFactory.create(() -> Options.fromMap(table.options()))) { - lineage.saveSinkTableLineage(entity); - } catch (Exception e) { - throw new RuntimeException(e); - } - }); - } return new FlinkTableSink( context.getObjectIdentifier(), - table, + buildPaimonTable(context), context, createOptionalLogStoreFactory(context).orElse(null)); } - private void storeTableLineage( - @Nullable LineageMetaFactory lineageMetaFactory, - Context context, - BiConsumer tableLineage) { - if (lineageMetaFactory != null) { - String pipelineName = context.getConfiguration().get(PipelineOptions.NAME); - if (pipelineName == null) { - throw new ValidationException("Cannot get pipeline name for lineage meta."); - } - tableLineage.accept( - new TableLineageEntityImpl( - context.getObjectIdentifier().getDatabaseName(), - context.getObjectIdentifier().getObjectName(), - pipelineName, - Timestamp.fromEpochMillis(System.currentTimeMillis())), - lineageMetaFactory); - } - } - @Override public Set> requiredOptions() { return Collections.emptySet(); @@ -227,11 +181,11 @@ static CatalogContext createCatalogContext(DynamicTableFactory.Context context) Options.fromMap(context.getCatalogTable().getOptions()), new FlinkFileIOLoader()); } - static Table buildPaimonTable(DynamicTableFactory.Context context) { + Table buildPaimonTable(DynamicTableFactory.Context context) { CatalogTable origin = context.getCatalogTable().getOrigin(); Table table; - Map dynamicOptions = getDynamicTableConfigOptions(context); + Map dynamicOptions = getDynamicConfigOptions(context); dynamicOptions.forEach( (key, newValue) -> { String oldValue = origin.getOptions().get(key); @@ -241,18 +195,31 @@ static Table buildPaimonTable(DynamicTableFactory.Context context) { }); Map newOptions = new HashMap<>(); newOptions.putAll(origin.getOptions()); + // dynamic options should override origin options newOptions.putAll(dynamicOptions); - // notice that the Paimon table schema must be the same with the Flink's + FileStoreTable fileStoreTable; if (origin instanceof DataCatalogTable) { - FileStoreTable fileStoreTable = (FileStoreTable) ((DataCatalogTable) origin).table(); - table = fileStoreTable.copyWithoutTimeTravel(newOptions); + fileStoreTable = (FileStoreTable) ((DataCatalogTable) origin).table(); + } else if (flinkCatalog == null) { + // In case Paimon is directly used as a Flink connector, instead of through catalog. + fileStoreTable = FileStoreTableFactory.create(createCatalogContext(context)); } else { - table = - FileStoreTableFactory.create(createCatalogContext(context)) - .copyWithoutTimeTravel(newOptions); + // In cases like materialized table, the Paimon table might not be DataCatalogTable, + // but can still be acquired through the catalog. + Identifier identifier = + Identifier.create( + context.getObjectIdentifier().getDatabaseName(), + context.getObjectIdentifier().getObjectName()); + try { + fileStoreTable = (FileStoreTable) flinkCatalog.catalog().getTable(identifier); + } catch (Catalog.TableNotExistException e) { + throw new RuntimeException(e); + } } + table = fileStoreTable.copyWithoutTimeTravel(newOptions); + // notice that the Paimon table schema must be the same with the Flink's Schema schema = FlinkCatalog.fromCatalogTable(context.getCatalogTable()); RowType rowType = toLogicalType(schema.rowType()); @@ -304,16 +271,19 @@ static boolean schemaEquals(RowType rowType1, RowType rowType2) { /** * The dynamic option's format is: * - *

    {@link - * FlinkConnectorOptions#TABLE_DYNAMIC_OPTION_PREFIX}.${catalog}.${database}.${tableName}.key = - * value. These job level configs will be extracted and injected into the target table option. + *

    Global Options: key = value . + * + *

    Table Options: {@link + * FlinkConnectorOptions#TABLE_DYNAMIC_OPTION_PREFIX}${catalog}.${database}.${tableName}.key = + * value. + * + *

    These job level options will be extracted and injected into the target table option. Table + * options will override global options if there are conflicts. * * @param context The table factory context. * @return The dynamic options of this target table. */ - static Map getDynamicTableConfigOptions(DynamicTableFactory.Context context) { - - Map optionsFromTableConfig = new HashMap<>(); + static Map getDynamicConfigOptions(DynamicTableFactory.Context context) { ReadableConfig config = context.getConfiguration(); @@ -329,23 +299,14 @@ static Map getDynamicTableConfigOptions(DynamicTableFactory.Cont String template = String.format( - "(%s)\\.(%s|\\*)\\.(%s|\\*)\\.(%s|\\*)\\.(.+)", + "(%s)(%s|\\*)\\.(%s|\\*)\\.(%s|\\*)\\.(.+)", FlinkConnectorOptions.TABLE_DYNAMIC_OPTION_PREFIX, context.getObjectIdentifier().getCatalogName(), context.getObjectIdentifier().getDatabaseName(), context.getObjectIdentifier().getObjectName()); Pattern pattern = Pattern.compile(template); - - conf.keySet() - .forEach( - (key) -> { - if (key.startsWith(FlinkConnectorOptions.TABLE_DYNAMIC_OPTION_PREFIX)) { - Matcher matcher = pattern.matcher(key); - if (matcher.find()) { - optionsFromTableConfig.put(matcher.group(5), conf.get(key)); - } - } - }); + Map optionsFromTableConfig = + OptionsUtils.convertToDynamicTableProperties(conf, "", pattern, 5); if (!optionsFromTableConfig.isEmpty()) { LOG.info( diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/DataCatalogTable.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/DataCatalogTable.java index 019d7bd6892f..e141581b476b 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/DataCatalogTable.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/DataCatalogTable.java @@ -23,33 +23,55 @@ import org.apache.paimon.types.DataField; import org.apache.flink.table.api.Schema; -import org.apache.flink.table.api.TableColumn; -import org.apache.flink.table.api.TableSchema; -import org.apache.flink.table.api.constraints.UniqueConstraint; import org.apache.flink.table.catalog.CatalogBaseTable; import org.apache.flink.table.catalog.CatalogTable; -import org.apache.flink.table.catalog.CatalogTableImpl; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.stream.Collectors; -/** A {@link CatalogTableImpl} to wrap {@link FileStoreTable}. */ -public class DataCatalogTable extends CatalogTableImpl { +import static org.apache.flink.util.Preconditions.checkArgument; +import static org.apache.flink.util.Preconditions.checkNotNull; + +/** A {@link CatalogTable} to wrap {@link FileStoreTable}. */ +public class DataCatalogTable implements CatalogTable { + // Schema of the table (column names and types) + private final Schema schema; + + // Partition keys if this is a partitioned table. It's an empty set if the table is not + // partitioned + private final List partitionKeys; + + // Properties of the table + private final Map options; + + // Comment of the table + private final String comment; private final Table table; private final Map nonPhysicalColumnComments; public DataCatalogTable( Table table, - TableSchema tableSchema, + Schema resolvedSchema, List partitionKeys, - Map properties, + Map options, String comment, Map nonPhysicalColumnComments) { - super(tableSchema, partitionKeys, properties, comment); + this.schema = resolvedSchema; + this.partitionKeys = checkNotNull(partitionKeys, "partitionKeys cannot be null"); + this.options = checkNotNull(options, "options cannot be null"); + + checkArgument( + options.entrySet().stream() + .allMatch(e -> e.getKey() != null && e.getValue() != null), + "properties cannot have null keys or values"); + + this.comment = comment; + this.table = table; this.nonPhysicalColumnComments = nonPhysicalColumnComments; } @@ -66,32 +88,30 @@ public Schema getUnresolvedSchema() { .filter(dataField -> dataField.description() != null) .collect(Collectors.toMap(DataField::name, DataField::description)); - return toSchema(getSchema(), columnComments); + return toSchema(schema, columnComments); } - /** Copied from {@link TableSchema#toSchema(Map)} to support versions lower than 1.17. */ - private Schema toSchema(TableSchema tableSchema, Map comments) { + private Schema toSchema(Schema tableSchema, Map comments) { final Schema.Builder builder = Schema.newBuilder(); - tableSchema - .getTableColumns() + .getColumns() .forEach( column -> { - if (column instanceof TableColumn.PhysicalColumn) { - final TableColumn.PhysicalColumn c = - (TableColumn.PhysicalColumn) column; - builder.column(c.getName(), c.getType()); - } else if (column instanceof TableColumn.MetadataColumn) { - final TableColumn.MetadataColumn c = - (TableColumn.MetadataColumn) column; + if (column instanceof Schema.UnresolvedPhysicalColumn) { + final Schema.UnresolvedPhysicalColumn c = + (Schema.UnresolvedPhysicalColumn) column; + builder.column(c.getName(), c.getDataType()); + } else if (column instanceof Schema.UnresolvedMetadataColumn) { + final Schema.UnresolvedMetadataColumn c = + (Schema.UnresolvedMetadataColumn) column; builder.columnByMetadata( c.getName(), - c.getType(), - c.getMetadataAlias().orElse(null), + c.getDataType(), + c.getMetadataKey(), c.isVirtual()); - } else if (column instanceof TableColumn.ComputedColumn) { - final TableColumn.ComputedColumn c = - (TableColumn.ComputedColumn) column; + } else if (column instanceof Schema.UnresolvedComputedColumn) { + final Schema.UnresolvedComputedColumn c = + (Schema.UnresolvedComputedColumn) column; builder.columnByExpression(c.getName(), c.getExpression()); } else { throw new IllegalArgumentException( @@ -104,19 +124,16 @@ private Schema toSchema(TableSchema tableSchema, Map comments) { builder.withComment(nonPhysicalColumnComments.get(colName)); } }); - tableSchema .getWatermarkSpecs() .forEach( spec -> builder.watermark( - spec.getRowtimeAttribute(), spec.getWatermarkExpr())); - + spec.getColumnName(), spec.getWatermarkExpression())); if (tableSchema.getPrimaryKey().isPresent()) { - UniqueConstraint primaryKey = tableSchema.getPrimaryKey().get(); - builder.primaryKeyNamed(primaryKey.getName(), primaryKey.getColumns()); + Schema.UnresolvedPrimaryKey primaryKey = tableSchema.getPrimaryKey().get(); + builder.primaryKeyNamed(primaryKey.getConstraintName(), primaryKey.getColumnNames()); } - return builder.build(); } @@ -124,7 +141,7 @@ private Schema toSchema(TableSchema tableSchema, Map comments) { public CatalogBaseTable copy() { return new DataCatalogTable( table, - getSchema().copy(), + schema, new ArrayList<>(getPartitionKeys()), new HashMap<>(getOptions()), getComment(), @@ -135,10 +152,40 @@ public CatalogBaseTable copy() { public CatalogTable copy(Map options) { return new DataCatalogTable( table, - getSchema(), + schema, getPartitionKeys(), options, getComment(), nonPhysicalColumnComments); } + + @Override + public Optional getDescription() { + return Optional.of(getComment()); + } + + @Override + public Optional getDetailedDescription() { + return Optional.of("This is a catalog table in an im-memory catalog"); + } + + @Override + public boolean isPartitioned() { + return !partitionKeys.isEmpty(); + } + + @Override + public List getPartitionKeys() { + return partitionKeys; + } + + @Override + public Map getOptions() { + return options; + } + + @Override + public String getComment() { + return comment != null ? comment : ""; + } } diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/FlinkCatalog.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/FlinkCatalog.java index 0f46f2965c86..3a7f9790ccca 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/FlinkCatalog.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/FlinkCatalog.java @@ -24,22 +24,29 @@ import org.apache.paimon.catalog.Identifier; import org.apache.paimon.flink.procedure.ProcedureUtil; import org.apache.paimon.flink.utils.FlinkCatalogPropertiesUtil; +import org.apache.paimon.flink.utils.FlinkDescriptorProperties; import org.apache.paimon.fs.Path; import org.apache.paimon.manifest.PartitionEntry; +import org.apache.paimon.operation.FileStoreCommit; import org.apache.paimon.options.Options; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaChange; import org.apache.paimon.schema.SchemaManager; +import org.apache.paimon.stats.Statistics; import org.apache.paimon.table.FileStoreTable; import org.apache.paimon.table.FormatTable; import org.apache.paimon.table.Table; +import org.apache.paimon.table.sink.BatchWriteBuilder; import org.apache.paimon.table.source.ReadBuilder; +import org.apache.paimon.types.DataField; +import org.apache.paimon.types.DataTypeRoot; import org.apache.paimon.utils.FileStorePathFactory; import org.apache.paimon.utils.InternalRowPartitionComputer; import org.apache.paimon.utils.Preconditions; import org.apache.paimon.utils.StringUtils; +import org.apache.paimon.view.View; +import org.apache.paimon.view.ViewImpl; -import org.apache.flink.table.api.TableSchema; import org.apache.flink.table.catalog.AbstractCatalog; import org.apache.flink.table.catalog.CatalogBaseTable; import org.apache.flink.table.catalog.CatalogDatabase; @@ -50,10 +57,12 @@ import org.apache.flink.table.catalog.CatalogPartitionImpl; import org.apache.flink.table.catalog.CatalogPartitionSpec; import org.apache.flink.table.catalog.CatalogTable; +import org.apache.flink.table.catalog.CatalogView; import org.apache.flink.table.catalog.Column; import org.apache.flink.table.catalog.IntervalFreshness; import org.apache.flink.table.catalog.ObjectPath; import org.apache.flink.table.catalog.ResolvedCatalogBaseTable; +import org.apache.flink.table.catalog.ResolvedCatalogView; import org.apache.flink.table.catalog.ResolvedSchema; import org.apache.flink.table.catalog.TableChange; import org.apache.flink.table.catalog.TableChange.AddColumn; @@ -79,6 +88,7 @@ import org.apache.flink.table.catalog.exceptions.DatabaseNotEmptyException; import org.apache.flink.table.catalog.exceptions.DatabaseNotExistException; import org.apache.flink.table.catalog.exceptions.FunctionNotExistException; +import org.apache.flink.table.catalog.exceptions.PartitionAlreadyExistsException; import org.apache.flink.table.catalog.exceptions.PartitionNotExistException; import org.apache.flink.table.catalog.exceptions.ProcedureNotExistException; import org.apache.flink.table.catalog.exceptions.TableAlreadyExistException; @@ -86,11 +96,9 @@ import org.apache.flink.table.catalog.exceptions.TableNotPartitionedException; import org.apache.flink.table.catalog.stats.CatalogColumnStatistics; import org.apache.flink.table.catalog.stats.CatalogTableStatistics; -import org.apache.flink.table.descriptors.DescriptorProperties; import org.apache.flink.table.expressions.Expression; import org.apache.flink.table.factories.Factory; import org.apache.flink.table.procedures.Procedure; -import org.apache.flink.table.types.logical.LogicalType; import org.apache.flink.table.types.logical.RowType; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -102,20 +110,16 @@ import java.util.Arrays; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.Set; +import java.util.function.Function; import java.util.stream.Collectors; -import static org.apache.flink.table.descriptors.DescriptorProperties.COMMENT; -import static org.apache.flink.table.descriptors.DescriptorProperties.NAME; -import static org.apache.flink.table.descriptors.DescriptorProperties.WATERMARK; -import static org.apache.flink.table.descriptors.DescriptorProperties.WATERMARK_ROWTIME; -import static org.apache.flink.table.descriptors.DescriptorProperties.WATERMARK_STRATEGY_DATA_TYPE; -import static org.apache.flink.table.descriptors.DescriptorProperties.WATERMARK_STRATEGY_EXPR; -import static org.apache.flink.table.descriptors.Schema.SCHEMA; import static org.apache.flink.table.factories.FactoryUtil.CONNECTOR; import static org.apache.flink.table.types.utils.TypeConversions.fromLogicalToDataType; import static org.apache.flink.table.utils.EncodingUtils.decodeBase64ToBytes; @@ -128,8 +132,11 @@ import static org.apache.paimon.CoreOptions.MATERIALIZED_TABLE_REFRESH_HANDLER_DESCRIPTION; import static org.apache.paimon.CoreOptions.MATERIALIZED_TABLE_REFRESH_MODE; import static org.apache.paimon.CoreOptions.MATERIALIZED_TABLE_REFRESH_STATUS; -import static org.apache.paimon.CoreOptions.MATERIALIZED_TABLE_SNAPSHOT; import static org.apache.paimon.CoreOptions.PATH; +import static org.apache.paimon.catalog.Catalog.LAST_UPDATE_TIME_PROP; +import static org.apache.paimon.catalog.Catalog.NUM_FILES_PROP; +import static org.apache.paimon.catalog.Catalog.NUM_ROWS_PROP; +import static org.apache.paimon.catalog.Catalog.TOTAL_SIZE_PROP; import static org.apache.paimon.flink.FlinkCatalogOptions.DISABLE_CREATE_TABLE_IN_DEFAULT_DB; import static org.apache.paimon.flink.FlinkCatalogOptions.LOG_SYSTEM_AUTO_REGISTER; import static org.apache.paimon.flink.FlinkCatalogOptions.REGISTER_TIMEOUT; @@ -137,11 +144,20 @@ import static org.apache.paimon.flink.LogicalTypeConversion.toLogicalType; import static org.apache.paimon.flink.log.LogStoreRegister.registerLogSystem; import static org.apache.paimon.flink.log.LogStoreRegister.unRegisterLogSystem; +import static org.apache.paimon.flink.utils.FlinkCatalogPropertiesUtil.SCHEMA; import static org.apache.paimon.flink.utils.FlinkCatalogPropertiesUtil.compoundKey; import static org.apache.paimon.flink.utils.FlinkCatalogPropertiesUtil.deserializeNonPhysicalColumn; import static org.apache.paimon.flink.utils.FlinkCatalogPropertiesUtil.deserializeWatermarkSpec; import static org.apache.paimon.flink.utils.FlinkCatalogPropertiesUtil.nonPhysicalColumnsCount; import static org.apache.paimon.flink.utils.FlinkCatalogPropertiesUtil.serializeNewWatermarkSpec; +import static org.apache.paimon.flink.utils.FlinkDescriptorProperties.COMMENT; +import static org.apache.paimon.flink.utils.FlinkDescriptorProperties.NAME; +import static org.apache.paimon.flink.utils.FlinkDescriptorProperties.WATERMARK; +import static org.apache.paimon.flink.utils.FlinkDescriptorProperties.WATERMARK_ROWTIME; +import static org.apache.paimon.flink.utils.FlinkDescriptorProperties.WATERMARK_STRATEGY_DATA_TYPE; +import static org.apache.paimon.flink.utils.FlinkDescriptorProperties.WATERMARK_STRATEGY_EXPR; +import static org.apache.paimon.flink.utils.TableStatsUtil.createTableColumnStats; +import static org.apache.paimon.flink.utils.TableStatsUtil.createTableStats; import static org.apache.paimon.utils.Preconditions.checkArgument; import static org.apache.paimon.utils.Preconditions.checkNotNull; @@ -149,12 +165,8 @@ public class FlinkCatalog extends AbstractCatalog { private static final Logger LOG = LoggerFactory.getLogger(FlinkCatalog.class); - public static final String NUM_ROWS_KEY = "numRows"; - public static final String LAST_UPDATE_TIME_KEY = "lastUpdateTime"; - public static final String TOTAL_SIZE_KEY = "totalSize"; - public static final String NUM_FILES_KEY = "numFiles"; - private final ClassLoader classLoader; + private final ClassLoader classLoader; private final Catalog catalog; private final String name; private final boolean logStoreAutoRegister; @@ -177,7 +189,9 @@ public FlinkCatalog( this.logStoreAutoRegisterTimeout = options.get(REGISTER_TIMEOUT); this.disableCreateTableInDefaultDatabase = options.get(DISABLE_CREATE_TABLE_IN_DEFAULT_DB); if (!disableCreateTableInDefaultDatabase) { - if (!catalog.databaseExists(defaultDatabase)) { + try { + getDatabase(defaultDatabase); + } catch (DatabaseNotExistException e) { try { catalog.createDatabase(defaultDatabase, true); } catch (Catalog.DatabaseAlreadyExistException ignore) { @@ -192,7 +206,7 @@ public Catalog catalog() { @Override public Optional getFactory() { - return Optional.of(new FlinkTableFactory()); + return Optional.of(new FlinkTableFactory(this)); } @Override @@ -202,7 +216,12 @@ public List listDatabases() throws CatalogException { @Override public boolean databaseExists(String databaseName) throws CatalogException { - return catalog.databaseExists(databaseName); + try { + catalog.getDatabase(databaseName); + return true; + } catch (Catalog.DatabaseNotExistException e) { + return false; + } } @Override @@ -277,6 +296,11 @@ private CatalogBaseTable getTable(ObjectPath tablePath, @Nullable Long timestamp try { table = catalog.getTable(toIdentifier(tablePath)); } catch (Catalog.TableNotExistException e) { + Optional view = getView(tablePath, timestamp); + if (view.isPresent()) { + return view.get(); + } + throw new TableNotExistException(getName(), tablePath); } @@ -302,19 +326,70 @@ private CatalogBaseTable getTable(ObjectPath tablePath, @Nullable Long timestamp } } + private Optional getView(ObjectPath tablePath, @Nullable Long timestamp) { + View view; + try { + view = catalog.getView(toIdentifier(tablePath)); + } catch (Catalog.ViewNotExistException e) { + return Optional.empty(); + } + + if (timestamp != null) { + throw new UnsupportedOperationException( + String.format("View %s does not support time travel.", tablePath)); + } + + org.apache.flink.table.api.Schema schema = + org.apache.flink.table.api.Schema.newBuilder() + .fromRowDataType(fromLogicalToDataType(toLogicalType(view.rowType()))) + .build(); + return Optional.of( + CatalogView.of( + schema, + view.comment().orElse(null), + view.query(), + view.query(), + view.options())); + } + @Override public boolean tableExists(ObjectPath tablePath) throws CatalogException { - return catalog.tableExists(toIdentifier(tablePath)); + Identifier identifier = toIdentifier(tablePath); + try { + catalog.getTable(identifier); + return true; + } catch (Catalog.TableNotExistException e) { + try { + catalog.getView(identifier); + return true; + } catch (Catalog.ViewNotExistException ex) { + return false; + } + } } @Override public void dropTable(ObjectPath tablePath, boolean ignoreIfNotExists) throws TableNotExistException, CatalogException { Identifier identifier = toIdentifier(tablePath); - Table table = null; try { - if (logStoreAutoRegister && catalog.tableExists(identifier)) { - table = catalog.getTable(identifier); + catalog.getView(identifier); + try { + catalog.dropView(identifier, ignoreIfNotExists); + return; + } catch (Catalog.ViewNotExistException e) { + throw new RuntimeException("Unexpected exception.", e); + } + } catch (Catalog.ViewNotExistException ignored) { + } + + try { + Table table = null; + if (logStoreAutoRegister) { + try { + table = catalog.getTable(identifier); + } catch (Catalog.TableNotExistException ignored) { + } } catalog.dropTable(toIdentifier(tablePath), ignoreIfNotExists); if (logStoreAutoRegister && table != null) { @@ -328,18 +403,17 @@ public void dropTable(ObjectPath tablePath, boolean ignoreIfNotExists) @Override public void createTable(ObjectPath tablePath, CatalogBaseTable table, boolean ignoreIfExists) throws TableAlreadyExistException, DatabaseNotExistException, CatalogException { - if (!(table instanceof CatalogTable || table instanceof CatalogMaterializedTable)) { - throw new UnsupportedOperationException( - "Only support CatalogTable and CatalogMaterializedTable, but is: " - + table.getClass()); - } - if (Objects.equals(getDefaultDatabase(), tablePath.getDatabaseName()) && disableCreateTableInDefaultDatabase) { throw new UnsupportedOperationException( "Creating table in default database is disabled, please specify a database name."); } + if (table instanceof CatalogView) { + createView(tablePath, (ResolvedCatalogView) table, ignoreIfExists); + return; + } + Identifier identifier = toIdentifier(tablePath); // the returned value of "table.getOptions" may be unmodifiable (for example from // TableDescriptor) @@ -365,11 +439,38 @@ public void createTable(ObjectPath tablePath, CatalogBaseTable table, boolean ig } } + private void createView(ObjectPath tablePath, ResolvedCatalogView table, boolean ignoreIfExists) + throws TableAlreadyExistException, DatabaseNotExistException { + Identifier identifier = toIdentifier(tablePath); + org.apache.paimon.types.RowType.Builder builder = org.apache.paimon.types.RowType.builder(); + table.getResolvedSchema() + .getColumns() + .forEach( + column -> + builder.field( + column.getName(), + toDataType(column.getDataType().getLogicalType()), + column.getComment().orElse(null))); + View view = + new ViewImpl( + identifier, + builder.build(), + table.getOriginalQuery(), + table.getComment(), + table.getOptions()); + try { + catalog.createView(identifier, view, ignoreIfExists); + } catch (Catalog.ViewAlreadyExistException e) { + throw new TableAlreadyExistException(getName(), tablePath); + } catch (Catalog.DatabaseNotExistException e) { + throw new DatabaseNotExistException(getName(), tablePath.getDatabaseName()); + } + } + private static void fillOptionsForMaterializedTable( CatalogMaterializedTable mt, Map options) { Options mtOptions = new Options(); mtOptions.set(CoreOptions.TYPE, TableType.MATERIALIZED_TABLE); - mt.getSnapshot().ifPresent(x -> mtOptions.set(MATERIALIZED_TABLE_SNAPSHOT, x)); mtOptions.set(MATERIALIZED_TABLE_DEFINITION_QUERY, mt.getDefinitionQuery()); mtOptions.set( MATERIALIZED_TABLE_INTERVAL_FRESHNESS, mt.getDefinitionFreshness().getInterval()); @@ -496,17 +597,12 @@ private List toSchemaChange( if (!oldTableNonPhysicalColumnIndex.containsKey( ((ModifyPhysicalColumnType) change).getOldColumn().getName())) { ModifyPhysicalColumnType modify = (ModifyPhysicalColumnType) change; - LogicalType newColumnType = modify.getNewType().getLogicalType(); - LogicalType oldColumnType = modify.getOldColumn().getDataType().getLogicalType(); - if (newColumnType.isNullable() != oldColumnType.isNullable()) { - schemaChanges.add( - SchemaChange.updateColumnNullability( - modify.getNewColumn().getName(), newColumnType.isNullable())); - } - schemaChanges.add( - SchemaChange.updateColumnType( - modify.getOldColumn().getName(), - LogicalTypeConversion.toDataType(newColumnType))); + generateNestedColumnUpdates( + Collections.singletonList(modify.getOldColumn().getName()), + LogicalTypeConversion.toDataType( + modify.getOldColumn().getDataType().getLogicalType()), + LogicalTypeConversion.toDataType(modify.getNewType().getLogicalType()), + schemaChanges); } return schemaChanges; } else if (change instanceof ModifyColumnPosition) { @@ -571,6 +667,139 @@ && handleMaterializedTableChange(change, schemaChanges)) { throw new UnsupportedOperationException("Change is not supported: " + change.getClass()); } + private void generateNestedColumnUpdates( + List fieldNames, + org.apache.paimon.types.DataType oldType, + org.apache.paimon.types.DataType newType, + List schemaChanges) { + String joinedNames = String.join(".", fieldNames); + if (oldType.getTypeRoot() == DataTypeRoot.ROW) { + Preconditions.checkArgument( + newType.getTypeRoot() == DataTypeRoot.ROW, + "Column %s can only be updated to row type, and cannot be updated to %s type", + joinedNames, + newType.getTypeRoot()); + org.apache.paimon.types.RowType oldRowType = (org.apache.paimon.types.RowType) oldType; + org.apache.paimon.types.RowType newRowType = (org.apache.paimon.types.RowType) newType; + + // check that existing fields have same order + Map oldFieldOrders = new HashMap<>(); + for (int i = 0; i < oldRowType.getFieldCount(); i++) { + oldFieldOrders.put(oldRowType.getFields().get(i).name(), i); + } + int lastIdx = -1; + String lastFieldName = ""; + for (DataField newField : newRowType.getFields()) { + String name = newField.name(); + if (oldFieldOrders.containsKey(name)) { + int idx = oldFieldOrders.get(name); + Preconditions.checkState( + lastIdx < idx, + "Order of existing fields in column %s must be kept the same. " + + "However, field %s and %s have changed their orders.", + joinedNames, + lastFieldName, + name); + lastIdx = idx; + lastFieldName = name; + } + } + + // drop fields + Set newFieldNames = new HashSet<>(newRowType.getFieldNames()); + for (String name : oldRowType.getFieldNames()) { + if (!newFieldNames.contains(name)) { + List dropColumnNames = new ArrayList<>(fieldNames); + dropColumnNames.add(name); + schemaChanges.add( + SchemaChange.dropColumn(dropColumnNames.toArray(new String[0]))); + } + } + + for (int i = 0; i < newRowType.getFieldCount(); i++) { + DataField field = newRowType.getFields().get(i); + String name = field.name(); + List fullFieldNames = new ArrayList<>(fieldNames); + fullFieldNames.add(name); + if (!oldFieldOrders.containsKey(name)) { + // add fields + SchemaChange.Move move; + if (i == 0) { + move = SchemaChange.Move.first(name); + } else { + String lastName = newRowType.getFields().get(i - 1).name(); + move = SchemaChange.Move.after(name, lastName); + } + schemaChanges.add( + SchemaChange.addColumn( + fullFieldNames.toArray(new String[0]), + field.type(), + field.description(), + move)); + } else { + // update existing fields + DataField oldField = oldRowType.getFields().get(oldFieldOrders.get(name)); + if (!Objects.equals(oldField.description(), field.description())) { + schemaChanges.add( + SchemaChange.updateColumnComment( + fullFieldNames.toArray(new String[0]), + field.description())); + } + generateNestedColumnUpdates( + fullFieldNames, oldField.type(), field.type(), schemaChanges); + } + } + } else if (oldType.getTypeRoot() == DataTypeRoot.ARRAY) { + Preconditions.checkArgument( + newType.getTypeRoot() == DataTypeRoot.ARRAY, + "Column %s can only be updated to array type, and cannot be updated to %s type", + joinedNames, + newType); + List fullFieldNames = new ArrayList<>(fieldNames); + // add a dummy column name indicating the element of array + fullFieldNames.add("element"); + generateNestedColumnUpdates( + fullFieldNames, + ((org.apache.paimon.types.ArrayType) oldType).getElementType(), + ((org.apache.paimon.types.ArrayType) newType).getElementType(), + schemaChanges); + } else if (oldType.getTypeRoot() == DataTypeRoot.MAP) { + Preconditions.checkArgument( + newType.getTypeRoot() == DataTypeRoot.MAP, + "Column %s can only be updated to map type, and cannot be updated to %s type", + joinedNames, + newType); + org.apache.paimon.types.MapType oldMapType = (org.apache.paimon.types.MapType) oldType; + org.apache.paimon.types.MapType newMapType = (org.apache.paimon.types.MapType) newType; + Preconditions.checkArgument( + oldMapType.getKeyType().equals(newMapType.getKeyType()), + "Cannot update key type of column %s from %s type to %s type", + joinedNames, + oldMapType.getKeyType(), + newMapType.getKeyType()); + List fullFieldNames = new ArrayList<>(fieldNames); + // add a dummy column name indicating the value of map + fullFieldNames.add("value"); + generateNestedColumnUpdates( + fullFieldNames, + oldMapType.getValueType(), + newMapType.getValueType(), + schemaChanges); + } else { + if (!oldType.equalsIgnoreNullable(newType)) { + schemaChanges.add( + SchemaChange.updateColumnType( + fieldNames.toArray(new String[0]), newType, false)); + } + } + + if (oldType.isNullable() != newType.isNullable()) { + schemaChanges.add( + SchemaChange.updateColumnNullability( + fieldNames.toArray(new String[0]), newType.isNullable())); + } + } + /** * Try handle change related to materialized table. * @@ -669,7 +898,9 @@ public void alterTable( throw new TableNotExistException(getName(), tablePath); } - Preconditions.checkArgument(table instanceof FileStoreTable, "Can't alter system table."); + checkArgument( + table instanceof FileStoreTable, + "Only support alter data table, but is: " + table.getClass()); validateAlterTable(toCatalogTable(table), newTable); Map oldTableNonPhysicalColumnIndex = FlinkCatalogPropertiesUtil.nonPhysicalColumns( @@ -776,18 +1007,18 @@ private static void validateAlterTable(CatalogBaseTable ct1, CatalogBaseTable ct } // materialized table is not resolved at this time. if (!table1IsMaterialized) { - org.apache.flink.table.api.TableSchema ts1 = ct1.getSchema(); - org.apache.flink.table.api.TableSchema ts2 = ct2.getSchema(); + org.apache.flink.table.api.Schema ts1 = ct1.getUnresolvedSchema(); + org.apache.flink.table.api.Schema ts2 = ct2.getUnresolvedSchema(); boolean pkEquality = false; if (ts1.getPrimaryKey().isPresent() && ts2.getPrimaryKey().isPresent()) { pkEquality = Objects.equals( - ts1.getPrimaryKey().get().getType(), - ts2.getPrimaryKey().get().getType()) + ts1.getPrimaryKey().get().getConstraintName(), + ts2.getPrimaryKey().get().getConstraintName()) && Objects.equals( - ts1.getPrimaryKey().get().getColumns(), - ts2.getPrimaryKey().get().getColumns()); + ts1.getPrimaryKey().get().getColumnNames(), + ts2.getPrimaryKey().get().getColumnNames()); } else if (!ts1.getPrimaryKey().isPresent() && !ts2.getPrimaryKey().isPresent()) { pkEquality = true; } @@ -831,7 +1062,8 @@ public final void close() throws CatalogException { private CatalogBaseTable toCatalogTable(Table table) { Map newOptions = new HashMap<>(table.options()); - TableSchema.Builder builder = TableSchema.builder(); + org.apache.flink.table.api.Schema.Builder builder = + org.apache.flink.table.api.Schema.newBuilder(); Map nonPhysicalColumnComments = new HashMap<>(); // add columns @@ -846,10 +1078,10 @@ private CatalogBaseTable toCatalogTable(Table table) { if (optionalName == null || physicalColumns.contains(optionalName)) { // build physical column from table row field RowType.RowField field = physicalRowFields.get(physicalColumnIndex++); - builder.field(field.getName(), fromLogicalToDataType(field.getType())); + builder.column(field.getName(), fromLogicalToDataType(field.getType())); } else { // build non-physical column from options - builder.add(deserializeNonPhysicalColumn(newOptions, i)); + deserializeNonPhysicalColumn(newOptions, i, builder); if (newOptions.containsKey(compoundKey(SCHEMA, i, COMMENT))) { nonPhysicalColumnComments.put( optionalName, newOptions.get(compoundKey(SCHEMA, i, COMMENT))); @@ -861,22 +1093,18 @@ private CatalogBaseTable toCatalogTable(Table table) { // extract watermark information if (newOptions.keySet().stream() .anyMatch(key -> key.startsWith(compoundKey(SCHEMA, WATERMARK)))) { - builder.watermark(deserializeWatermarkSpec(newOptions)); + deserializeWatermarkSpec(newOptions, builder); } // add primary keys if (table.primaryKeys().size() > 0) { - builder.primaryKey( - table.primaryKeys().stream().collect(Collectors.joining("_", "PK_", "")), - table.primaryKeys().toArray(new String[0])); + builder.primaryKey(table.primaryKeys()); } - TableSchema schema = builder.build(); + org.apache.flink.table.api.Schema schema = builder.build(); // remove schema from options - DescriptorProperties removeProperties = new DescriptorProperties(false); - removeProperties.putTableSchema(SCHEMA, schema); - removeProperties.asMap().keySet().forEach(newOptions::remove); + FlinkDescriptorProperties.removeSchemaKeys(SCHEMA, schema, newOptions); Options options = Options.fromMap(newOptions); if (TableType.MATERIALIZED_TABLE == options.get(CoreOptions.TYPE)) { @@ -892,8 +1120,10 @@ private CatalogBaseTable toCatalogTable(Table table) { } private CatalogMaterializedTable buildMaterializedTable( - Table table, Map newOptions, TableSchema schema, Options options) { - Long snapshot = options.get(MATERIALIZED_TABLE_SNAPSHOT); + Table table, + Map newOptions, + org.apache.flink.table.api.Schema schema, + Options options) { String definitionQuery = options.get(MATERIALIZED_TABLE_DEFINITION_QUERY); IntervalFreshness freshness = IntervalFreshness.of( @@ -917,11 +1147,10 @@ private CatalogMaterializedTable buildMaterializedTable( // remove materialized table related options allMaterializedTableAttributes().forEach(newOptions::remove); return CatalogMaterializedTable.newBuilder() - .schema(schema.toSchema()) + .schema(schema) .comment(table.comment().orElse("")) .partitionKeys(table.partitionKeys()) .options(newOptions) - .snapshot(snapshot) .definitionQuery(definitionQuery) .freshness(freshness) .logicalRefreshMode(logicalRefreshMode) @@ -1011,15 +1240,27 @@ public final void renameTable( try { catalog.renameTable(toIdentifier(tablePath), toIdentifier(toTable), ignoreIfNotExists); } catch (Catalog.TableNotExistException e) { - throw new TableNotExistException(getName(), tablePath); + try { + catalog.renameView( + toIdentifier(tablePath), toIdentifier(toTable), ignoreIfNotExists); + } catch (Catalog.ViewNotExistException ex) { + throw new TableNotExistException(getName(), tablePath); + } catch (Catalog.ViewAlreadyExistException ex) { + throw new TableAlreadyExistException(getName(), toTable); + } } catch (Catalog.TableAlreadyExistException e) { throw new TableAlreadyExistException(getName(), toTable); } } @Override - public final List listViews(String databaseName) throws CatalogException { - return Collections.emptyList(); + public final List listViews(String databaseName) + throws DatabaseNotExistException, CatalogException { + try { + return catalog.listViews(databaseName); + } catch (Catalog.DatabaseNotExistException e) { + throw new DatabaseNotExistException(getName(), databaseName); + } } @Override @@ -1068,9 +1309,12 @@ private List getPartitionSpecs( getPartitionEntries(table, tablePath, partitionSpec); org.apache.paimon.types.RowType partitionRowType = table.schema().logicalPartitionType(); + CoreOptions options = new CoreOptions(table.options()); InternalRowPartitionComputer partitionComputer = FileStorePathFactory.getPartitionComputer( - partitionRowType, new CoreOptions(table.options()).partitionDefaultName()); + partitionRowType, + options.partitionDefaultName(), + options.legacyPartitionName()); return partitionEntries.stream() .map( @@ -1111,11 +1355,11 @@ public CatalogPartition getPartition(ObjectPath tablePath, CatalogPartitionSpec // This was already filtered by the expected partition. PartitionEntry partitionEntry = partitionEntries.get(0); Map properties = new HashMap<>(); - properties.put(NUM_ROWS_KEY, String.valueOf(partitionEntry.recordCount())); + properties.put(NUM_ROWS_PROP, String.valueOf(partitionEntry.recordCount())); properties.put( - LAST_UPDATE_TIME_KEY, String.valueOf(partitionEntry.lastFileCreationTime())); - properties.put(NUM_FILES_KEY, String.valueOf(partitionEntry.fileCount())); - properties.put(TOTAL_SIZE_KEY, String.valueOf(partitionEntry.fileSizeInBytes())); + LAST_UPDATE_TIME_PROP, String.valueOf(partitionEntry.lastFileCreationTime())); + properties.put(NUM_FILES_PROP, String.valueOf(partitionEntry.fileCount())); + properties.put(TOTAL_SIZE_PROP, String.valueOf(partitionEntry.fileSizeInBytes())); return new CatalogPartitionImpl(properties, ""); } catch (TableNotPartitionedException | TableNotExistException e) { throw new PartitionNotExistException(getName(), tablePath, partitionSpec); @@ -1139,8 +1383,19 @@ public final void createPartition( CatalogPartitionSpec partitionSpec, CatalogPartition partition, boolean ignoreIfExists) - throws CatalogException { - throw new UnsupportedOperationException(); + throws CatalogException, PartitionAlreadyExistsException { + if (partitionExists(tablePath, partitionSpec)) { + if (!ignoreIfExists) { + throw new PartitionAlreadyExistsException(getName(), tablePath, partitionSpec); + } + } + + try { + Identifier identifier = toIdentifier(tablePath); + catalog.createPartition(identifier, partitionSpec.getPartitionSpec()); + } catch (Catalog.TableNotExistException e) { + throw new CatalogException(e); + } } @Override @@ -1239,8 +1494,9 @@ public final CatalogColumnStatistics getPartitionColumnStatistics( @Override public final void alterTableStatistics( ObjectPath tablePath, CatalogTableStatistics tableStatistics, boolean ignoreIfNotExists) - throws CatalogException { - throw new UnsupportedOperationException(); + throws CatalogException, TableNotExistException { + alterTableStatisticsInternal( + tablePath, t -> createTableStats(t, tableStatistics), ignoreIfNotExists); } @Override @@ -1248,8 +1504,38 @@ public final void alterTableColumnStatistics( ObjectPath tablePath, CatalogColumnStatistics columnStatistics, boolean ignoreIfNotExists) - throws CatalogException { - throw new UnsupportedOperationException(); + throws CatalogException, TableNotExistException { + alterTableStatisticsInternal( + tablePath, t -> createTableColumnStats(t, columnStatistics), ignoreIfNotExists); + } + + private void alterTableStatisticsInternal( + ObjectPath tablePath, + Function statistics, + boolean ignoreIfNotExists) + throws TableNotExistException { + try { + Table table = catalog.getTable(toIdentifier(tablePath)); + checkArgument( + table instanceof FileStoreTable, "Now only support analyze FileStoreTable."); + if (!table.latestSnapshotId().isPresent()) { + LOG.info("Skipping analyze table because the snapshot is null."); + return; + } + + FileStoreTable storeTable = (FileStoreTable) table; + Statistics tableStats = statistics.apply(storeTable); + if (tableStats != null) { + String commitUser = storeTable.coreOptions().createCommitUser(); + try (FileStoreCommit commit = storeTable.store().newCommit(commitUser)) { + commit.commitStatistics(tableStats, BatchWriteBuilder.COMMIT_IDENTIFIER); + } + } + } catch (Catalog.TableNotExistException e) { + if (!ignoreIfNotExists) { + throw new TableNotExistException(getName(), tablePath); + } + } } @Override @@ -1305,7 +1591,6 @@ private boolean isCalledFromFlinkRecomputeStatisticsProgram() { private List allMaterializedTableAttributes() { return Arrays.asList( - MATERIALIZED_TABLE_SNAPSHOT.key(), MATERIALIZED_TABLE_DEFINITION_QUERY.key(), MATERIALIZED_TABLE_INTERVAL_FRESHNESS.key(), MATERIALIZED_TABLE_INTERVAL_FRESHNESS_TIME_UNIT.key(), diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/FlinkConnectorOptions.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/FlinkConnectorOptions.java index d181d7b5a0c6..5716cfca1baa 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/FlinkConnectorOptions.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/FlinkConnectorOptions.java @@ -44,7 +44,7 @@ public class FlinkConnectorOptions { public static final String NONE = "none"; - public static final String TABLE_DYNAMIC_OPTION_PREFIX = "paimon"; + public static final String TABLE_DYNAMIC_OPTION_PREFIX = "paimon."; public static final int MIN_CLUSTERING_SAMPLE_FACTOR = 20; @@ -311,6 +311,15 @@ public class FlinkConnectorOptions { .withDescription( "If the pending snapshot count exceeds the threshold, lookup operator will refresh the table in sync."); + public static final ConfigOption LOOKUP_REFRESH_TIME_PERIODS_BLACKLIST = + ConfigOptions.key("lookup.refresh.time-periods-blacklist") + .stringType() + .noDefaultValue() + .withDescription( + "The blacklist contains several time periods. During these time periods, the lookup table's " + + "cache refreshing is forbidden. Blacklist format is start1->end1,start2->end2,... , " + + "and the time format is yyyy-MM-dd HH:mm. Only used when lookup table is FULL cache mode."); + public static final ConfigOption SINK_AUTO_TAG_FOR_SAVEPOINT = ConfigOptions.key("sink.savepoint.auto-tag") .booleanType() @@ -355,12 +364,13 @@ public class FlinkConnectorOptions { "You can specify time interval for partition, for example, " + "daily partition is '1 d', hourly partition is '1 h'."); - public static final ConfigOption PARTITION_MARK_DONE_WHEN_END_INPUT = - ConfigOptions.key("partition.end-input-to-done") - .booleanType() - .defaultValue(false) + public static final ConfigOption PARTITION_IDLE_TIME_TO_REPORT_STATISTIC = + key("partition.idle-time-to-report-statistic") + .durationType() + .defaultValue(Duration.ofHours(1)) .withDescription( - "Whether mark the done status to indicate that the data is ready when end input."); + "Set a time duration when a partition has no new data after this time duration, " + + "start to report the partition statistics to hms."); public static final ConfigOption CLUSTERING_COLUMNS = key("sink.clustering.by-columns") @@ -404,6 +414,33 @@ public class FlinkConnectorOptions { .withDescription( "Optional endInput watermark used in case of batch mode or bounded stream."); + public static final ConfigOption CHANGELOG_PRECOMMIT_COMPACT = + key("changelog.precommit-compact") + .booleanType() + .defaultValue(false) + .withDescription( + "If true, it will add a changelog compact coordinator and worker operator after the writer operator," + + "in order to compact several changelog files from the same partition into large ones, " + + "which can decrease the number of small files. "); + + public static final ConfigOption SOURCE_OPERATOR_UID_SUFFIX = + key("source.operator-uid.suffix") + .stringType() + .noDefaultValue() + .withDescription( + "Set the uid suffix for the source operators. After setting, the uid format is " + + "${UID_PREFIX}_${TABLE_NAME}_${USER_UID_SUFFIX}. If the uid suffix is not set, flink will " + + "automatically generate the operator uid, which may be incompatible when the topology changes."); + + public static final ConfigOption SINK_OPERATOR_UID_SUFFIX = + key("sink.operator-uid.suffix") + .stringType() + .noDefaultValue() + .withDescription( + "Set the uid suffix for the writer, dynamic bucket assigner and committer operators. The uid format is " + + "${UID_PREFIX}_${TABLE_NAME}_${USER_UID_SUFFIX}. If the uid suffix is not set, flink will " + + "automatically generate the operator uid, which may be incompatible when the topology changes."); + public static List> getOptions() { final Field[] fields = FlinkConnectorOptions.class.getFields(); final List> list = new ArrayList<>(fields.length); @@ -419,6 +456,11 @@ public static List> getOptions() { return list; } + public static String generateCustomUid( + String uidPrefix, String tableName, String userDefinedSuffix) { + return String.format("%s_%s_%s", uidPrefix, tableName, userDefinedSuffix); + } + /** The mode of lookup cache. */ public enum LookupCacheMode { /** Auto mode, try to use partial mode. */ diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/FlinkFileIO.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/FlinkFileIO.java index db66379f3108..617d25125f37 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/FlinkFileIO.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/FlinkFileIO.java @@ -29,10 +29,10 @@ import org.apache.flink.core.fs.FSDataOutputStream; import org.apache.flink.core.fs.FileSystem; import org.apache.flink.core.fs.FileSystem.WriteMode; -import org.apache.flink.core.fs.FileSystemKind; import java.io.IOException; import java.io.UncheckedIOException; +import java.util.Locale; /** Flink {@link FileIO} to use {@link FileSystem}. */ public class FlinkFileIO implements FileIO { @@ -48,7 +48,27 @@ public FlinkFileIO(Path path) { @Override public boolean isObjectStore() { try { - return path.getFileSystem().getKind() != FileSystemKind.FILE_SYSTEM; + FileSystem fs = path.getFileSystem(); + String scheme = fs.getUri().getScheme().toLowerCase(Locale.US); + + if (scheme.startsWith("s3") + || scheme.startsWith("emr") + || scheme.startsWith("oss") + || scheme.startsWith("wasb") + || scheme.startsWith("gs")) { + // the Amazon S3 storage or Aliyun OSS storage or Azure Blob Storage + // or Google Cloud Storage + return true; + } else if (scheme.startsWith("http") || scheme.startsWith("ftp")) { + // file servers instead of file systems + // they might actually be consistent, but we have no hard guarantees + // currently to rely on that + return true; + } else { + // the remainder should include hdfs, kosmos, ceph, ... + // this also includes federated HDFS (viewfs). + return false; + } } catch (IOException e) { throw new UncheckedIOException(e); } @@ -227,5 +247,10 @@ public Path getPath() { public long getModificationTime() { return status.getModificationTime(); } + + @Override + public long getAccessTime() { + return status.getAccessTime(); + } } } diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/FlinkGenericCatalog.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/FlinkGenericCatalog.java index f6c206a7f422..75af5917bb49 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/FlinkGenericCatalog.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/FlinkGenericCatalog.java @@ -48,7 +48,6 @@ import org.apache.flink.table.expressions.Expression; import org.apache.flink.table.factories.Factory; import org.apache.flink.table.factories.FunctionDefinitionFactory; -import org.apache.flink.table.factories.TableFactory; import org.apache.flink.table.procedures.Procedure; import java.util.List; @@ -86,11 +85,6 @@ public Optional getFactory() { new FlinkGenericTableFactory(paimon.getFactory().get(), flink.getFactory().get())); } - @Override - public Optional getTableFactory() { - return flink.getTableFactory(); - } - @Override public Optional getFunctionDefinitionFactory() { return flink.getFunctionDefinitionFactory(); @@ -498,7 +492,11 @@ public List listProcedures(String dbName) */ public Procedure getProcedure(ObjectPath procedurePath) throws ProcedureNotExistException, CatalogException { - return ProcedureUtil.getProcedure(paimon.catalog(), procedurePath) - .orElse(flink.getProcedure(procedurePath)); + Optional procedure = ProcedureUtil.getProcedure(paimon.catalog(), procedurePath); + if (procedure.isPresent()) { + return procedure.get(); + } else { + return flink.getProcedure(procedurePath); + } } } diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/FlinkGenericCatalogFactory.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/FlinkGenericCatalogFactory.java index dc2a0f06b648..7c3a13c6f377 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/FlinkGenericCatalogFactory.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/FlinkGenericCatalogFactory.java @@ -89,6 +89,7 @@ public static FlinkGenericCatalog createCatalog( ClassLoader cl, Map optionMap, String name, Catalog flinkCatalog) { Options options = Options.fromMap(optionMap); options.set(CatalogOptions.METASTORE, "hive"); + options.set(CatalogOptions.FORMAT_TABLE_ENABLED, false); FlinkCatalog paimon = new FlinkCatalog( org.apache.paimon.catalog.CatalogFactory.createCatalog( diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/FlinkTableFactory.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/FlinkTableFactory.java index 96c81fdb720c..d5c1ed043b56 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/FlinkTableFactory.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/FlinkTableFactory.java @@ -30,11 +30,20 @@ import org.apache.flink.table.connector.source.DynamicTableSource; import org.apache.flink.table.factories.DynamicTableFactory; +import javax.annotation.Nullable; + import static org.apache.paimon.CoreOptions.AUTO_CREATE; import static org.apache.paimon.flink.FlinkCatalogFactory.IDENTIFIER; /** A paimon {@link DynamicTableFactory} to create source and sink. */ public class FlinkTableFactory extends AbstractFlinkTableFactory { + public FlinkTableFactory() { + this(null); + } + + public FlinkTableFactory(@Nullable FlinkCatalog flinkCatalog) { + super(flinkCatalog); + } @Override public String factoryIdentifier() { diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/FormatCatalogTable.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/FormatCatalogTable.java index 95aff5d84796..2e944f930cbb 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/FormatCatalogTable.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/FormatCatalogTable.java @@ -20,7 +20,6 @@ import org.apache.paimon.table.FormatTable; -import org.apache.flink.connector.file.table.FileSystemTableFactory; import org.apache.flink.table.api.Schema; import org.apache.flink.table.catalog.CatalogTable; import org.apache.flink.table.connector.sink.DynamicTableSink; @@ -30,17 +29,16 @@ import java.util.Collections; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.Set; import static org.apache.flink.connector.file.table.FileSystemConnectorOptions.PATH; import static org.apache.flink.table.factories.FactoryUtil.CONNECTOR; import static org.apache.flink.table.factories.FactoryUtil.FORMAT; import static org.apache.flink.table.types.utils.TypeConversions.fromLogicalToDataType; import static org.apache.paimon.flink.LogicalTypeConversion.toLogicalType; +import static org.apache.paimon.table.FormatTableOptions.FIELD_DELIMITER; /** A {@link CatalogTable} to represent format table. */ public class FormatCatalogTable implements CatalogTable { @@ -83,18 +81,17 @@ public CatalogTable copy(Map map) { public Map getOptions() { if (cachedOptions == null) { cachedOptions = new HashMap<>(); - FileSystemTableFactory fileSystemFactory = new FileSystemTableFactory(); - Set validOptions = new HashSet<>(); - fileSystemFactory.requiredOptions().forEach(o -> validOptions.add(o.key())); - fileSystemFactory.optionalOptions().forEach(o -> validOptions.add(o.key())); String format = table.format().name().toLowerCase(); - table.options() - .forEach( - (k, v) -> { - if (validOptions.contains(k) || k.startsWith(format + ".")) { - cachedOptions.put(k, v); - } - }); + Map options = table.options(); + options.forEach( + (k, v) -> { + if (k.startsWith(format + ".")) { + cachedOptions.put(k, v); + } + }); + if (options.containsKey(FIELD_DELIMITER.key())) { + cachedOptions.put("csv.field-delimiter", options.get(FIELD_DELIMITER.key())); + } cachedOptions.put(CONNECTOR.key(), "filesystem"); cachedOptions.put(PATH.key(), table.location()); cachedOptions.put(FORMAT.key(), format); diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/SystemCatalogTable.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/SystemCatalogTable.java index d5d843d91bb1..f88a808713c2 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/SystemCatalogTable.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/SystemCatalogTable.java @@ -22,7 +22,6 @@ import org.apache.paimon.table.system.AuditLogTable; import org.apache.flink.table.api.Schema; -import org.apache.flink.table.api.WatermarkSpec; import org.apache.flink.table.catalog.CatalogTable; import org.apache.flink.table.types.utils.TypeConversions; @@ -32,11 +31,11 @@ import java.util.Map; import java.util.Optional; -import static org.apache.flink.table.descriptors.DescriptorProperties.WATERMARK; -import static org.apache.flink.table.descriptors.Schema.SCHEMA; import static org.apache.paimon.flink.LogicalTypeConversion.toLogicalType; +import static org.apache.paimon.flink.utils.FlinkCatalogPropertiesUtil.SCHEMA; import static org.apache.paimon.flink.utils.FlinkCatalogPropertiesUtil.compoundKey; import static org.apache.paimon.flink.utils.FlinkCatalogPropertiesUtil.deserializeWatermarkSpec; +import static org.apache.paimon.flink.utils.FlinkDescriptorProperties.WATERMARK; /** A {@link CatalogTable} to represent system table. */ public class SystemCatalogTable implements CatalogTable { @@ -60,11 +59,8 @@ public Schema getUnresolvedSchema() { Map newOptions = new HashMap<>(table.options()); if (newOptions.keySet().stream() .anyMatch(key -> key.startsWith(compoundKey(SCHEMA, WATERMARK)))) { - WatermarkSpec watermarkSpec = deserializeWatermarkSpec(newOptions); - return builder.watermark( - watermarkSpec.getRowtimeAttribute(), - watermarkSpec.getWatermarkExpr()) - .build(); + deserializeWatermarkSpec(newOptions, builder); + return builder.build(); } } return builder.build(); diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/ActionFactory.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/ActionFactory.java index 43719f715d9d..fbf8f12f49eb 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/ActionFactory.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/ActionFactory.java @@ -58,6 +58,10 @@ public interface ActionFactory extends Factory { String TIMESTAMPFORMATTER = "timestamp_formatter"; String EXPIRE_STRATEGY = "expire_strategy"; String TIMESTAMP_PATTERN = "timestamp_pattern"; + // Supports `full` and `minor`. + String COMPACT_STRATEGY = "compact_strategy"; + String MINOR = "minor"; + String FULL = "full"; Optional create(MultipleParameterToolAdapter params); diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/CloneAction.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/CloneAction.java index 2f90147eeb2a..bac030dd0496 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/CloneAction.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/CloneAction.java @@ -32,7 +32,7 @@ import org.apache.flink.streaming.api.datastream.DataStream; import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; -import org.apache.flink.streaming.api.functions.sink.DiscardingSink; +import org.apache.flink.streaming.api.functions.sink.v2.DiscardingSink; import java.util.HashMap; import java.util.Map; @@ -141,7 +141,7 @@ copyFiles, new SnapshotHintChannelComputer(), parallelism) new SnapshotHintOperator(targetCatalogConfig)) .setParallelism(parallelism); - snapshotHintOperator.addSink(new DiscardingSink<>()).name("end").setParallelism(1); + snapshotHintOperator.sinkTo(new DiscardingSink<>()).name("end").setParallelism(1); } @Override diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/CompactAction.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/CompactAction.java index 8ea120015609..84e37a5b10f9 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/CompactAction.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/CompactAction.java @@ -59,6 +59,8 @@ public class CompactAction extends TableActionBase { @Nullable private Duration partitionIdleTime = null; + private Boolean fullCompaction; + public CompactAction(String warehouse, String database, String tableName) { this(warehouse, database, tableName, Collections.emptyMap(), Collections.emptyMap()); } @@ -100,6 +102,11 @@ public CompactAction withPartitionIdleTime(@Nullable Duration partitionIdleTime) return this; } + public CompactAction withFullCompaction(Boolean fullCompaction) { + this.fullCompaction = fullCompaction; + return this; + } + @Override public void build() throws Exception { ReadableConfig conf = env.getConfiguration(); @@ -124,6 +131,13 @@ public void build() throws Exception { private void buildForTraditionalCompaction( StreamExecutionEnvironment env, FileStoreTable table, boolean isStreaming) throws Exception { + if (fullCompaction == null) { + fullCompaction = !isStreaming; + } else { + Preconditions.checkArgument( + !(fullCompaction && isStreaming), + "The full compact strategy is only supported in batch mode. Please add -Dexecution.runtime-mode=BATCH."); + } if (isStreaming) { // for completely asynchronous compaction HashMap dynamicOptions = @@ -138,7 +152,7 @@ private void buildForTraditionalCompaction( } CompactorSourceBuilder sourceBuilder = new CompactorSourceBuilder(identifier.getFullName(), table); - CompactorSinkBuilder sinkBuilder = new CompactorSinkBuilder(table); + CompactorSinkBuilder sinkBuilder = new CompactorSinkBuilder(table, fullCompaction); sourceBuilder.withPartitionPredicate(getPredicate()); DataStreamSource source = diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/CompactActionFactory.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/CompactActionFactory.java index f43c7a747c99..fc60a870eabe 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/CompactActionFactory.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/CompactActionFactory.java @@ -76,6 +76,10 @@ public Optional create(MultipleParameterToolAdapter params) { action.withPartitionIdleTime( TimeUtils.parseDuration(params.get(PARTITION_IDLE_TIME))); } + String compactStrategy = params.get(COMPACT_STRATEGY); + if (checkCompactStrategy(compactStrategy)) { + action.withFullCompaction(compactStrategy.trim().equalsIgnoreCase(FULL)); + } } if (params.has(PARTITION)) { @@ -88,6 +92,19 @@ public Optional create(MultipleParameterToolAdapter params) { return Optional.of(action); } + public static boolean checkCompactStrategy(String compactStrategy) { + if (compactStrategy != null) { + Preconditions.checkArgument( + compactStrategy.equalsIgnoreCase(MINOR) + || compactStrategy.equalsIgnoreCase(FULL), + String.format( + "The compact strategy only supports 'full' or 'minor', but '%s' is configured.", + compactStrategy)); + return true; + } + return false; + } + @Override public void printHelp() { System.out.println( @@ -101,7 +118,8 @@ public void printHelp() { + "[--order_strategy ]" + "[--table_conf =]" + "[--order_by ]" - + "[--partition_idle_time ]"); + + "[--partition_idle_time ]" + + "[--compact_strategy ]"); System.out.println( " compact --warehouse s3://path/to/warehouse --database " + "--table [--catalog_conf [--catalog_conf ...]]"); @@ -132,6 +150,10 @@ public void printHelp() { System.out.println( " compact --warehouse hdfs:///path/to/warehouse --database test_db --table test_table " + "--partition_idle_time 10s"); + System.out.println( + "--compact_strategy determines how to pick files to be merged, the default is determined by the runtime execution mode. " + + "`full` : Only supports batch mode. All files will be selected for merging." + + "`minor`: Pick the set of files that need to be merged based on specified conditions."); System.out.println( " compact --warehouse s3:///path/to/warehouse " + "--database test_db " diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/CompactDatabaseAction.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/CompactDatabaseAction.java index ef6772e36eed..124d3ca68776 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/CompactDatabaseAction.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/CompactDatabaseAction.java @@ -54,6 +54,7 @@ import java.util.regex.Pattern; import static org.apache.paimon.flink.sink.FlinkStreamPartitioner.partition; +import static org.apache.paimon.flink.sink.FlinkStreamPartitioner.rebalance; /** Database compact action for Flink. */ public class CompactDatabaseAction extends ActionBase { @@ -71,6 +72,10 @@ public class CompactDatabaseAction extends ActionBase { @Nullable private Duration partitionIdleTime = null; + private Boolean fullCompaction; + + private boolean isStreaming; + public CompactDatabaseAction(String warehouse, Map catalogConfig) { super(warehouse, catalogConfig); } @@ -109,6 +114,11 @@ public CompactDatabaseAction withPartitionIdleTime(@Nullable Duration partitionI return this; } + public CompactDatabaseAction withFullCompaction(boolean fullCompaction) { + this.fullCompaction = fullCompaction; + return this; + } + private boolean shouldCompactionTable(String paimonFullTableName) { boolean shouldCompaction = includingPattern.matcher(paimonFullTableName).matches(); if (excludingPattern != null) { @@ -123,6 +133,12 @@ private boolean shouldCompactionTable(String paimonFullTableName) { @Override public void build() { + ReadableConfig conf = env.getConfiguration(); + isStreaming = conf.get(ExecutionOptions.RUNTIME_MODE) == RuntimeExecutionMode.STREAMING; + + if (fullCompaction == null) { + fullCompaction = !isStreaming; + } if (databaseCompactMode == MultiTablesSinkMode.DIVIDED) { buildForDividedMode(); } else { @@ -169,24 +185,19 @@ private void buildForDividedMode() { !tableMap.isEmpty(), "no tables to be compacted. possible cause is that there are no tables detected after pattern matching"); - ReadableConfig conf = env.getConfiguration(); - boolean isStreaming = - conf.get(ExecutionOptions.RUNTIME_MODE) == RuntimeExecutionMode.STREAMING; for (Map.Entry entry : tableMap.entrySet()) { FileStoreTable fileStoreTable = entry.getValue(); switch (fileStoreTable.bucketMode()) { case BUCKET_UNAWARE: { - buildForUnawareBucketCompaction( - env, entry.getKey(), fileStoreTable, isStreaming); + buildForUnawareBucketCompaction(env, entry.getKey(), fileStoreTable); break; } case HASH_FIXED: case HASH_DYNAMIC: default: { - buildForTraditionalCompaction( - env, entry.getKey(), fileStoreTable, isStreaming); + buildForTraditionalCompaction(env, entry.getKey(), fileStoreTable); } } } @@ -194,9 +205,6 @@ private void buildForDividedMode() { private void buildForCombinedMode() { - ReadableConfig conf = env.getConfiguration(); - boolean isStreaming = - conf.get(ExecutionOptions.RUNTIME_MODE) == RuntimeExecutionMode.STREAMING; CombinedTableCompactorSourceBuilder sourceBuilder = new CombinedTableCompactorSourceBuilder( catalogLoader(), @@ -208,6 +216,11 @@ private void buildForCombinedMode() { .toMillis()) .withPartitionIdleTime(partitionIdleTime); + Integer parallelism = + tableOptions.get(FlinkConnectorOptions.SINK_PARALLELISM) == null + ? env.getParallelism() + : tableOptions.get(FlinkConnectorOptions.SINK_PARALLELISM); + // multi bucket table which has multi bucket in a partition like fix bucket and dynamic // bucket DataStream awareBucketTableSource = @@ -217,24 +230,28 @@ private void buildForCombinedMode() { .withContinuousMode(isStreaming) .buildAwareBucketTableSource(), new BucketsRowChannelComputer(), - tableOptions.get(FlinkConnectorOptions.SINK_PARALLELISM)); + parallelism); // unaware bucket table DataStream unawareBucketTableSource = - sourceBuilder - .withEnv(env) - .withContinuousMode(isStreaming) - .buildForUnawareBucketsTableSource(); + rebalance( + sourceBuilder + .withEnv(env) + .withContinuousMode(isStreaming) + .buildForUnawareBucketsTableSource(), + parallelism); - new CombinedTableCompactorSink(catalogLoader(), tableOptions) + new CombinedTableCompactorSink(catalogLoader(), tableOptions, fullCompaction) .sinkFrom(awareBucketTableSource, unawareBucketTableSource); } private void buildForTraditionalCompaction( - StreamExecutionEnvironment env, - String fullName, - FileStoreTable table, - boolean isStreaming) { + StreamExecutionEnvironment env, String fullName, FileStoreTable table) { + + Preconditions.checkArgument( + !(fullCompaction && isStreaming), + "The full compact strategy is only supported in batch mode. Please add -Dexecution.runtime-mode=BATCH."); + if (isStreaming) { // for completely asynchronous compaction HashMap dynamicOptions = @@ -251,7 +268,7 @@ private void buildForTraditionalCompaction( CompactorSourceBuilder sourceBuilder = new CompactorSourceBuilder(fullName, table) .withPartitionIdleTime(partitionIdleTime); - CompactorSinkBuilder sinkBuilder = new CompactorSinkBuilder(table); + CompactorSinkBuilder sinkBuilder = new CompactorSinkBuilder(table, fullCompaction); DataStreamSource source = sourceBuilder.withEnv(env).withContinuousMode(isStreaming).build(); @@ -259,10 +276,7 @@ private void buildForTraditionalCompaction( } private void buildForUnawareBucketCompaction( - StreamExecutionEnvironment env, - String fullName, - FileStoreTable table, - boolean isStreaming) { + StreamExecutionEnvironment env, String fullName, FileStoreTable table) { UnawareBucketCompactionTopoBuilder unawareBucketCompactionTopoBuilder = new UnawareBucketCompactionTopoBuilder(env, fullName, table); diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/CompactDatabaseActionFactory.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/CompactDatabaseActionFactory.java index b26870907809..5672f99dc30f 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/CompactDatabaseActionFactory.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/CompactDatabaseActionFactory.java @@ -22,6 +22,8 @@ import java.util.Optional; +import static org.apache.paimon.flink.action.CompactActionFactory.checkCompactStrategy; + /** Factory to create {@link CompactDatabaseAction}. */ public class CompactDatabaseActionFactory implements ActionFactory { @@ -55,6 +57,11 @@ public Optional create(MultipleParameterToolAdapter params) { action.withPartitionIdleTime(TimeUtils.parseDuration(partitionIdleTime)); } + String compactStrategy = params.get(COMPACT_STRATEGY); + if (checkCompactStrategy(compactStrategy)) { + action.withFullCompaction(compactStrategy.trim().equalsIgnoreCase(FULL)); + } + return Optional.of(action); } @@ -70,7 +77,8 @@ public void printHelp() { + "[--including_tables ] " + "[--excluding_tables ] " + "[--mode ]" - + "[--partition_idle_time ]"); + + "[--partition_idle_time ]" + + "[--compact_strategy ]"); System.out.println( " compact_database --warehouse s3://path/to/warehouse --including_databases " + "[--catalog_conf [--catalog_conf ...]]"); @@ -93,6 +101,11 @@ public void printHelp() { System.out.println( "--partition_idle_time is used to do a full compaction for partition which had not receive any new data for 'partition_idle_time' time. And only these partitions will be compacted."); System.out.println("--partition_idle_time is only supported in batch mode. "); + System.out.println( + "--compact_strategy determines how to pick files to be merged, the default is determined by the runtime execution mode. " + + "`full` : Only supports batch mode. All files will be selected for merging." + + "`minor`: Pick the set of files that need to be merged based on specified conditions."); + System.out.println(); System.out.println("Examples:"); diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/CreateOrReplaceTagActionFactory.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/CreateOrReplaceTagActionFactory.java new file mode 100644 index 000000000000..fecb6895b682 --- /dev/null +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/CreateOrReplaceTagActionFactory.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.flink.action; + +import org.apache.paimon.utils.TimeUtils; + +import org.apache.flink.api.java.tuple.Tuple3; + +import java.time.Duration; +import java.util.Map; +import java.util.Optional; + +/** Factory to create {@link ReplaceTagAction} or {@link ReplaceTagAction}. */ +public abstract class CreateOrReplaceTagActionFactory implements ActionFactory { + + private static final String TAG_NAME = "tag_name"; + private static final String SNAPSHOT = "snapshot"; + private static final String TIME_RETAINED = "time_retained"; + + @Override + public Optional create(MultipleParameterToolAdapter params) { + checkRequiredArgument(params, TAG_NAME); + + Tuple3 tablePath = getTablePath(params); + Map catalogConfig = optionalConfigMap(params, CATALOG_CONF); + String tagName = params.get(TAG_NAME); + + Long snapshot = null; + if (params.has(SNAPSHOT)) { + snapshot = Long.parseLong(params.get(SNAPSHOT)); + } + + Duration timeRetained = null; + if (params.has(TIME_RETAINED)) { + timeRetained = TimeUtils.parseDuration(params.get(TIME_RETAINED)); + } + + return Optional.of( + createOrReplaceTagAction( + tablePath, catalogConfig, tagName, snapshot, timeRetained)); + } + + abstract Action createOrReplaceTagAction( + Tuple3 tablePath, + Map catalogConfig, + String tagName, + Long snapshot, + Duration timeRetained); +} diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/CreateTagActionFactory.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/CreateTagActionFactory.java index 7769fa1d792f..c525943122bc 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/CreateTagActionFactory.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/CreateTagActionFactory.java @@ -18,56 +18,36 @@ package org.apache.paimon.flink.action; -import org.apache.paimon.utils.TimeUtils; - import org.apache.flink.api.java.tuple.Tuple3; import java.time.Duration; import java.util.Map; -import java.util.Optional; /** Factory to create {@link CreateTagAction}. */ -public class CreateTagActionFactory implements ActionFactory { +public class CreateTagActionFactory extends CreateOrReplaceTagActionFactory { public static final String IDENTIFIER = "create_tag"; - private static final String TAG_NAME = "tag_name"; - private static final String SNAPSHOT = "snapshot"; - private static final String TIME_RETAINED = "time_retained"; - @Override public String identifier() { return IDENTIFIER; } @Override - public Optional create(MultipleParameterToolAdapter params) { - checkRequiredArgument(params, TAG_NAME); - - Tuple3 tablePath = getTablePath(params); - Map catalogConfig = optionalConfigMap(params, CATALOG_CONF); - String tagName = params.get(TAG_NAME); - - Long snapshot = null; - if (params.has(SNAPSHOT)) { - snapshot = Long.parseLong(params.get(SNAPSHOT)); - } - - Duration timeRetained = null; - if (params.has(TIME_RETAINED)) { - timeRetained = TimeUtils.parseDuration(params.get(TIME_RETAINED)); - } - - CreateTagAction action = - new CreateTagAction( - tablePath.f0, - tablePath.f1, - tablePath.f2, - catalogConfig, - tagName, - snapshot, - timeRetained); - return Optional.of(action); + Action createOrReplaceTagAction( + Tuple3 tablePath, + Map catalogConfig, + String tagName, + Long snapshot, + Duration timeRetained) { + return new CreateTagAction( + tablePath.f0, + tablePath.f1, + tablePath.f2, + catalogConfig, + tagName, + snapshot, + timeRetained); } @Override diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/ExpirePartitionsAction.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/ExpirePartitionsAction.java index 9528bc137d6f..0fa17e1a8ddb 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/ExpirePartitionsAction.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/ExpirePartitionsAction.java @@ -72,7 +72,8 @@ public ExpirePartitionsAction( .catalogEnvironment() .metastoreClientFactory()) .map(MetastoreClient.Factory::create) - .orElse(null)); + .orElse(null), + fileStore.options().partitionExpireMaxNum()); } @Override diff --git a/paimon-common/src/main/java/org/apache/paimon/lineage/TableLineageEntityImpl.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/ExpireTagsAction.java similarity index 53% rename from paimon-common/src/main/java/org/apache/paimon/lineage/TableLineageEntityImpl.java rename to paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/ExpireTagsAction.java index ef11ee87f15c..c1231ed3ad54 100644 --- a/paimon-common/src/main/java/org/apache/paimon/lineage/TableLineageEntityImpl.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/ExpireTagsAction.java @@ -16,41 +16,31 @@ * limitations under the License. */ -package org.apache.paimon.lineage; +package org.apache.paimon.flink.action; -import org.apache.paimon.data.Timestamp; +import org.apache.paimon.flink.procedure.ExpireTagsProcedure; -/** Default implementation for {@link TableLineageEntity}. */ -public class TableLineageEntityImpl implements TableLineageEntity { - private final String database; - private final String table; - private final String job; - private final Timestamp timestamp; +import org.apache.flink.table.procedure.DefaultProcedureContext; - public TableLineageEntityImpl(String database, String table, String job, Timestamp timestamp) { - this.database = database; - this.table = table; - this.job = job; - this.timestamp = timestamp; - } +import java.util.Map; - @Override - public String getDatabase() { - return database; - } +/** Expire tags action for Flink. */ +public class ExpireTagsAction extends ActionBase { - @Override - public String getTable() { - return table; - } + private final String table; + private final String olderThan; - @Override - public String getJob() { - return job; + public ExpireTagsAction( + String warehouse, String table, String olderThan, Map catalogConfig) { + super(warehouse, catalogConfig); + this.table = table; + this.olderThan = olderThan; } @Override - public Timestamp getCreateTime() { - return timestamp; + public void run() throws Exception { + ExpireTagsProcedure expireTagsProcedure = new ExpireTagsProcedure(); + expireTagsProcedure.withCatalog(catalog); + expireTagsProcedure.call(new DefaultProcedureContext(env), table, olderThan); } } diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/ExpireTagsActionFactory.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/ExpireTagsActionFactory.java new file mode 100644 index 000000000000..e9bbb0a3bdc7 --- /dev/null +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/ExpireTagsActionFactory.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.flink.action; + +import java.util.Map; +import java.util.Optional; + +/** Factory to create {@link ExpireTagsAction}. */ +public class ExpireTagsActionFactory implements ActionFactory { + + private static final String IDENTIFIER = "expire_tags"; + + private static final String OLDER_THAN = "older_than"; + + @Override + public String identifier() { + return IDENTIFIER; + } + + @Override + public Optional create(MultipleParameterToolAdapter params) { + String warehouse = params.get(WAREHOUSE); + String table = params.get(TABLE); + String olderThan = params.get(OLDER_THAN); + Map catalogConfig = optionalConfigMap(params, CATALOG_CONF); + + ExpireTagsAction expireTagsAction = + new ExpireTagsAction(warehouse, table, olderThan, catalogConfig); + return Optional.of(expireTagsAction); + } + + @Override + public void printHelp() { + System.out.println("Action \"expire_tags\" expire tags by time."); + System.out.println(); + + System.out.println("Syntax:"); + System.out.println( + " expire_tags --warehouse " + + "--table " + + "[--older_than ]"); + } +} diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/FlinkActions.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/FlinkActions.java index 3ed70ddcc42b..03d751d5f167 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/FlinkActions.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/FlinkActions.java @@ -22,12 +22,7 @@ import static org.apache.paimon.flink.action.ActionFactory.printDefaultHelp; -/** - * Table maintenance actions for Flink. - * - * @deprecated Compatible with older versions of usage - */ -@Deprecated +/** Table maintenance actions for Flink. */ public class FlinkActions { // ------------------------------------------------------------------------ diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/ReplaceTagAction.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/ReplaceTagAction.java new file mode 100644 index 000000000000..09a85fe8a25a --- /dev/null +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/ReplaceTagAction.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.flink.action; + +import javax.annotation.Nullable; + +import java.time.Duration; +import java.util.Map; + +/** Replace tag action for Flink. */ +public class ReplaceTagAction extends TableActionBase { + + private final String tagName; + private final @Nullable Long snapshotId; + private final @Nullable Duration timeRetained; + + public ReplaceTagAction( + String warehouse, + String databaseName, + String tableName, + Map catalogConfig, + String tagName, + @Nullable Long snapshotId, + @Nullable Duration timeRetained) { + super(warehouse, databaseName, tableName, catalogConfig); + this.tagName = tagName; + this.timeRetained = timeRetained; + this.snapshotId = snapshotId; + } + + @Override + public void run() throws Exception { + table.replaceTag(tagName, snapshotId, timeRetained); + } +} diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/ReplaceTagActionFactory.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/ReplaceTagActionFactory.java new file mode 100644 index 000000000000..a734e9cfbdc5 --- /dev/null +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/ReplaceTagActionFactory.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.flink.action; + +import org.apache.flink.api.java.tuple.Tuple3; + +import java.time.Duration; +import java.util.Map; + +/** Factory to create {@link ReplaceTagAction}. */ +public class ReplaceTagActionFactory extends CreateOrReplaceTagActionFactory { + + public static final String IDENTIFIER = "replace_tag"; + + @Override + public String identifier() { + return IDENTIFIER; + } + + @Override + Action createOrReplaceTagAction( + Tuple3 tablePath, + Map catalogConfig, + String tagName, + Long snapshot, + Duration timeRetained) { + return new ReplaceTagAction( + tablePath.f0, + tablePath.f1, + tablePath.f2, + catalogConfig, + tagName, + snapshot, + timeRetained); + } + + @Override + public void printHelp() { + System.out.println("Action \"replace_tag\" to replace an existing tag with new tag info."); + System.out.println(); + + System.out.println("Syntax:"); + System.out.println( + " replace_tag --warehouse --database " + + "--table --tag_name [--snapshot ] [--time_retained ]"); + System.out.println(); + } +} diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/ResetConsumerAction.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/ResetConsumerAction.java index 615b448ec9bb..6db8ab4fef75 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/ResetConsumerAction.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/ResetConsumerAction.java @@ -57,6 +57,7 @@ public void run() throws Exception { if (Objects.isNull(nextSnapshotId)) { consumerManager.deleteConsumer(consumerId); } else { + dataTable.snapshotManager().snapshot(nextSnapshotId); consumerManager.resetConsumer(consumerId, new Consumer(nextSnapshotId)); } } diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/RollbackToTimestampAction.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/RollbackToTimestampAction.java new file mode 100644 index 000000000000..ca706101cc1d --- /dev/null +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/RollbackToTimestampAction.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.flink.action; + +import org.apache.paimon.Snapshot; +import org.apache.paimon.table.DataTable; +import org.apache.paimon.table.FileStoreTable; +import org.apache.paimon.utils.Preconditions; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Map; + +/** Rollback to specific timestamp action for Flink. */ +public class RollbackToTimestampAction extends TableActionBase { + + private static final Logger LOG = LoggerFactory.getLogger(RollbackToTimestampAction.class); + + private final Long timestamp; + + public RollbackToTimestampAction( + String warehouse, + String databaseName, + String tableName, + Long timestamp, + Map catalogConfig) { + super(warehouse, databaseName, tableName, catalogConfig); + this.timestamp = timestamp; + } + + @Override + public void run() throws Exception { + LOG.debug("Run rollback-to-timestamp action with timestamp '{}'.", timestamp); + + if (!(table instanceof DataTable)) { + throw new IllegalArgumentException("Unknown table: " + identifier); + } + + FileStoreTable fileStoreTable = (FileStoreTable) table; + Snapshot snapshot = fileStoreTable.snapshotManager().earlierOrEqualTimeMills(timestamp); + Preconditions.checkNotNull( + snapshot, String.format("count not find snapshot earlier than %s", timestamp)); + fileStoreTable.rollbackTo(snapshot.id()); + } +} diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/RollbackToTimestampActionFactory.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/RollbackToTimestampActionFactory.java new file mode 100644 index 000000000000..c694ac0041b5 --- /dev/null +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/RollbackToTimestampActionFactory.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.flink.action; + +import org.apache.flink.api.java.tuple.Tuple3; + +import java.util.Map; +import java.util.Optional; + +/** Factory to create {@link RollbackToTimestampAction}. */ +public class RollbackToTimestampActionFactory implements ActionFactory { + + public static final String IDENTIFIER = "rollback_to_timestamp"; + + private static final String TIMESTAMP = "timestamp"; + + @Override + public String identifier() { + return IDENTIFIER; + } + + @Override + public Optional create(MultipleParameterToolAdapter params) { + Tuple3 tablePath = getTablePath(params); + + checkRequiredArgument(params, TIMESTAMP); + String timestamp = params.get(TIMESTAMP); + + Map catalogConfig = optionalConfigMap(params, CATALOG_CONF); + + RollbackToTimestampAction action = + new RollbackToTimestampAction( + tablePath.f0, + tablePath.f1, + tablePath.f2, + Long.parseLong(timestamp), + catalogConfig); + + return Optional.of(action); + } + + @Override + public void printHelp() { + System.out.println( + "Action \"rollback_to_timestamp\" roll back a table to a specific timestamp."); + System.out.println(); + + System.out.println("Syntax:"); + System.out.println( + " rollback_to --warehouse --database " + + "--table --timestamp "); + System.out.println(" can be a long value representing a timestamp."); + System.out.println(); + } +} diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/clone/CloneSourceBuilder.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/clone/CloneSourceBuilder.java index a0f4ef33dee2..585c73cb952c 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/clone/CloneSourceBuilder.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/clone/CloneSourceBuilder.java @@ -34,6 +34,7 @@ import java.util.Map; import static org.apache.paimon.utils.Preconditions.checkArgument; +import static org.apache.paimon.utils.Preconditions.checkState; /** * Pick the tables to be cloned based on the user input parameters. The record type of the build @@ -114,6 +115,8 @@ private DataStream> build(Catalog sourceCatalog) throws E database + "." + tableName, targetDatabase + "." + targetTableName)); } + checkState(!result.isEmpty(), "Didn't find any table in source catalog."); + if (LOG.isDebugEnabled()) { LOG.debug("The clone identifiers of source table and target table are: {}", result); } diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/clone/PickFilesForCloneOperator.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/clone/PickFilesForCloneOperator.java index 883d7b06ab5f..67eecbc6f2ae 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/clone/PickFilesForCloneOperator.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/clone/PickFilesForCloneOperator.java @@ -18,15 +18,21 @@ package org.apache.paimon.flink.clone; +import org.apache.paimon.CoreOptions; import org.apache.paimon.catalog.Catalog; import org.apache.paimon.catalog.Identifier; import org.apache.paimon.flink.FlinkCatalogFactory; import org.apache.paimon.fs.Path; import org.apache.paimon.options.Options; import org.apache.paimon.schema.Schema; +import org.apache.paimon.schema.TableSchema; import org.apache.paimon.table.FileStoreTable; import org.apache.paimon.utils.Preconditions; +import org.apache.paimon.shade.guava30.com.google.common.collect.ImmutableList; +import org.apache.paimon.shade.guava30.com.google.common.collect.ImmutableMap; +import org.apache.paimon.shade.guava30.com.google.common.collect.Iterables; + import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.streaming.api.operators.AbstractStreamOperator; import org.apache.flink.streaming.api.operators.OneInputStreamOperator; @@ -37,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Objects; /** * Pick the files to be cloned of a table based on the input record. The record type it produce is @@ -77,7 +84,7 @@ public void processElement(StreamRecord> streamRecord) th FileStoreTable sourceTable = (FileStoreTable) sourceCatalog.getTable(sourceIdentifier); targetCatalog.createDatabase(targetIdentifier.getDatabaseName(), true); targetCatalog.createTable( - targetIdentifier, Schema.fromTableSchema(sourceTable.schema()), true); + targetIdentifier, newSchemaFromTableSchema(sourceTable.schema()), true); List result = toCloneFileInfos( @@ -95,6 +102,18 @@ public void processElement(StreamRecord> streamRecord) th } } + private static Schema newSchemaFromTableSchema(TableSchema tableSchema) { + return new Schema( + ImmutableList.copyOf(tableSchema.fields()), + ImmutableList.copyOf(tableSchema.partitionKeys()), + ImmutableList.copyOf(tableSchema.primaryKeys()), + ImmutableMap.copyOf( + Iterables.filter( + tableSchema.options().entrySet(), + entry -> !Objects.equals(entry.getKey(), CoreOptions.PATH.key()))), + tableSchema.comment()); + } + private List toCloneFileInfos( List files, Path sourceTableRoot, diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/compact/MultiAwareBucketTableScan.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/compact/MultiAwareBucketTableScan.java index 747995d20d67..88730132ef68 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/compact/MultiAwareBucketTableScan.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/compact/MultiAwareBucketTableScan.java @@ -32,7 +32,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.concurrent.atomic.AtomicBoolean; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -52,15 +51,8 @@ public MultiAwareBucketTableScan( Pattern includingPattern, Pattern excludingPattern, Pattern databasePattern, - boolean isStreaming, - AtomicBoolean isRunning) { - super( - catalogLoader, - includingPattern, - excludingPattern, - databasePattern, - isStreaming, - isRunning); + boolean isStreaming) { + super(catalogLoader, includingPattern, excludingPattern, databasePattern, isStreaming); tablesMap = new HashMap<>(); scansMap = new HashMap<>(); } diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/compact/MultiTableScanBase.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/compact/MultiTableScanBase.java index bd4ffe83a4ca..f5940740b691 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/compact/MultiTableScanBase.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/compact/MultiTableScanBase.java @@ -26,12 +26,11 @@ import org.apache.paimon.table.source.EndOfScanException; import org.apache.paimon.table.source.Split; -import org.apache.flink.streaming.api.functions.source.SourceFunction; +import org.apache.flink.api.connector.source.ReaderOutput; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.List; -import java.util.concurrent.atomic.AtomicBoolean; import java.util.regex.Pattern; import static org.apache.paimon.flink.utils.MultiTablesCompactorUtil.shouldCompactTable; @@ -57,7 +56,6 @@ public abstract class MultiTableScanBase implements AutoCloseable { protected transient Catalog catalog; - protected AtomicBoolean isRunning; protected boolean isStreaming; public MultiTableScanBase( @@ -65,14 +63,12 @@ public MultiTableScanBase( Pattern includingPattern, Pattern excludingPattern, Pattern databasePattern, - boolean isStreaming, - AtomicBoolean isRunning) { + boolean isStreaming) { catalog = catalogLoader.load(); this.includingPattern = includingPattern; this.excludingPattern = excludingPattern; this.databasePattern = databasePattern; - this.isRunning = isRunning; this.isStreaming = isStreaming; } @@ -104,13 +100,9 @@ protected void updateTableMap() } } - public ScanResult scanTable(SourceFunction.SourceContext ctx) + public ScanResult scanTable(ReaderOutput ctx) throws Catalog.TableNotExistException, Catalog.DatabaseNotExistException { try { - if (!isRunning.get()) { - return ScanResult.FINISHED; - } - updateTableMap(); List tasks = doScan(); diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/compact/MultiUnawareBucketTableScan.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/compact/MultiUnawareBucketTableScan.java index 56bf971240e7..da86b93af512 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/compact/MultiUnawareBucketTableScan.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/compact/MultiUnawareBucketTableScan.java @@ -29,7 +29,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.concurrent.atomic.AtomicBoolean; import java.util.regex.Pattern; /** @@ -46,15 +45,8 @@ public MultiUnawareBucketTableScan( Pattern includingPattern, Pattern excludingPattern, Pattern databasePattern, - boolean isStreaming, - AtomicBoolean isRunning) { - super( - catalogLoader, - includingPattern, - excludingPattern, - databasePattern, - isStreaming, - isRunning); + boolean isStreaming) { + super(catalogLoader, includingPattern, excludingPattern, databasePattern, isStreaming); tablesMap = new HashMap<>(); } diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/compact/UnawareBucketCompactionTopoBuilder.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/compact/UnawareBucketCompactionTopoBuilder.java index 8c6ed4c9f59e..a572354e8984 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/compact/UnawareBucketCompactionTopoBuilder.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/compact/UnawareBucketCompactionTopoBuilder.java @@ -126,7 +126,7 @@ private DataStreamSource buildSource() { new BucketUnawareCompactSource( table, isContinuous, scanInterval, partitionPredicate); - return BucketUnawareCompactSource.buildSource(env, source, isContinuous, tableIdentifier); + return BucketUnawareCompactSource.buildSource(env, source, tableIdentifier); } private void sinkFromSource(DataStreamSource input) { diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/compact/UnawareBucketCompactor.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/compact/UnawareBucketCompactor.java index a34f009072be..e8021329c9ce 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/compact/UnawareBucketCompactor.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/compact/UnawareBucketCompactor.java @@ -100,14 +100,31 @@ public void processElement(UnawareAppendCompactionTask task) throws Exception { metricsReporter.reportCompactionTime( System.currentTimeMillis() - startMillis); + metricsReporter + .increaseCompactionsCompletedCount(); } }, LOG); return commitMessage; } finally { MetricUtils.safeCall(this::stopTimer, LOG); + MetricUtils.safeCall( + this::decreaseCompactionsQueuedCount, LOG); } })); + recordCompactionsQueuedRequest(); + } + + private void recordCompactionsQueuedRequest() { + if (metricsReporter != null) { + metricsReporter.increaseCompactionsQueuedCount(); + } + } + + private void decreaseCompactionsQueuedCount() { + if (metricsReporter != null) { + metricsReporter.decreaseCompactionsQueuedCount(); + } } private void startTimer() { diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/compact/changelog/ChangelogCompactCoordinateOperator.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/compact/changelog/ChangelogCompactCoordinateOperator.java new file mode 100644 index 000000000000..cd0b8716a7d1 --- /dev/null +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/compact/changelog/ChangelogCompactCoordinateOperator.java @@ -0,0 +1,173 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.flink.compact.changelog; + +import org.apache.paimon.data.BinaryRow; +import org.apache.paimon.flink.sink.Committable; +import org.apache.paimon.io.CompactIncrement; +import org.apache.paimon.io.DataFileMeta; +import org.apache.paimon.io.DataIncrement; +import org.apache.paimon.table.FileStoreTable; +import org.apache.paimon.table.sink.CommitMessageImpl; + +import org.apache.flink.streaming.api.operators.AbstractStreamOperator; +import org.apache.flink.streaming.api.operators.BoundedOneInput; +import org.apache.flink.streaming.api.operators.OneInputStreamOperator; +import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; +import org.apache.flink.types.Either; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Coordinator operator for compacting changelog files. + * + *

    {@link ChangelogCompactCoordinateOperator} calculates the file size of changelog files + * contained in all buckets within each partition from {@link Committable} message emitted from + * writer operator. And emit {@link ChangelogCompactTask} to {@link ChangelogCompactWorkerOperator}. + */ +public class ChangelogCompactCoordinateOperator + extends AbstractStreamOperator> + implements OneInputStreamOperator>, + BoundedOneInput { + private final FileStoreTable table; + + private transient long checkpointId; + private transient Map partitionChangelogs; + + public ChangelogCompactCoordinateOperator(FileStoreTable table) { + this.table = table; + } + + @Override + public void open() throws Exception { + super.open(); + + checkpointId = Long.MIN_VALUE; + partitionChangelogs = new HashMap<>(); + } + + public void processElement(StreamRecord record) { + Committable committable = record.getValue(); + checkpointId = Math.max(checkpointId, committable.checkpointId()); + if (committable.kind() != Committable.Kind.FILE) { + output.collect(new StreamRecord<>(Either.Left(record.getValue()))); + return; + } + + CommitMessageImpl message = (CommitMessageImpl) committable.wrappedCommittable(); + if (message.newFilesIncrement().changelogFiles().isEmpty() + && message.compactIncrement().changelogFiles().isEmpty()) { + output.collect(new StreamRecord<>(Either.Left(record.getValue()))); + return; + } + + BinaryRow partition = message.partition(); + Integer bucket = message.bucket(); + long targetFileSize = table.coreOptions().targetFileSize(false); + for (DataFileMeta meta : message.newFilesIncrement().changelogFiles()) { + partitionChangelogs + .computeIfAbsent(partition, k -> new PartitionChangelog()) + .addNewChangelogFile(bucket, meta); + PartitionChangelog partitionChangelog = partitionChangelogs.get(partition); + if (partitionChangelog.totalFileSize >= targetFileSize) { + emitPartitionChanglogCompactTask(partition); + } + } + for (DataFileMeta meta : message.compactIncrement().changelogFiles()) { + partitionChangelogs + .computeIfAbsent(partition, k -> new PartitionChangelog()) + .addCompactChangelogFile(bucket, meta); + PartitionChangelog partitionChangelog = partitionChangelogs.get(partition); + if (partitionChangelog.totalFileSize >= targetFileSize) { + emitPartitionChanglogCompactTask(partition); + } + } + + CommitMessageImpl newMessage = + new CommitMessageImpl( + message.partition(), + message.bucket(), + new DataIncrement( + message.newFilesIncrement().newFiles(), + message.newFilesIncrement().deletedFiles(), + Collections.emptyList()), + new CompactIncrement( + message.compactIncrement().compactBefore(), + message.compactIncrement().compactAfter(), + Collections.emptyList()), + message.indexIncrement()); + Committable newCommittable = + new Committable(committable.checkpointId(), Committable.Kind.FILE, newMessage); + output.collect(new StreamRecord<>(Either.Left(newCommittable))); + } + + public void prepareSnapshotPreBarrier(long checkpointId) { + emitAllPartitionsChanglogCompactTask(); + } + + public void endInput() { + emitAllPartitionsChanglogCompactTask(); + } + + private void emitPartitionChanglogCompactTask(BinaryRow partition) { + PartitionChangelog partitionChangelog = partitionChangelogs.get(partition); + output.collect( + new StreamRecord<>( + Either.Right( + new ChangelogCompactTask( + checkpointId, + partition, + partitionChangelog.newFileChangelogFiles, + partitionChangelog.compactChangelogFiles)))); + partitionChangelogs.remove(partition); + } + + private void emitAllPartitionsChanglogCompactTask() { + List partitions = new ArrayList<>(partitionChangelogs.keySet()); + for (BinaryRow partition : partitions) { + emitPartitionChanglogCompactTask(partition); + } + } + + private static class PartitionChangelog { + private long totalFileSize; + private final Map> newFileChangelogFiles; + private final Map> compactChangelogFiles; + + public PartitionChangelog() { + totalFileSize = 0; + newFileChangelogFiles = new HashMap<>(); + compactChangelogFiles = new HashMap<>(); + } + + public void addNewChangelogFile(Integer bucket, DataFileMeta file) { + totalFileSize += file.fileSize(); + newFileChangelogFiles.computeIfAbsent(bucket, k -> new ArrayList<>()).add(file); + } + + public void addCompactChangelogFile(Integer bucket, DataFileMeta file) { + totalFileSize += file.fileSize(); + compactChangelogFiles.computeIfAbsent(bucket, k -> new ArrayList<>()).add(file); + } + } +} diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/compact/changelog/ChangelogCompactTask.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/compact/changelog/ChangelogCompactTask.java new file mode 100644 index 000000000000..6b95e369074b --- /dev/null +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/compact/changelog/ChangelogCompactTask.java @@ -0,0 +1,283 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.flink.compact.changelog; + +import org.apache.paimon.data.BinaryRow; +import org.apache.paimon.flink.compact.changelog.format.CompactedChangelogReadOnlyFormat; +import org.apache.paimon.flink.sink.Committable; +import org.apache.paimon.fs.Path; +import org.apache.paimon.fs.PositionOutputStream; +import org.apache.paimon.fs.SeekableInputStream; +import org.apache.paimon.io.CompactIncrement; +import org.apache.paimon.io.DataFileMeta; +import org.apache.paimon.io.DataFilePathFactory; +import org.apache.paimon.io.DataIncrement; +import org.apache.paimon.table.FileStoreTable; +import org.apache.paimon.table.sink.CommitMessageImpl; +import org.apache.paimon.utils.FileStorePathFactory; +import org.apache.paimon.utils.IOUtils; +import org.apache.paimon.utils.Preconditions; + +import java.io.IOException; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.UUID; + +/** + * {@link ChangelogCompactTask} to compact several changelog files from the same partition into one + * file, in order to reduce the number of small files. + */ +public class ChangelogCompactTask implements Serializable { + private final long checkpointId; + private final BinaryRow partition; + private final Map> newFileChangelogFiles; + private final Map> compactChangelogFiles; + + public ChangelogCompactTask( + long checkpointId, + BinaryRow partition, + Map> newFileChangelogFiles, + Map> compactChangelogFiles) { + this.checkpointId = checkpointId; + this.partition = partition; + this.newFileChangelogFiles = newFileChangelogFiles; + this.compactChangelogFiles = compactChangelogFiles; + } + + public long checkpointId() { + return checkpointId; + } + + public BinaryRow partition() { + return partition; + } + + public Map> newFileChangelogFiles() { + return newFileChangelogFiles; + } + + public Map> compactChangelogFiles() { + return compactChangelogFiles; + } + + public List doCompact(FileStoreTable table) throws Exception { + FileStorePathFactory pathFactory = table.store().pathFactory(); + OutputStream outputStream = new OutputStream(); + List results = new ArrayList<>(); + + // copy all changelog files to a new big file + for (Map.Entry> entry : newFileChangelogFiles.entrySet()) { + int bucket = entry.getKey(); + DataFilePathFactory dataFilePathFactory = + pathFactory.createDataFilePathFactory(partition, bucket); + for (DataFileMeta meta : entry.getValue()) { + copyFile( + outputStream, + results, + table, + dataFilePathFactory.toPath(meta.fileName()), + bucket, + false, + meta); + } + } + for (Map.Entry> entry : compactChangelogFiles.entrySet()) { + Integer bucket = entry.getKey(); + DataFilePathFactory dataFilePathFactory = + pathFactory.createDataFilePathFactory(partition, bucket); + for (DataFileMeta meta : entry.getValue()) { + copyFile( + outputStream, + results, + table, + dataFilePathFactory.toPath(meta.fileName()), + bucket, + true, + meta); + } + } + outputStream.out.close(); + + return produceNewCommittables(results, table, pathFactory, outputStream.path); + } + + private void copyFile( + OutputStream outputStream, + List results, + FileStoreTable table, + Path path, + int bucket, + boolean isCompactResult, + DataFileMeta meta) + throws Exception { + if (!outputStream.isInitialized) { + Path outputPath = + new Path(path.getParent(), "tmp-compacted-changelog-" + UUID.randomUUID()); + outputStream.init(outputPath, table.fileIO().newOutputStream(outputPath, false)); + } + long offset = outputStream.out.getPos(); + try (SeekableInputStream in = table.fileIO().newInputStream(path)) { + IOUtils.copyBytes(in, outputStream.out, IOUtils.BLOCKSIZE, false); + } + table.fileIO().deleteQuietly(path); + results.add( + new Result( + bucket, isCompactResult, meta, offset, outputStream.out.getPos() - offset)); + } + + private List produceNewCommittables( + List results, + FileStoreTable table, + FileStorePathFactory pathFactory, + Path changelogTempPath) + throws IOException { + Result baseResult = results.get(0); + Preconditions.checkArgument(baseResult.offset == 0); + DataFilePathFactory dataFilePathFactory = + pathFactory.createDataFilePathFactory(partition, baseResult.bucket); + // see Java docs of `CompactedChangelogFormatReaderFactory` + String realName = + "compacted-changelog-" + + UUID.randomUUID() + + "$" + + baseResult.bucket + + "-" + + baseResult.length; + table.fileIO() + .rename( + changelogTempPath, + dataFilePathFactory.toPath( + realName + + "." + + CompactedChangelogReadOnlyFormat.getIdentifier( + baseResult.meta.fileFormat()))); + + List newCommittables = new ArrayList<>(); + + Map> bucketedResults = new HashMap<>(); + for (Result result : results) { + bucketedResults.computeIfAbsent(result.bucket, b -> new ArrayList<>()).add(result); + } + + for (Map.Entry> entry : bucketedResults.entrySet()) { + List newFilesChangelog = new ArrayList<>(); + List compactChangelog = new ArrayList<>(); + for (Result result : entry.getValue()) { + // see Java docs of `CompactedChangelogFormatReaderFactory` + String name = + (result.offset == 0 + ? realName + : realName + "-" + result.offset + "-" + result.length) + + "." + + CompactedChangelogReadOnlyFormat.getIdentifier( + result.meta.fileFormat()); + if (result.isCompactResult) { + compactChangelog.add(result.meta.rename(name)); + } else { + newFilesChangelog.add(result.meta.rename(name)); + } + } + + CommitMessageImpl newMessage = + new CommitMessageImpl( + partition, + entry.getKey(), + new DataIncrement( + Collections.emptyList(), + Collections.emptyList(), + newFilesChangelog), + new CompactIncrement( + Collections.emptyList(), + Collections.emptyList(), + compactChangelog)); + newCommittables.add(new Committable(checkpointId, Committable.Kind.FILE, newMessage)); + } + return newCommittables; + } + + public int hashCode() { + return Objects.hash(checkpointId, partition, newFileChangelogFiles, compactChangelogFiles); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + ChangelogCompactTask that = (ChangelogCompactTask) o; + return checkpointId == that.checkpointId + && Objects.equals(partition, that.partition) + && Objects.equals(newFileChangelogFiles, that.newFileChangelogFiles) + && Objects.equals(compactChangelogFiles, that.compactChangelogFiles); + } + + @Override + public String toString() { + return String.format( + "ChangelogCompactionTask {" + + "partition = %s, " + + "newFileChangelogFiles = %s, " + + "compactChangelogFiles = %s}", + partition, newFileChangelogFiles, compactChangelogFiles); + } + + private static class OutputStream { + + private Path path; + private PositionOutputStream out; + private boolean isInitialized; + + private OutputStream() { + this.isInitialized = false; + } + + private void init(Path path, PositionOutputStream out) { + this.path = path; + this.out = out; + this.isInitialized = true; + } + } + + private static class Result { + + private final int bucket; + private final boolean isCompactResult; + private final DataFileMeta meta; + private final long offset; + private final long length; + + private Result( + int bucket, boolean isCompactResult, DataFileMeta meta, long offset, long length) { + this.bucket = bucket; + this.isCompactResult = isCompactResult; + this.meta = meta; + this.offset = offset; + this.length = length; + } + } +} diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/compact/changelog/ChangelogCompactTaskSerializer.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/compact/changelog/ChangelogCompactTaskSerializer.java new file mode 100644 index 000000000000..e21220b26db5 --- /dev/null +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/compact/changelog/ChangelogCompactTaskSerializer.java @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.flink.compact.changelog; + +import org.apache.paimon.io.DataFileMeta; +import org.apache.paimon.io.DataFileMetaSerializer; +import org.apache.paimon.io.DataInputDeserializer; +import org.apache.paimon.io.DataInputView; +import org.apache.paimon.io.DataOutputView; +import org.apache.paimon.io.DataOutputViewStreamWrapper; + +import org.apache.flink.core.io.SimpleVersionedSerializer; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.apache.paimon.utils.SerializationUtils.deserializeBinaryRow; +import static org.apache.paimon.utils.SerializationUtils.serializeBinaryRow; + +/** Serializer for {@link ChangelogCompactTask}. */ +public class ChangelogCompactTaskSerializer + implements SimpleVersionedSerializer { + private static final int CURRENT_VERSION = 1; + + private final DataFileMetaSerializer dataFileSerializer; + + public ChangelogCompactTaskSerializer() { + this.dataFileSerializer = new DataFileMetaSerializer(); + } + + @Override + public int getVersion() { + return CURRENT_VERSION; + } + + @Override + public byte[] serialize(ChangelogCompactTask obj) throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + DataOutputViewStreamWrapper view = new DataOutputViewStreamWrapper(out); + serialize(obj, view); + return out.toByteArray(); + } + + @Override + public ChangelogCompactTask deserialize(int version, byte[] serialized) throws IOException { + DataInputDeserializer view = new DataInputDeserializer(serialized); + return deserialize(version, view); + } + + private void serialize(ChangelogCompactTask task, DataOutputView view) throws IOException { + view.writeLong(task.checkpointId()); + serializeBinaryRow(task.partition(), view); + // serialize newFileChangelogFiles map + serializeMap(task.newFileChangelogFiles(), view); + serializeMap(task.compactChangelogFiles(), view); + } + + private ChangelogCompactTask deserialize(int version, DataInputView view) throws IOException { + if (version != getVersion()) { + throw new RuntimeException("Can not deserialize version: " + version); + } + + return new ChangelogCompactTask( + view.readLong(), + deserializeBinaryRow(view), + deserializeMap(view), + deserializeMap(view)); + } + + private void serializeMap(Map> map, DataOutputView view) + throws IOException { + view.writeInt(map.size()); + for (Map.Entry> entry : map.entrySet()) { + view.writeInt(entry.getKey()); + if (entry.getValue() == null) { + throw new IllegalArgumentException( + "serialize error. no value for bucket-" + entry.getKey()); + } + dataFileSerializer.serializeList(entry.getValue(), view); + } + } + + private Map> deserializeMap(DataInputView view) throws IOException { + final int size = view.readInt(); + + final Map> map = new HashMap<>(size); + for (int i = 0; i < size; i++) { + map.put(view.readInt(), dataFileSerializer.deserializeList(view)); + } + + return map; + } +} diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/compact/changelog/ChangelogCompactWorkerOperator.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/compact/changelog/ChangelogCompactWorkerOperator.java new file mode 100644 index 000000000000..260c25a31561 --- /dev/null +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/compact/changelog/ChangelogCompactWorkerOperator.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.flink.compact.changelog; + +import org.apache.paimon.flink.sink.Committable; +import org.apache.paimon.table.FileStoreTable; + +import org.apache.flink.streaming.api.operators.AbstractStreamOperator; +import org.apache.flink.streaming.api.operators.OneInputStreamOperator; +import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; +import org.apache.flink.types.Either; + +import java.util.List; + +/** + * Receive and process the {@link ChangelogCompactTask}s emitted by {@link + * ChangelogCompactCoordinateOperator}. + */ +public class ChangelogCompactWorkerOperator extends AbstractStreamOperator + implements OneInputStreamOperator, Committable> { + private final FileStoreTable table; + + public ChangelogCompactWorkerOperator(FileStoreTable table) { + this.table = table; + } + + public void processElement(StreamRecord> record) + throws Exception { + + if (record.getValue().isLeft()) { + output.collect(new StreamRecord<>(record.getValue().left())); + } else { + ChangelogCompactTask task = record.getValue().right(); + List committables = task.doCompact(table); + committables.forEach(committable -> output.collect(new StreamRecord<>(committable))); + } + } +} diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/compact/changelog/ChangelogTaskTypeInfo.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/compact/changelog/ChangelogTaskTypeInfo.java new file mode 100644 index 000000000000..a529e6764fae --- /dev/null +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/compact/changelog/ChangelogTaskTypeInfo.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.flink.compact.changelog; + +import org.apache.paimon.flink.sink.NoneCopyVersionedSerializerTypeSerializerProxy; + +import org.apache.flink.api.common.ExecutionConfig; +import org.apache.flink.api.common.serialization.SerializerConfig; +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.api.common.typeutils.TypeSerializer; + +/** Type information for {@link ChangelogCompactTask}. */ +public class ChangelogTaskTypeInfo extends TypeInformation { + @Override + public boolean isBasicType() { + return false; + } + + @Override + public boolean isTupleType() { + return false; + } + + @Override + public int getArity() { + return 1; + } + + @Override + public int getTotalFields() { + return 1; + } + + @Override + public Class getTypeClass() { + return ChangelogCompactTask.class; + } + + @Override + public boolean isKeyType() { + return false; + } + + /** + * Do not annotate with @override here to maintain compatibility with Flink 1.18-. + */ + public TypeSerializer createSerializer( + SerializerConfig serializerConfig) { + return this.createSerializer((ExecutionConfig) null); + } + + /** + * Do not annotate with @override here to maintain compatibility with Flink 2.0+. + */ + public TypeSerializer createSerializer(ExecutionConfig config) { + // we don't need copy for task + return new NoneCopyVersionedSerializerTypeSerializerProxy( + ChangelogCompactTaskSerializer::new) {}; + } + + @Override + public int hashCode() { + return 0; + } + + @Override + public boolean canEqual(Object obj) { + return obj instanceof ChangelogTaskTypeInfo; + } + + @Override + public boolean equals(Object obj) { + return obj instanceof ChangelogTaskTypeInfo; + } + + @Override + public String toString() { + return "ChangelogCompactionTask"; + } +} diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/compact/changelog/format/CompactedChangelogFormatReaderFactory.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/compact/changelog/format/CompactedChangelogFormatReaderFactory.java new file mode 100644 index 000000000000..e0aed448db93 --- /dev/null +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/compact/changelog/format/CompactedChangelogFormatReaderFactory.java @@ -0,0 +1,258 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.flink.compact.changelog.format; + +import org.apache.paimon.catalog.CatalogContext; +import org.apache.paimon.data.InternalRow; +import org.apache.paimon.fileindex.FileIndexResult; +import org.apache.paimon.format.FormatReaderFactory; +import org.apache.paimon.fs.FileIO; +import org.apache.paimon.fs.FileStatus; +import org.apache.paimon.fs.Path; +import org.apache.paimon.fs.PositionOutputStream; +import org.apache.paimon.fs.SeekableInputStream; +import org.apache.paimon.reader.FileRecordReader; + +import java.io.EOFException; +import java.io.IOException; + +/** + * {@link FormatReaderFactory} for compacted changelog. + * + *

    File Name Protocol + * + *

    There are two kinds of file name. In the following description, bid1 and + * bid2 are bucket id, off is offset, len1 and len2 + * are lengths. + * + *

      + *
    • bucket-bid1/compacted-changelog-xxx$bid1-len1: This is the real file name. If + * this file name is recorded in manifest file meta, reader should read the bytes of this file + * starting from offset 0 with length len1. + *
    • bucket-bid2/compacted-changelog-xxx$bid1-len1-off-len2: This is the fake file + * name. Reader should read the bytes of file + * bucket-bid1/compacted-changelog-xxx$bid1-len1 starting from offset off + * with length len2. + *
    + */ +public class CompactedChangelogFormatReaderFactory implements FormatReaderFactory { + + private final FormatReaderFactory wrapped; + + public CompactedChangelogFormatReaderFactory(FormatReaderFactory wrapped) { + this.wrapped = wrapped; + } + + @Override + public FileRecordReader createReader(Context context) throws IOException { + OffsetReadOnlyFileIO fileIO = new OffsetReadOnlyFileIO(context.fileIO()); + long length = decodePath(context.filePath()).length; + + return wrapped.createReader( + new Context() { + + @Override + public FileIO fileIO() { + return fileIO; + } + + @Override + public Path filePath() { + return context.filePath(); + } + + @Override + public long fileSize() { + return length; + } + + @Override + public FileIndexResult fileIndex() { + return context.fileIndex(); + } + }); + } + + private static DecodeResult decodePath(Path path) { + String[] nameAndFormat = path.getName().split("\\."); + String[] names = nameAndFormat[0].split("\\$"); + String[] split = names[1].split("-"); + if (split.length == 2) { + return new DecodeResult(path, 0, Long.parseLong(split[1])); + } else { + Path realPath = + new Path( + path.getParent().getParent(), + "bucket-" + + split[0] + + "/" + + names[0] + + "$" + + split[0] + + "-" + + split[1] + + "." + + nameAndFormat[1]); + return new DecodeResult(realPath, Long.parseLong(split[2]), Long.parseLong(split[3])); + } + } + + private static class DecodeResult { + + private final Path path; + private final long offset; + private final long length; + + private DecodeResult(Path path, long offset, long length) { + this.path = path; + this.offset = offset; + this.length = length; + } + } + + private static class OffsetReadOnlyFileIO implements FileIO { + + private final FileIO wrapped; + + private OffsetReadOnlyFileIO(FileIO wrapped) { + this.wrapped = wrapped; + } + + @Override + public boolean isObjectStore() { + return wrapped.isObjectStore(); + } + + @Override + public void configure(CatalogContext context) { + wrapped.configure(context); + } + + @Override + public SeekableInputStream newInputStream(Path path) throws IOException { + DecodeResult result = decodePath(path); + return new OffsetSeekableInputStream( + wrapped.newInputStream(result.path), result.offset, result.length); + } + + @Override + public PositionOutputStream newOutputStream(Path path, boolean overwrite) + throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public FileStatus getFileStatus(Path path) throws IOException { + DecodeResult result = decodePath(path); + FileStatus status = wrapped.getFileStatus(result.path); + + return new FileStatus() { + + @Override + public long getLen() { + return result.length; + } + + @Override + public boolean isDir() { + return status.isDir(); + } + + @Override + public Path getPath() { + return path; + } + + @Override + public long getModificationTime() { + return status.getModificationTime(); + } + }; + } + + @Override + public FileStatus[] listStatus(Path path) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public boolean exists(Path path) throws IOException { + return wrapped.exists(decodePath(path).path); + } + + @Override + public boolean delete(Path path, boolean recursive) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public boolean mkdirs(Path path) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public boolean rename(Path src, Path dst) throws IOException { + throw new UnsupportedOperationException(); + } + } + + private static class OffsetSeekableInputStream extends SeekableInputStream { + + private final SeekableInputStream wrapped; + private final long offset; + private final long length; + + private OffsetSeekableInputStream(SeekableInputStream wrapped, long offset, long length) + throws IOException { + this.wrapped = wrapped; + this.offset = offset; + this.length = length; + wrapped.seek(offset); + } + + @Override + public void seek(long desired) throws IOException { + wrapped.seek(offset + desired); + } + + @Override + public long getPos() throws IOException { + return wrapped.getPos() - offset; + } + + @Override + public int read() throws IOException { + if (getPos() >= length) { + throw new EOFException(); + } + return wrapped.read(); + } + + @Override + public int read(byte[] b, int off, int len) throws IOException { + long realLen = Math.min(len, length - getPos()); + return wrapped.read(b, off, (int) realLen); + } + + @Override + public void close() throws IOException { + wrapped.close(); + } + } +} diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/compact/changelog/format/CompactedChangelogReadOnlyFormat.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/compact/changelog/format/CompactedChangelogReadOnlyFormat.java new file mode 100644 index 000000000000..39bed81505c6 --- /dev/null +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/compact/changelog/format/CompactedChangelogReadOnlyFormat.java @@ -0,0 +1,106 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.flink.compact.changelog.format; + +import org.apache.paimon.format.FileFormat; +import org.apache.paimon.format.FileFormatFactory; +import org.apache.paimon.format.FormatReaderFactory; +import org.apache.paimon.format.FormatWriterFactory; +import org.apache.paimon.predicate.Predicate; +import org.apache.paimon.types.RowType; + +import javax.annotation.Nullable; + +import java.util.List; + +/** {@link FileFormat} for compacted changelog. */ +public class CompactedChangelogReadOnlyFormat extends FileFormat { + + private final FileFormat wrapped; + + protected CompactedChangelogReadOnlyFormat(String formatIdentifier, FileFormat wrapped) { + super(formatIdentifier); + this.wrapped = wrapped; + } + + @Override + public FormatReaderFactory createReaderFactory( + RowType projectedRowType, @Nullable List filters) { + return new CompactedChangelogFormatReaderFactory( + wrapped.createReaderFactory(projectedRowType, filters)); + } + + @Override + public FormatWriterFactory createWriterFactory(RowType type) { + throw new UnsupportedOperationException(); + } + + @Override + public void validateDataFields(RowType rowType) { + wrapped.validateDataFields(rowType); + } + + public static String getIdentifier(String wrappedFormat) { + return "cc-" + wrappedFormat; + } + + static class AbstractFactory implements FileFormatFactory { + + private final String format; + + AbstractFactory(String format) { + this.format = format; + } + + @Override + public String identifier() { + return getIdentifier(format); + } + + @Override + public FileFormat create(FormatContext formatContext) { + return new CompactedChangelogReadOnlyFormat( + getIdentifier(format), FileFormat.fromIdentifier(format, formatContext)); + } + } + + /** {@link FileFormatFactory} for compacted changelog, with orc as the real format. */ + public static class OrcFactory extends AbstractFactory { + + public OrcFactory() { + super("orc"); + } + } + + /** {@link FileFormatFactory} for compacted changelog, with parquet as the real format. */ + public static class ParquetFactory extends AbstractFactory { + + public ParquetFactory() { + super("parquet"); + } + } + + /** {@link FileFormatFactory} for compacted changelog, with avro as the real format. */ + public static class AvroFactory extends AbstractFactory { + + public AvroFactory() { + super("avro"); + } + } +} diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/log/LogStoreRegister.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/log/LogStoreRegister.java index b730d289b31b..ad501e204ce6 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/log/LogStoreRegister.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/log/LogStoreRegister.java @@ -46,11 +46,14 @@ static void registerLogSystem( ClassLoader classLoader) { Options tableOptions = Options.fromMap(options); String logStore = tableOptions.get(LOG_SYSTEM); - if (!tableOptions.get(LOG_SYSTEM).equalsIgnoreCase(NONE) - && !catalog.tableExists(identifier)) { - LogStoreRegister logStoreRegister = - getLogStoreRegister(identifier, classLoader, tableOptions, logStore); - options.putAll(logStoreRegister.registerTopic()); + if (!tableOptions.get(LOG_SYSTEM).equalsIgnoreCase(NONE)) { + try { + catalog.getTable(identifier); + } catch (Catalog.TableNotExistException e) { + LogStoreRegister logStoreRegister = + getLogStoreRegister(identifier, classLoader, tableOptions, logStore); + options.putAll(logStoreRegister.registerTopic()); + } } } diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/FileStoreLookupFunction.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/FileStoreLookupFunction.java index 4090193de285..e3f2fe110c6c 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/FileStoreLookupFunction.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/FileStoreLookupFunction.java @@ -33,6 +33,7 @@ import org.apache.paimon.table.source.OutOfRangeException; import org.apache.paimon.types.RowType; import org.apache.paimon.utils.FileIOUtils; +import org.apache.paimon.utils.Filter; import org.apache.paimon.utils.RowDataToObjectArrayConverter; import org.apache.paimon.shade.guava30.com.google.common.primitives.Ints; @@ -68,6 +69,7 @@ import static org.apache.paimon.CoreOptions.CONTINUOUS_DISCOVERY_INTERVAL; import static org.apache.paimon.flink.FlinkConnectorOptions.LOOKUP_CACHE_MODE; +import static org.apache.paimon.flink.FlinkConnectorOptions.LOOKUP_REFRESH_TIME_PERIODS_BLACKLIST; import static org.apache.paimon.flink.query.RemoteTableQuery.isRemoteServiceAvailable; import static org.apache.paimon.lookup.RocksDBOptions.LOOKUP_CACHE_ROWS; import static org.apache.paimon.lookup.RocksDBOptions.LOOKUP_CONTINUOUS_DISCOVERY_INTERVAL; @@ -86,16 +88,20 @@ public class FileStoreLookupFunction implements Serializable, Closeable { private final List projectFields; private final List joinKeys; @Nullable private final Predicate predicate; + @Nullable private final RefreshBlacklist refreshBlacklist; - private transient Duration refreshInterval; private transient File path; private transient LookupTable lookupTable; - // timestamp when cache expires - private transient long nextLoadTime; + // interval of refreshing lookup table + private transient Duration refreshInterval; + // timestamp when refreshing lookup table + private transient long nextRefreshTime; protected FunctionContext functionContext; + @Nullable private Filter cacheRowFilter; + public FileStoreLookupFunction( Table table, int[] projection, int[] joinKeyIndex, @Nullable Predicate predicate) { if (!TableScanUtils.supportCompactDiffStreamingReading(table)) { @@ -128,6 +134,10 @@ public FileStoreLookupFunction( } this.predicate = predicate; + + this.refreshBlacklist = + RefreshBlacklist.create( + table.options().get(LOOKUP_REFRESH_TIME_PERIODS_BLACKLIST.key())); } public void open(FunctionContext context) throws Exception { @@ -146,11 +156,7 @@ void open(String tmpDirectory) throws Exception { } private void open() throws Exception { - if (partitionLoader != null) { - partitionLoader.open(); - } - - this.nextLoadTime = -1; + this.nextRefreshTime = -1; Options options = Options.fromMap(table.options()); this.refreshInterval = @@ -194,7 +200,18 @@ private void open() throws Exception { this.lookupTable = FullCacheLookupTable.create(context, options.get(LOOKUP_CACHE_ROWS)); } - refreshDynamicPartition(false); + if (partitionLoader != null) { + partitionLoader.open(); + partitionLoader.checkRefresh(); + BinaryRow partition = partitionLoader.partition(); + if (partition != null) { + lookupTable.specificPartitionFilter(createSpecificPartFilter(partition)); + } + } + + if (cacheRowFilter != null) { + lookupTable.specifyCacheRowFilter(cacheRowFilter); + } lookupTable.open(); } @@ -216,15 +233,14 @@ private Predicate createProjectedPredicate(int[] projection) { public Collection lookup(RowData keyRow) { try { - checkRefresh(); + tryRefresh(); InternalRow key = new FlinkRowWrapper(keyRow); if (partitionLoader != null) { - InternalRow partition = refreshDynamicPartition(true); - if (partition == null) { + if (partitionLoader.partition() == null) { return Collections.emptyList(); } - key = JoinedRow.join(key, partition); + key = JoinedRow.join(key, partitionLoader.partition()); } List results = lookupTable.get(key); @@ -233,7 +249,7 @@ public Collection lookup(RowData keyRow) { rows.add(new FlinkRowData(matchedRow)); } return rows; - } catch (OutOfRangeException e) { + } catch (OutOfRangeException | ReopenException e) { reopen(); return lookup(keyRow); } catch (Exception e) { @@ -241,28 +257,6 @@ public Collection lookup(RowData keyRow) { } } - @Nullable - private BinaryRow refreshDynamicPartition(boolean reopen) throws Exception { - if (partitionLoader == null) { - return null; - } - - boolean partitionChanged = partitionLoader.checkRefresh(); - BinaryRow partition = partitionLoader.partition(); - if (partition == null) { - return null; - } - - lookupTable.specificPartitionFilter(createSpecificPartFilter(partition)); - - if (partitionChanged && reopen) { - lookupTable.close(); - lookupTable.open(); - } - - return partition; - } - private Predicate createSpecificPartFilter(BinaryRow partition) { RowType rowType = table.rowType(); List partitionKeys = table.partitionKeys(); @@ -287,20 +281,51 @@ private void reopen() { } } - private void checkRefresh() throws Exception { - if (nextLoadTime > System.currentTimeMillis()) { + @VisibleForTesting + void tryRefresh() throws Exception { + // 1. check if this time is in black list + if (refreshBlacklist != null && !refreshBlacklist.canRefresh()) { return; } - if (nextLoadTime > 0) { + + // 2. refresh dynamic partition + if (partitionLoader != null) { + boolean partitionChanged = partitionLoader.checkRefresh(); + BinaryRow partition = partitionLoader.partition(); + if (partition == null) { + // no data to be load, fast exit + return; + } + + if (partitionChanged) { + // reopen with latest partition + lookupTable.specificPartitionFilter(createSpecificPartFilter(partition)); + lookupTable.close(); + lookupTable.open(); + // no need to refresh the lookup table because it is reopened + return; + } + } + + // 3. refresh lookup table + if (shouldRefreshLookupTable()) { + lookupTable.refresh(); + nextRefreshTime = System.currentTimeMillis() + refreshInterval.toMillis(); + } + } + + private boolean shouldRefreshLookupTable() { + if (nextRefreshTime > System.currentTimeMillis()) { + return false; + } + + if (nextRefreshTime > 0) { LOG.info( "Lookup table {} has refreshed after {} second(s), refreshing", table.name(), refreshInterval.toMillis() / 1000); } - - refresh(); - - nextLoadTime = System.currentTimeMillis() + refreshInterval.toMillis(); + return true; } @VisibleForTesting @@ -308,8 +333,9 @@ LookupTable lookupTable() { return lookupTable; } - private void refresh() throws Exception { - lookupTable.refresh(); + @VisibleForTesting + long nextBlacklistCheckTime() { + return refreshBlacklist == null ? -1 : refreshBlacklist.nextBlacklistCheckTime(); } @Override @@ -361,4 +387,8 @@ protected Set getRequireCachedBucketIds() { // TODO: Implement the method when Flink support bucket shuffle for lookup join. return null; } + + protected void setCacheRowFilter(@Nullable Filter cacheRowFilter) { + this.cacheRowFilter = cacheRowFilter; + } } diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/FullCacheLookupTable.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/FullCacheLookupTable.java index e9389f1f291a..de69c67a4c44 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/FullCacheLookupTable.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/FullCacheLookupTable.java @@ -37,6 +37,7 @@ import org.apache.paimon.utils.ExecutorUtils; import org.apache.paimon.utils.FieldsComparator; import org.apache.paimon.utils.FileIOUtils; +import org.apache.paimon.utils.Filter; import org.apache.paimon.utils.MutableObjectIterator; import org.apache.paimon.utils.PartialRow; import org.apache.paimon.utils.TypeUtils; @@ -85,12 +86,14 @@ public abstract class FullCacheLookupTable implements LookupTable { private Future refreshFuture; private LookupStreamingReader reader; private Predicate specificPartition; + @Nullable private Filter cacheRowFilter; public FullCacheLookupTable(Context context) { this.table = context.table; List sequenceFields = new ArrayList<>(); + CoreOptions coreOptions = new CoreOptions(table.options()); if (table.primaryKeys().size() > 0) { - sequenceFields = new CoreOptions(table.options()).sequenceField(); + sequenceFields = coreOptions.sequenceField(); } RowType projectedType = TypeUtils.project(table.rowType(), context.projection); if (sequenceFields.size() > 0) { @@ -109,7 +112,10 @@ public FullCacheLookupTable(Context context) { projectedType = builder.build(); context = context.copy(table.rowType().getFieldIndices(projectedType.getFieldNames())); this.userDefinedSeqComparator = - UserDefinedSeqComparator.create(projectedType, sequenceFields); + UserDefinedSeqComparator.create( + projectedType, + sequenceFields, + coreOptions.sequenceFieldSortOrderIsAscending()); this.appendUdsFieldNumber = appendUdsFieldNumber.get(); } else { this.userDefinedSeqComparator = null; @@ -138,6 +144,11 @@ public void specificPartitionFilter(Predicate filter) { this.specificPartition = filter; } + @Override + public void specifyCacheRowFilter(Filter filter) { + this.cacheRowFilter = filter; + } + protected void openStateFactory() throws Exception { this.stateFactory = new RocksDBStateFactory( @@ -154,7 +165,8 @@ protected void bootstrap() throws Exception { context.table, context.projection, scanPredicate, - context.requiredCachedBucketIds); + context.requiredCachedBucketIds, + cacheRowFilter); BinaryExternalSortBuffer bulkLoadSorter = RocksDBState.createBulkLoadSorter( IOManager.create(context.tempPath.toString()), context.table.coreOptions()); diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/LookupDataTableScan.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/LookupDataTableScan.java index 908884a573c0..f43d80321ecc 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/LookupDataTableScan.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/LookupDataTableScan.java @@ -19,6 +19,7 @@ package org.apache.paimon.flink.lookup; import org.apache.paimon.CoreOptions; +import org.apache.paimon.Snapshot; import org.apache.paimon.operation.DefaultValueAssigner; import org.apache.paimon.table.source.DataTableStreamScan; import org.apache.paimon.table.source.snapshot.AllDeltaFollowUpScanner; @@ -29,6 +30,8 @@ import org.apache.paimon.table.source.snapshot.StartingScanner; import org.apache.paimon.utils.SnapshotManager; +import javax.annotation.Nullable; + import static org.apache.paimon.CoreOptions.StartupMode; import static org.apache.paimon.flink.lookup.LookupFileStoreTable.LookupStreamScanMode; @@ -56,6 +59,17 @@ public LookupDataTableScan( defaultValueAssigner); this.startupMode = options.startupMode(); this.lookupScanMode = lookupScanMode; + dropStats(); + } + + @Override + @Nullable + protected SnapshotReader.Plan handleOverwriteSnapshot(Snapshot snapshot) { + SnapshotReader.Plan plan = super.handleOverwriteSnapshot(snapshot); + if (plan != null) { + return plan; + } + throw new ReopenException(); } @Override diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/LookupStreamingReader.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/LookupStreamingReader.java index e6dfd41f8f0e..132b30138d0a 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/LookupStreamingReader.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/LookupStreamingReader.java @@ -30,6 +30,7 @@ import org.apache.paimon.table.source.Split; import org.apache.paimon.table.source.StreamTableScan; import org.apache.paimon.types.RowType; +import org.apache.paimon.utils.Filter; import org.apache.paimon.utils.FunctionWithIOException; import org.apache.paimon.utils.TypeUtils; @@ -51,6 +52,7 @@ public class LookupStreamingReader { private final LookupFileStoreTable table; private final int[] projection; + @Nullable private final Filter cacheRowFilter; private final ReadBuilder readBuilder; @Nullable private final Predicate projectedPredicate; private final StreamTableScan scan; @@ -59,9 +61,11 @@ public LookupStreamingReader( LookupFileStoreTable table, int[] projection, @Nullable Predicate predicate, - Set requireCachedBucketIds) { + Set requireCachedBucketIds, + @Nullable Filter cacheRowFilter) { this.table = table; this.projection = projection; + this.cacheRowFilter = cacheRowFilter; this.readBuilder = this.table .newReadBuilder() @@ -125,6 +129,10 @@ public RecordReader nextBatch(boolean useParallelism) throws Except if (projectedPredicate != null) { reader = reader.filter(projectedPredicate::test); } + + if (cacheRowFilter != null) { + reader = reader.filter(cacheRowFilter); + } return reader; } diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/LookupTable.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/LookupTable.java index c13947bbd819..8ea1931b96aa 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/LookupTable.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/LookupTable.java @@ -20,6 +20,7 @@ import org.apache.paimon.data.InternalRow; import org.apache.paimon.predicate.Predicate; +import org.apache.paimon.utils.Filter; import java.io.Closeable; import java.io.IOException; @@ -35,4 +36,6 @@ public interface LookupTable extends Closeable { List get(InternalRow key) throws IOException; void refresh() throws Exception; + + void specifyCacheRowFilter(Filter filter); } diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/PrimaryKeyPartialLookupTable.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/PrimaryKeyPartialLookupTable.java index 6c6979eeebc2..7bd7a652b56e 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/PrimaryKeyPartialLookupTable.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/PrimaryKeyPartialLookupTable.java @@ -31,6 +31,7 @@ import org.apache.paimon.table.source.DataSplit; import org.apache.paimon.table.source.Split; import org.apache.paimon.table.source.StreamTableScan; +import org.apache.paimon.utils.Filter; import org.apache.paimon.utils.ProjectedRow; import javax.annotation.Nullable; @@ -41,23 +42,21 @@ import java.util.Collections; import java.util.List; import java.util.Set; -import java.util.function.Function; /** Lookup table for primary key which supports to read the LSM tree directly. */ public class PrimaryKeyPartialLookupTable implements LookupTable { - private final Function executorFactory; + private final QueryExecutorFactory executorFactory; private final FixedBucketFromPkExtractor extractor; @Nullable private final ProjectedRow keyRearrange; @Nullable private final ProjectedRow trimmedKeyRearrange; private Predicate specificPartition; + @Nullable private Filter cacheRowFilter; private QueryExecutor queryExecutor; private PrimaryKeyPartialLookupTable( - Function executorFactory, - FileStoreTable table, - List joinKey) { + QueryExecutorFactory executorFactory, FileStoreTable table, List joinKey) { this.executorFactory = executorFactory; if (table.bucketMode() != BucketMode.HASH_FIXED) { @@ -103,7 +102,7 @@ public void specificPartitionFilter(Predicate filter) { @Override public void open() throws Exception { - this.queryExecutor = executorFactory.apply(specificPartition); + this.queryExecutor = executorFactory.create(specificPartition, cacheRowFilter); refresh(); } @@ -135,6 +134,11 @@ public void refresh() { queryExecutor.refresh(); } + @Override + public void specifyCacheRowFilter(Filter filter) { + this.cacheRowFilter = filter; + } + @Override public void close() throws IOException { if (queryExecutor != null) { @@ -149,13 +153,14 @@ public static PrimaryKeyPartialLookupTable createLocalTable( List joinKey, Set requireCachedBucketIds) { return new PrimaryKeyPartialLookupTable( - filter -> + (filter, cacheRowFilter) -> new LocalQueryExecutor( new LookupFileStoreTable(table, joinKey), projection, tempPath, filter, - requireCachedBucketIds), + requireCachedBucketIds, + cacheRowFilter), table, joinKey); } @@ -163,7 +168,13 @@ public static PrimaryKeyPartialLookupTable createLocalTable( public static PrimaryKeyPartialLookupTable createRemoteTable( FileStoreTable table, int[] projection, List joinKey) { return new PrimaryKeyPartialLookupTable( - filter -> new RemoteQueryExecutor(table, projection), table, joinKey); + (filter, cacheRowFilter) -> new RemoteQueryExecutor(table, projection), + table, + joinKey); + } + + interface QueryExecutorFactory { + QueryExecutor create(Predicate filter, @Nullable Filter cacheRowFilter); } interface QueryExecutor extends Closeable { @@ -183,14 +194,20 @@ private LocalQueryExecutor( int[] projection, File tempPath, @Nullable Predicate filter, - Set requireCachedBucketIds) { + Set requireCachedBucketIds, + @Nullable Filter cacheRowFilter) { this.tableQuery = table.newLocalTableQuery() .withValueProjection(projection) .withIOManager(new IOManagerImpl(tempPath.toString())); + if (cacheRowFilter != null) { + this.tableQuery.withCacheRowFilter(cacheRowFilter); + } + this.scan = table.newReadBuilder() + .dropStats() .withFilter(filter) .withBucketFilter( requireCachedBucketIds == null diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/RefreshBlacklist.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/RefreshBlacklist.java new file mode 100644 index 000000000000..e20294fe0676 --- /dev/null +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/RefreshBlacklist.java @@ -0,0 +1,124 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.flink.lookup; + +import org.apache.paimon.utils.DateTimeUtils; +import org.apache.paimon.utils.Pair; +import org.apache.paimon.utils.StringUtils; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.annotation.Nullable; + +import java.time.format.DateTimeParseException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.TimeZone; + +/** Refresh black list for {@link FileStoreLookupFunction}. */ +public class RefreshBlacklist { + + private static final Logger LOG = LoggerFactory.getLogger(RefreshBlacklist.class); + + private final List> timePeriodsBlacklist; + + private long nextBlacklistCheckTime; + + public RefreshBlacklist(List> timePeriodsBlacklist) { + this.timePeriodsBlacklist = timePeriodsBlacklist; + this.nextBlacklistCheckTime = -1; + } + + @Nullable + public static RefreshBlacklist create(String blacklist) { + List> timePeriodsBlacklist = parseTimePeriodsBlacklist(blacklist); + if (timePeriodsBlacklist.isEmpty()) { + return null; + } + + return new RefreshBlacklist(timePeriodsBlacklist); + } + + private static List> parseTimePeriodsBlacklist(String blacklist) { + if (StringUtils.isNullOrWhitespaceOnly(blacklist)) { + return Collections.emptyList(); + } + String[] timePeriods = blacklist.split(","); + List> result = new ArrayList<>(); + for (String period : timePeriods) { + String[] times = period.split("->"); + if (times.length != 2) { + throw new IllegalArgumentException( + String.format("Incorrect time periods format: [%s].", blacklist)); + } + + long left = parseToMillis(times[0]); + long right = parseToMillis(times[1]); + if (left > right) { + throw new IllegalArgumentException( + String.format("Incorrect time period: [%s->%s].", times[0], times[1])); + } + result.add(Pair.of(left, right)); + } + return result; + } + + private static long parseToMillis(String dateTime) { + try { + return DateTimeUtils.parseTimestampData(dateTime + ":00", 3, TimeZone.getDefault()) + .getMillisecond(); + } catch (DateTimeParseException e) { + throw new IllegalArgumentException( + String.format("Date time format error: [%s].", dateTime), e); + } + } + + public boolean canRefresh() { + long currentTimeMillis = System.currentTimeMillis(); + if (currentTimeMillis < nextBlacklistCheckTime) { + return false; + } + + Pair selectedPeriod = null; + for (Pair period : timePeriodsBlacklist) { + if (period.getLeft() <= currentTimeMillis && currentTimeMillis <= period.getRight()) { + selectedPeriod = period; + break; + } + } + + if (selectedPeriod != null) { + LOG.info( + "Current time {} is in black list {}-{}, so try to refresh cache next time.", + currentTimeMillis, + selectedPeriod.getLeft(), + selectedPeriod.getRight()); + nextBlacklistCheckTime = selectedPeriod.getRight() + 1; + return false; + } + + return true; + } + + public long nextBlacklistCheckTime() { + return nextBlacklistCheckTime; + } +} diff --git a/paimon-spark/paimon-spark-3.4/src/test/scala/org/apache/spark/paimon/Utils.scala b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/ReopenException.java similarity index 74% rename from paimon-spark/paimon-spark-3.4/src/test/scala/org/apache/spark/paimon/Utils.scala rename to paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/ReopenException.java index 1a899f500153..7149d591f8df 100644 --- a/paimon-spark/paimon-spark-3.4/src/test/scala/org/apache/spark/paimon/Utils.scala +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/ReopenException.java @@ -16,17 +16,14 @@ * limitations under the License. */ -package org.apache.spark.paimon +package org.apache.paimon.flink.lookup; -import org.apache.spark.util.{Utils => SparkUtils} +/** Signals that dim table source need to reopen. */ +public class ReopenException extends RuntimeException { -import java.io.File - -/** - * A wrapper that some Objects or Classes is limited to access beyond [[org.apache.spark]] package. - */ -object Utils { - - def createTempDir: File = SparkUtils.createTempDir() + private static final long serialVersionUID = 1L; + public ReopenException() { + super(); + } } diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/orphan/FlinkOrphanFilesClean.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/orphan/FlinkOrphanFilesClean.java index f50414620551..23bbbc9b609c 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/orphan/FlinkOrphanFilesClean.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/orphan/FlinkOrphanFilesClean.java @@ -27,12 +27,15 @@ import org.apache.paimon.fs.Path; import org.apache.paimon.manifest.ManifestEntry; import org.apache.paimon.manifest.ManifestFile; +import org.apache.paimon.operation.CleanOrphanFilesResult; import org.apache.paimon.operation.OrphanFilesClean; import org.apache.paimon.table.FileStoreTable; import org.apache.paimon.table.Table; +import org.apache.paimon.utils.Pair; import org.apache.paimon.utils.SerializableConsumer; import org.apache.flink.api.common.RuntimeExecutionMode; +import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.configuration.Configuration; import org.apache.flink.configuration.CoreOptions; @@ -61,7 +64,6 @@ import java.util.function.Consumer; import java.util.stream.Collectors; -import static org.apache.flink.api.common.typeinfo.BasicTypeInfo.LONG_TYPE_INFO; import static org.apache.flink.api.common.typeinfo.BasicTypeInfo.STRING_TYPE_INFO; import static org.apache.flink.util.Preconditions.checkState; import static org.apache.paimon.utils.Preconditions.checkArgument; @@ -81,7 +83,7 @@ public FlinkOrphanFilesClean( } @Nullable - public DataStream doOrphanClean(StreamExecutionEnvironment env) { + public DataStream doOrphanClean(StreamExecutionEnvironment env) { Configuration flinkConf = new Configuration(); flinkConf.set(ExecutionOptions.RUNTIME_MODE, RuntimeExecutionMode.BATCH); flinkConf.set(ExecutionOptions.SORT_INPUTS, false); @@ -97,8 +99,12 @@ public DataStream doOrphanClean(StreamExecutionEnvironment env) { // snapshot and changelog files are the root of everything, so they are handled specially // here, and subsequently, we will not count their orphan files. - AtomicLong deletedInLocal = new AtomicLong(0); - cleanSnapshotDir(branches, p -> deletedInLocal.incrementAndGet()); + AtomicLong deletedFilesCountInLocal = new AtomicLong(0); + AtomicLong deletedFilesLenInBytesInLocal = new AtomicLong(0); + cleanSnapshotDir( + branches, + path -> deletedFilesCountInLocal.incrementAndGet(), + deletedFilesLenInBytesInLocal::addAndGet); // branch and manifest file final OutputTag> manifestOutputTag = @@ -203,36 +209,45 @@ public void endInput() throws IOException { .map(Path::toUri) .map(Object::toString) .collect(Collectors.toList()); - DataStream candidates = + DataStream> candidates = env.fromCollection(fileDirs) .process( - new ProcessFunction() { + new ProcessFunction>() { @Override public void processElement( String dir, - ProcessFunction.Context ctx, - Collector out) { + ProcessFunction>.Context ctx, + Collector> out) { for (FileStatus fileStatus : tryBestListingDirs(new Path(dir))) { if (oldEnough(fileStatus)) { out.collect( - fileStatus.getPath().toUri().toString()); + Pair.of( + fileStatus + .getPath() + .toUri() + .toString(), + fileStatus.getLen())); } } } }); - DataStream deleted = + DataStream deleted = usedFiles .keyBy(f -> f) - .connect(candidates.keyBy(path -> new Path(path).getName())) + .connect( + candidates.keyBy( + pathAndSize -> new Path(pathAndSize.getKey()).getName())) .transform( "files_join", - LONG_TYPE_INFO, - new BoundedTwoInputOperator() { + TypeInformation.of(CleanOrphanFilesResult.class), + new BoundedTwoInputOperator< + String, Pair, CleanOrphanFilesResult>() { private boolean buildEnd; - private long emitted; + private long emittedFilesCount; + private long emittedFilesLen; private final Set used = new HashSet<>(); @@ -254,8 +269,15 @@ public void endInput(int inputId) { case 2: checkState(buildEnd, "Should build ended."); LOG.info("Finish probe phase."); - LOG.info("Clean files: {}", emitted); - output.collect(new StreamRecord<>(emitted)); + LOG.info( + "Clean files count : {}", + emittedFilesCount); + LOG.info("Clean files size : {}", emittedFilesLen); + output.collect( + new StreamRecord<>( + new CleanOrphanFilesResult( + emittedFilesCount, + emittedFilesLen))); break; } } @@ -266,25 +288,34 @@ public void processElement1(StreamRecord element) { } @Override - public void processElement2(StreamRecord element) { + public void processElement2( + StreamRecord> element) { checkState(buildEnd, "Should build ended."); - String value = element.getValue(); + Pair fileInfo = element.getValue(); + String value = fileInfo.getLeft(); Path path = new Path(value); if (!used.contains(path.getName())) { + emittedFilesCount++; + emittedFilesLen += fileInfo.getRight(); fileCleaner.accept(path); LOG.info("Dry clean: {}", path); - emitted++; } } }); - if (deletedInLocal.get() != 0) { - deleted = deleted.union(env.fromData(deletedInLocal.get())); + if (deletedFilesCountInLocal.get() != 0 || deletedFilesLenInBytesInLocal.get() != 0) { + deleted = + deleted.union( + env.fromElements( + new CleanOrphanFilesResult( + deletedFilesCountInLocal.get(), + deletedFilesLenInBytesInLocal.get()))); } + return deleted; } - public static long executeDatabaseOrphanFiles( + public static CleanOrphanFilesResult executeDatabaseOrphanFiles( StreamExecutionEnvironment env, Catalog catalog, long olderThanMillis, @@ -293,12 +324,13 @@ public static long executeDatabaseOrphanFiles( String databaseName, @Nullable String tableName) throws Catalog.DatabaseNotExistException, Catalog.TableNotExistException { - List> orphanFilesCleans = new ArrayList<>(); List tableNames = Collections.singletonList(tableName); if (tableName == null || "*".equals(tableName)) { tableNames = catalog.listTables(databaseName); } + List> orphanFilesCleans = + new ArrayList<>(tableNames.size()); for (String t : tableNames) { Identifier identifier = new Identifier(databaseName, t); Table table = catalog.getTable(identifier); @@ -307,7 +339,7 @@ public static long executeDatabaseOrphanFiles( "Only FileStoreTable supports remove-orphan-files action. The table type is '%s'.", table.getClass().getName()); - DataStream clean = + DataStream clean = new FlinkOrphanFilesClean( (FileStoreTable) table, olderThanMillis, @@ -319,8 +351,8 @@ public static long executeDatabaseOrphanFiles( } } - DataStream result = null; - for (DataStream clean : orphanFilesCleans) { + DataStream result = null; + for (DataStream clean : orphanFilesCleans) { if (result == null) { result = clean; } else { @@ -331,20 +363,24 @@ public static long executeDatabaseOrphanFiles( return sum(result); } - private static long sum(DataStream deleted) { - long deleteCount = 0; + private static CleanOrphanFilesResult sum(DataStream deleted) { + long deletedFilesCount = 0; + long deletedFilesLenInBytes = 0; if (deleted != null) { try { - CloseableIterator iterator = + CloseableIterator iterator = deleted.global().executeAndCollect("OrphanFilesClean"); while (iterator.hasNext()) { - deleteCount += iterator.next(); + CleanOrphanFilesResult cleanOrphanFilesResult = iterator.next(); + deletedFilesCount += cleanOrphanFilesResult.getDeletedFileCount(); + deletedFilesLenInBytes += + cleanOrphanFilesResult.getDeletedFileTotalLenInBytes(); } iterator.close(); } catch (Exception e) { throw new RuntimeException(e); } } - return deleteCount; + return new CleanOrphanFilesResult(deletedFilesCount, deletedFilesLenInBytes); } } diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/procedure/CompactDatabaseProcedure.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/procedure/CompactDatabaseProcedure.java index dd71e974c7b1..80602b755aa5 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/procedure/CompactDatabaseProcedure.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/procedure/CompactDatabaseProcedure.java @@ -29,6 +29,8 @@ import java.util.Map; +import static org.apache.paimon.flink.action.ActionFactory.FULL; +import static org.apache.paimon.flink.action.CompactActionFactory.checkCompactStrategy; import static org.apache.paimon.utils.ParameterUtils.parseCommaSeparatedKeyValues; /** @@ -82,6 +84,10 @@ public class CompactDatabaseProcedure extends ProcedureBase { @ArgumentHint( name = "partition_idle_time", type = @DataTypeHint("STRING"), + isOptional = true), + @ArgumentHint( + name = "compact_strategy", + type = @DataTypeHint("STRING"), isOptional = true) }) public String[] call( @@ -91,7 +97,8 @@ public String[] call( String includingTables, String excludingTables, String tableOptions, - String partitionIdleTime) + String partitionIdleTime, + String compactStrategy) throws Exception { partitionIdleTime = notnull(partitionIdleTime); String warehouse = catalog.warehouse(); @@ -109,6 +116,10 @@ public String[] call( action.withPartitionIdleTime(TimeUtils.parseDuration(partitionIdleTime)); } + if (checkCompactStrategy(compactStrategy)) { + action.withFullCompaction(compactStrategy.trim().equalsIgnoreCase(FULL)); + } + return execute(procedureContext, action, "Compact database job"); } diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/procedure/CompactManifestProcedure.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/procedure/CompactManifestProcedure.java new file mode 100644 index 000000000000..3e52322a6f58 --- /dev/null +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/procedure/CompactManifestProcedure.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.flink.procedure; + +import org.apache.paimon.CoreOptions; +import org.apache.paimon.operation.FileStoreCommit; +import org.apache.paimon.table.FileStoreTable; + +import org.apache.flink.table.annotation.ArgumentHint; +import org.apache.flink.table.annotation.DataTypeHint; +import org.apache.flink.table.annotation.ProcedureHint; +import org.apache.flink.table.procedure.ProcedureContext; + +import java.util.Collections; + +/** Compact manifest file to reduce deleted manifest entries. */ +public class CompactManifestProcedure extends ProcedureBase { + + private static final String COMMIT_USER = "Compact-Manifest-Procedure-Committer"; + + @Override + public String identifier() { + return "compact_manifest"; + } + + @ProcedureHint(argument = {@ArgumentHint(name = "table", type = @DataTypeHint("STRING"))}) + public String[] call(ProcedureContext procedureContext, String tableId) throws Exception { + + FileStoreTable table = + (FileStoreTable) + table(tableId) + .copy( + Collections.singletonMap( + CoreOptions.COMMIT_USER_PREFIX.key(), COMMIT_USER)); + + try (FileStoreCommit commit = + table.store() + .newCommit(table.coreOptions().createCommitUser()) + .ignoreEmptyCommit(false)) { + commit.compactManifest(); + } + return new String[] {"success"}; + } +} diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/procedure/CompactProcedure.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/procedure/CompactProcedure.java index 858906912698..282f5af34043 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/procedure/CompactProcedure.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/procedure/CompactProcedure.java @@ -32,6 +32,8 @@ import java.util.Collections; import java.util.Map; +import static org.apache.paimon.flink.action.ActionFactory.FULL; +import static org.apache.paimon.flink.action.CompactActionFactory.checkCompactStrategy; import static org.apache.paimon.utils.ParameterUtils.getPartitions; import static org.apache.paimon.utils.ParameterUtils.parseCommaSeparatedKeyValues; import static org.apache.paimon.utils.StringUtils.isNullOrWhitespaceOnly; @@ -58,6 +60,10 @@ public class CompactProcedure extends ProcedureBase { @ArgumentHint( name = "partition_idle_time", type = @DataTypeHint("STRING"), + isOptional = true), + @ArgumentHint( + name = "compact_strategy", + type = @DataTypeHint("STRING"), isOptional = true) }) public String[] call( @@ -68,7 +74,8 @@ public String[] call( String orderByColumns, String tableOptions, String where, - String partitionIdleTime) + String partitionIdleTime, + String compactStrategy) throws Exception { String warehouse = catalog.warehouse(); Map catalogOptions = catalog.options(); @@ -90,6 +97,10 @@ public String[] call( if (!isNullOrWhitespaceOnly(partitionIdleTime)) { action.withPartitionIdleTime(TimeUtils.parseDuration(partitionIdleTime)); } + + if (checkCompactStrategy(compactStrategy)) { + action.withFullCompaction(compactStrategy.trim().equalsIgnoreCase(FULL)); + } jobName = "Compact Job"; } else if (!isNullOrWhitespaceOnly(orderStrategy) && !isNullOrWhitespaceOnly(orderByColumns)) { diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/procedure/CreateOrReplaceTagBaseProcedure.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/procedure/CreateOrReplaceTagBaseProcedure.java new file mode 100644 index 000000000000..dba9d46636e6 --- /dev/null +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/procedure/CreateOrReplaceTagBaseProcedure.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.flink.procedure; + +import org.apache.paimon.catalog.Catalog; +import org.apache.paimon.catalog.Identifier; +import org.apache.paimon.table.Table; +import org.apache.paimon.utils.TimeUtils; + +import org.apache.flink.table.annotation.ArgumentHint; +import org.apache.flink.table.annotation.DataTypeHint; +import org.apache.flink.table.annotation.ProcedureHint; +import org.apache.flink.table.procedure.ProcedureContext; + +import javax.annotation.Nullable; + +import java.time.Duration; + +/** A base procedure to create or replace a tag. */ +public abstract class CreateOrReplaceTagBaseProcedure extends ProcedureBase { + + @ProcedureHint( + argument = { + @ArgumentHint(name = "table", type = @DataTypeHint("STRING")), + @ArgumentHint(name = "tag", type = @DataTypeHint("STRING")), + @ArgumentHint( + name = "snapshot_id", + type = @DataTypeHint("BIGINT"), + isOptional = true), + @ArgumentHint( + name = "time_retained", + type = @DataTypeHint("STRING"), + isOptional = true) + }) + public String[] call( + ProcedureContext procedureContext, + String tableId, + String tagName, + @Nullable Long snapshotId, + @Nullable String timeRetained) + throws Catalog.TableNotExistException { + Table table = catalog.getTable(Identifier.fromString(tableId)); + createOrReplaceTag(table, tagName, snapshotId, toDuration(timeRetained)); + return new String[] {"Success"}; + } + + abstract void createOrReplaceTag( + Table table, String tagName, Long snapshotId, Duration timeRetained); + + @Nullable + private static Duration toDuration(@Nullable String s) { + if (s == null) { + return null; + } + + return TimeUtils.parseDuration(s); + } +} diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/procedure/CreateTagProcedure.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/procedure/CreateTagProcedure.java index 3fb51c8d935c..b1af1c93942f 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/procedure/CreateTagProcedure.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/procedure/CreateTagProcedure.java @@ -18,17 +18,7 @@ package org.apache.paimon.flink.procedure; -import org.apache.paimon.catalog.Catalog; -import org.apache.paimon.catalog.Identifier; import org.apache.paimon.table.Table; -import org.apache.paimon.utils.TimeUtils; - -import org.apache.flink.table.annotation.ArgumentHint; -import org.apache.flink.table.annotation.DataTypeHint; -import org.apache.flink.table.annotation.ProcedureHint; -import org.apache.flink.table.procedure.ProcedureContext; - -import javax.annotation.Nullable; import java.time.Duration; @@ -39,46 +29,17 @@ * CALL sys.create_tag('tableId', 'tagName', snapshotId, 'timeRetained') * */ -public class CreateTagProcedure extends ProcedureBase { +public class CreateTagProcedure extends CreateOrReplaceTagBaseProcedure { public static final String IDENTIFIER = "create_tag"; - @ProcedureHint( - argument = { - @ArgumentHint(name = "table", type = @DataTypeHint("STRING")), - @ArgumentHint(name = "tag", type = @DataTypeHint("STRING")), - @ArgumentHint( - name = "snapshot_id", - type = @DataTypeHint("BIGINT"), - isOptional = true), - @ArgumentHint( - name = "time_retained", - type = @DataTypeHint("STRING"), - isOptional = true) - }) - public String[] call( - ProcedureContext procedureContext, - String tableId, - String tagName, - @Nullable Long snapshotId, - @Nullable String timeRetained) - throws Catalog.TableNotExistException { - Table table = catalog.getTable(Identifier.fromString(tableId)); + @Override + void createOrReplaceTag(Table table, String tagName, Long snapshotId, Duration timeRetained) { if (snapshotId == null) { - table.createTag(tagName, toDuration(timeRetained)); + table.createTag(tagName, timeRetained); } else { - table.createTag(tagName, snapshotId, toDuration(timeRetained)); + table.createTag(tagName, snapshotId, timeRetained); } - return new String[] {"Success"}; - } - - @Nullable - private static Duration toDuration(@Nullable String s) { - if (s == null) { - return null; - } - - return TimeUtils.parseDuration(s); } @Override diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/procedure/DeleteBranchProcedure.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/procedure/DeleteBranchProcedure.java index c95fd62bee40..56c649028650 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/procedure/DeleteBranchProcedure.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/procedure/DeleteBranchProcedure.java @@ -49,7 +49,10 @@ public String identifier() { }) public String[] call(ProcedureContext procedureContext, String tableId, String branchStr) throws Catalog.TableNotExistException { - catalog.getTable(Identifier.fromString(tableId)).deleteBranches(branchStr); + Identifier identifier = Identifier.fromString(tableId); + catalog.getTable(identifier).deleteBranches(branchStr); + catalog.invalidateTable( + new Identifier(identifier.getDatabaseName(), identifier.getTableName(), branchStr)); return new String[] {"Success"}; } } diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/procedure/ExpirePartitionsProcedure.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/procedure/ExpirePartitionsProcedure.java index ee6075a927d3..ce282c6800cc 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/procedure/ExpirePartitionsProcedure.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/procedure/ExpirePartitionsProcedure.java @@ -97,9 +97,10 @@ public String identifier() { .catalogEnvironment() .metastoreClientFactory()) .map(MetastoreClient.Factory::create) - .orElse(null)); + .orElse(null), + fileStore.options().partitionExpireMaxNum()); if (maxExpires != null) { - partitionExpire.withMaxExpires(maxExpires); + partitionExpire.withMaxExpireNum(maxExpires); } List> expired = partitionExpire.expire(Long.MAX_VALUE); return expired == null || expired.isEmpty() diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/procedure/ExpireTagsProcedure.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/procedure/ExpireTagsProcedure.java new file mode 100644 index 000000000000..3d8af1de70cc --- /dev/null +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/procedure/ExpireTagsProcedure.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.flink.procedure; + +import org.apache.paimon.catalog.Catalog; +import org.apache.paimon.table.FileStoreTable; +import org.apache.paimon.tag.TagTimeExpire; +import org.apache.paimon.utils.DateTimeUtils; + +import org.apache.flink.table.annotation.ArgumentHint; +import org.apache.flink.table.annotation.DataTypeHint; +import org.apache.flink.table.annotation.ProcedureHint; +import org.apache.flink.table.procedure.ProcedureContext; +import org.apache.flink.types.Row; + +import javax.annotation.Nullable; + +import java.time.LocalDateTime; +import java.util.List; +import java.util.TimeZone; + +/** A procedure to expire tags by time. */ +public class ExpireTagsProcedure extends ProcedureBase { + + private static final String IDENTIFIER = "expire_tags"; + + @ProcedureHint( + argument = { + @ArgumentHint(name = "table", type = @DataTypeHint("STRING")), + @ArgumentHint( + name = "older_than", + type = @DataTypeHint("STRING"), + isOptional = true) + }) + public @DataTypeHint("ROW") Row[] call( + ProcedureContext procedureContext, String tableId, @Nullable String olderThanStr) + throws Catalog.TableNotExistException { + FileStoreTable fileStoreTable = (FileStoreTable) table(tableId); + TagTimeExpire tagTimeExpire = + fileStoreTable.store().newTagCreationManager().getTagTimeExpire(); + if (olderThanStr != null) { + LocalDateTime olderThanTime = + DateTimeUtils.parseTimestampData(olderThanStr, 3, TimeZone.getDefault()) + .toLocalDateTime(); + tagTimeExpire.withOlderThanTime(olderThanTime); + } + List expired = tagTimeExpire.expire(); + return expired.isEmpty() + ? new Row[] {Row.of("No expired tags.")} + : expired.stream().map(Row::of).toArray(Row[]::new); + } + + @Override + public String identifier() { + return IDENTIFIER; + } +} diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/procedure/MergeIntoProcedure.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/procedure/MergeIntoProcedure.java index cf8d7191953e..e297c0bdbb4c 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/procedure/MergeIntoProcedure.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/procedure/MergeIntoProcedure.java @@ -102,7 +102,19 @@ public class MergeIntoProcedure extends ProcedureBase { @ArgumentHint( name = "matched_delete_condition", type = @DataTypeHint("STRING"), - isOptional = true) + isOptional = true), + @ArgumentHint( + name = "not_matched_by_source_upsert_condition", + type = @DataTypeHint("STRING"), + isOptional = true), + @ArgumentHint( + name = "not_matched_by_source_upsert_setting", + type = @DataTypeHint("STRING"), + isOptional = true), + @ArgumentHint( + name = "not_matched_by_source_delete_condition", + type = @DataTypeHint("STRING"), + isOptional = true), }) public String[] call( ProcedureContext procedureContext, @@ -115,7 +127,10 @@ public String[] call( String matchedUpsertSetting, String notMatchedInsertCondition, String notMatchedInsertValues, - String matchedDeleteCondition) { + String matchedDeleteCondition, + String notMatchedBySourceUpsertCondition, + String notMatchedBySourceUpsertSetting, + String notMatchedBySourceDeleteCondition) { targetAlias = notnull(targetAlias); sourceSqls = notnull(sourceSqls); sourceTable = notnull(sourceTable); @@ -125,6 +140,9 @@ public String[] call( notMatchedInsertCondition = notnull(notMatchedInsertCondition); notMatchedInsertValues = notnull(notMatchedInsertValues); matchedDeleteCondition = notnull(matchedDeleteCondition); + notMatchedBySourceUpsertCondition = notnull(notMatchedBySourceUpsertCondition); + notMatchedBySourceUpsertSetting = notnull(notMatchedBySourceUpsertSetting); + notMatchedBySourceDeleteCondition = notnull(notMatchedBySourceDeleteCondition); String warehouse = catalog.warehouse(); Map catalogOptions = catalog.options(); @@ -166,6 +184,20 @@ public String[] call( action.withMatchedDelete(matchedDeleteCondition); } + if (!notMatchedBySourceUpsertCondition.isEmpty() + || !notMatchedBySourceUpsertSetting.isEmpty()) { + String condition = nullable(notMatchedBySourceUpsertCondition); + String values = nullable(notMatchedBySourceUpsertSetting); + checkArgument( + !"*".equals(values), + "not-matched-by-source-upsert does not support setting notMatchedBySourceUpsertSetting to *."); + action.withNotMatchedBySourceUpsert(condition, values); + } + + if (!notMatchedBySourceDeleteCondition.isEmpty()) { + action.withNotMatchedBySourceDelete(notMatchedBySourceDeleteCondition); + } + action.withStreamExecutionEnvironment(procedureContext.getExecutionEnvironment()); action.validate(); diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/procedure/MigrateFileProcedure.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/procedure/MigrateFileProcedure.java index 34b016fe0d36..f2f10d087406 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/procedure/MigrateFileProcedure.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/procedure/MigrateFileProcedure.java @@ -18,6 +18,7 @@ package org.apache.paimon.flink.procedure; +import org.apache.paimon.catalog.Catalog; import org.apache.paimon.catalog.Identifier; import org.apache.paimon.flink.utils.TableMigrationUtils; import org.apache.paimon.migrate.Migrator; @@ -77,7 +78,9 @@ public void migrateHandle( Identifier sourceTableId = Identifier.fromString(sourceTablePath); Identifier targetTableId = Identifier.fromString(targetPaimonTablePath); - if (!(catalog.tableExists(targetTableId))) { + try { + catalog.getTable(targetTableId); + } catch (Catalog.TableNotExistException e) { throw new IllegalArgumentException( "Target paimon table does not exist: " + targetPaimonTablePath); } diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/procedure/PurgeFilesProcedure.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/procedure/PurgeFilesProcedure.java new file mode 100644 index 000000000000..7ee2a3610402 --- /dev/null +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/procedure/PurgeFilesProcedure.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.flink.procedure; + +import org.apache.paimon.catalog.Catalog; +import org.apache.paimon.catalog.Identifier; +import org.apache.paimon.fs.FileIO; +import org.apache.paimon.fs.Path; +import org.apache.paimon.table.FileStoreTable; +import org.apache.paimon.table.Table; + +import org.apache.flink.table.annotation.ArgumentHint; +import org.apache.flink.table.annotation.DataTypeHint; +import org.apache.flink.table.annotation.ProcedureHint; +import org.apache.flink.table.procedure.ProcedureContext; + +import java.io.IOException; +import java.util.Arrays; + +/** + * A procedure to purge files for a table. Usage: + * + * 
    
+ *  -- rollback to the snapshot which earlier or equal than watermark.
+ *  CALL sys.purge_files(`table` => 'tableId')
+ *
    + */ +public class PurgeFilesProcedure extends ProcedureBase { + + public static final String IDENTIFIER = "purge_files"; + + @ProcedureHint(argument = {@ArgumentHint(name = "table", type = @DataTypeHint("STRING"))}) + public String[] call(ProcedureContext procedureContext, String tableId) + throws Catalog.TableNotExistException { + Table table = catalog.getTable(Identifier.fromString(tableId)); + FileStoreTable fileStoreTable = (FileStoreTable) table; + FileIO fileIO = fileStoreTable.fileIO(); + Path tablePath = fileStoreTable.snapshotManager().tablePath(); + try { + Arrays.stream(fileIO.listStatus(tablePath)) + .filter(f -> !f.getPath().getName().contains("schema")) + .forEach( + fileStatus -> { + try { + fileIO.delete(fileStatus.getPath(), true); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + } catch (IOException e) { + throw new RuntimeException(e); + } + return new String[] { + String.format("Success purge files with table: %s.", fileStoreTable.name()) + }; + } + + @Override + public String identifier() { + return IDENTIFIER; + } +} diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/procedure/RefreshObjectTableProcedure.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/procedure/RefreshObjectTableProcedure.java new file mode 100644 index 000000000000..97eb3095f094 --- /dev/null +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/procedure/RefreshObjectTableProcedure.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.flink.procedure; + +import org.apache.paimon.catalog.Catalog; +import org.apache.paimon.table.object.ObjectTable; + +import org.apache.flink.table.annotation.ArgumentHint; +import org.apache.flink.table.annotation.DataTypeHint; +import org.apache.flink.table.annotation.ProcedureHint; +import org.apache.flink.table.procedure.ProcedureContext; +import org.apache.flink.types.Row; + +/** + * Refresh Object Table procedure. Usage: + * + * 
    
+ *  CALL sys.refresh_object_table('tableId')
+ *
    + */ +public class RefreshObjectTableProcedure extends ProcedureBase { + + private static final String IDENTIFIER = "refresh_object_table"; + + @ProcedureHint(argument = {@ArgumentHint(name = "table", type = @DataTypeHint("STRING"))}) + public @DataTypeHint("ROW") Row[] call( + ProcedureContext procedureContext, String tableId) + throws Catalog.TableNotExistException { + ObjectTable table = (ObjectTable) table(tableId); + long fileNumber = table.refresh(); + return new Row[] {Row.of(fileNumber)}; + } + + @Override + public String identifier() { + return IDENTIFIER; + } +} diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/procedure/RemoveOrphanFilesProcedure.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/procedure/RemoveOrphanFilesProcedure.java index 10ad878e0ccb..4cd1b3e00303 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/procedure/RemoveOrphanFilesProcedure.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/procedure/RemoveOrphanFilesProcedure.java @@ -20,6 +20,7 @@ import org.apache.paimon.catalog.Identifier; import org.apache.paimon.flink.orphan.FlinkOrphanFilesClean; +import org.apache.paimon.operation.CleanOrphanFilesResult; import org.apache.paimon.operation.LocalOrphanFilesClean; import org.apache.flink.table.annotation.ArgumentHint; @@ -75,11 +76,11 @@ public String[] call( if (mode == null) { mode = "DISTRIBUTED"; } - long deletedFiles; + CleanOrphanFilesResult cleanOrphanFilesResult; try { switch (mode.toUpperCase(Locale.ROOT)) { case "DISTRIBUTED": - deletedFiles = + cleanOrphanFilesResult = FlinkOrphanFilesClean.executeDatabaseOrphanFiles( procedureContext.getExecutionEnvironment(), catalog, @@ -90,7 +91,7 @@ public String[] call( tableName); break; case "LOCAL": - deletedFiles = + cleanOrphanFilesResult = LocalOrphanFilesClean.executeDatabaseOrphanFiles( catalog, databaseName, @@ -105,7 +106,10 @@ public String[] call( + mode + ". Only 'DISTRIBUTED' and 'LOCAL' are supported."); } - return new String[] {String.valueOf(deletedFiles)}; + return new String[] { + String.valueOf(cleanOrphanFilesResult.getDeletedFileCount()), + String.valueOf(cleanOrphanFilesResult.getDeletedFileTotalLenInBytes()) + }; } catch (Exception e) { throw new RuntimeException(e); } diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/procedure/ReplaceTagProcedure.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/procedure/ReplaceTagProcedure.java new file mode 100644 index 000000000000..6ed6ecc0e512 --- /dev/null +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/procedure/ReplaceTagProcedure.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.flink.procedure; + +import org.apache.paimon.table.Table; + +import java.time.Duration; + +/** A procedure to replace a tag. */ +public class ReplaceTagProcedure extends CreateOrReplaceTagBaseProcedure { + + private static final String IDENTIFIER = "replace_tag"; + + @Override + void createOrReplaceTag(Table table, String tagName, Long snapshotId, Duration timeRetained) { + table.replaceTag(tagName, snapshotId, timeRetained); + } + + @Override + public String identifier() { + return IDENTIFIER; + } +} diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/procedure/ResetConsumerProcedure.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/procedure/ResetConsumerProcedure.java index 5bd4cbaafac6..934ce182a09c 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/procedure/ResetConsumerProcedure.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/procedure/ResetConsumerProcedure.java @@ -67,6 +67,7 @@ public String[] call( fileStoreTable.location(), fileStoreTable.snapshotManager().branch()); if (nextSnapshotId != null) { + fileStoreTable.snapshotManager().snapshot(nextSnapshotId); consumerManager.resetConsumer(consumerId, new Consumer(nextSnapshotId)); } else { consumerManager.deleteConsumer(consumerId); diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/procedure/RollbackToTimestampProcedure.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/procedure/RollbackToTimestampProcedure.java new file mode 100644 index 000000000000..f84dab8eab89 --- /dev/null +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/procedure/RollbackToTimestampProcedure.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.flink.procedure; + +import org.apache.paimon.Snapshot; +import org.apache.paimon.catalog.Catalog; +import org.apache.paimon.catalog.Identifier; +import org.apache.paimon.table.FileStoreTable; +import org.apache.paimon.table.Table; +import org.apache.paimon.utils.Preconditions; + +import org.apache.flink.table.annotation.ArgumentHint; +import org.apache.flink.table.annotation.DataTypeHint; +import org.apache.flink.table.annotation.ProcedureHint; +import org.apache.flink.table.procedure.ProcedureContext; + +/** + * Rollback to timestamp procedure. Usage: + * + * 
    
+ *  -- rollback to the snapshot which earlier or equal than timestamp.
+ *  CALL sys.rollback_to_timestamp(`table` => 'tableId', timestamp => timestamp)
+ *
    + */ +public class RollbackToTimestampProcedure extends ProcedureBase { + + public static final String IDENTIFIER = "rollback_to_timestamp"; + + @ProcedureHint( + argument = { + @ArgumentHint(name = "table", type = @DataTypeHint("STRING")), + @ArgumentHint(name = "timestamp", type = @DataTypeHint("BIGINT")) + }) + public String[] call(ProcedureContext procedureContext, String tableId, Long timestamp) + throws Catalog.TableNotExistException { + Table table = catalog.getTable(Identifier.fromString(tableId)); + FileStoreTable fileStoreTable = (FileStoreTable) table; + Snapshot snapshot = fileStoreTable.snapshotManager().earlierOrEqualTimeMills(timestamp); + Preconditions.checkNotNull( + snapshot, String.format("count not find snapshot earlier than %s", timestamp)); + long snapshotId = snapshot.id(); + fileStoreTable.rollbackTo(snapshotId); + return new String[] {String.format("Success roll back to snapshot: %s .", snapshotId)}; + } + + @Override + public String identifier() { + return IDENTIFIER; + } +} diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/procedure/RollbackToWatermarkProcedure.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/procedure/RollbackToWatermarkProcedure.java new file mode 100644 index 000000000000..ab1ea8080de9 --- /dev/null +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/procedure/RollbackToWatermarkProcedure.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.flink.procedure; + +import org.apache.paimon.Snapshot; +import org.apache.paimon.catalog.Catalog; +import org.apache.paimon.catalog.Identifier; +import org.apache.paimon.table.FileStoreTable; +import org.apache.paimon.table.Table; +import org.apache.paimon.utils.Preconditions; + +import org.apache.flink.table.annotation.ArgumentHint; +import org.apache.flink.table.annotation.DataTypeHint; +import org.apache.flink.table.annotation.ProcedureHint; +import org.apache.flink.table.procedure.ProcedureContext; + +/** + * Rollback to watermark procedure. Usage: + * + * 
    
+ *  -- rollback to the snapshot which earlier or equal than watermark.
+ *  CALL sys.rollback_to_watermark(`table` => 'tableId', watermark => watermark)
+ *
    + */ +public class RollbackToWatermarkProcedure extends ProcedureBase { + + public static final String IDENTIFIER = "rollback_to_watermark"; + + @ProcedureHint( + argument = { + @ArgumentHint(name = "table", type = @DataTypeHint("STRING")), + @ArgumentHint(name = "watermark", type = @DataTypeHint("BIGINT")) + }) + public String[] call(ProcedureContext procedureContext, String tableId, Long watermark) + throws Catalog.TableNotExistException { + Table table = catalog.getTable(Identifier.fromString(tableId)); + FileStoreTable fileStoreTable = (FileStoreTable) table; + Snapshot snapshot = fileStoreTable.snapshotManager().earlierOrEqualWatermark(watermark); + Preconditions.checkNotNull( + snapshot, String.format("count not find snapshot earlier than %s", watermark)); + long snapshotId = snapshot.id(); + fileStoreTable.rollbackTo(snapshotId); + return new String[] {String.format("Success roll back to snapshot: %s .", snapshotId)}; + } + + @Override + public String identifier() { + return IDENTIFIER; + } +} diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/service/QueryAddressRegister.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/service/QueryAddressRegister.java index df3cf7abf2a5..00d527506cfe 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/service/QueryAddressRegister.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/service/QueryAddressRegister.java @@ -23,9 +23,9 @@ import org.apache.paimon.table.FileStoreTable; import org.apache.paimon.table.Table; -import org.apache.flink.configuration.Configuration; -import org.apache.flink.streaming.api.functions.sink.RichSinkFunction; -import org.apache.flink.streaming.api.functions.sink.SinkFunction; +import org.apache.flink.api.connector.sink2.Sink; +import org.apache.flink.api.connector.sink2.SinkWriter; +import org.apache.flink.api.connector.sink2.WriterInitContext; import java.net.InetSocketAddress; import java.util.TreeMap; @@ -33,48 +33,68 @@ import static org.apache.paimon.service.ServiceManager.PRIMARY_KEY_LOOKUP; /** Operator for address server to register addresses to {@link ServiceManager}. */ -public class QueryAddressRegister extends RichSinkFunction { - +public class QueryAddressRegister implements Sink { private final ServiceManager serviceManager; - private transient int numberExecutors; - private transient TreeMap executors; - public QueryAddressRegister(Table table) { this.serviceManager = ((FileStoreTable) table).store().newServiceManager(); } - @Override - public void open(Configuration parameters) throws Exception { - this.executors = new TreeMap<>(); + /** + * Do not annotate with @override here to maintain compatibility with Flink 2.0+. + */ + public SinkWriter createWriter(InitContext context) { + return new QueryAddressRegisterSinkWriter(serviceManager); + } + + /** + * Do not annotate with @override here to maintain compatibility with Flink 1.18-. + */ + public SinkWriter createWriter(WriterInitContext context) { + return new QueryAddressRegisterSinkWriter(serviceManager); } - @Override - public void invoke(InternalRow row, SinkFunction.Context context) { - int numberExecutors = row.getInt(0); - if (this.numberExecutors != 0 && this.numberExecutors != numberExecutors) { - throw new IllegalArgumentException( - String.format( - "Number Executors can not be changed! Old %s , New %s .", - this.numberExecutors, numberExecutors)); + private static class QueryAddressRegisterSinkWriter implements SinkWriter { + private final ServiceManager serviceManager; + + private final TreeMap executors; + + private int numberExecutors; + + private QueryAddressRegisterSinkWriter(ServiceManager serviceManager) { + this.serviceManager = serviceManager; + this.executors = new TreeMap<>(); } - this.numberExecutors = numberExecutors; - int executorId = row.getInt(1); - String hostname = row.getString(2).toString(); - int port = row.getInt(3); + @Override + public void write(InternalRow row, Context context) { + int numberExecutors = row.getInt(0); + if (this.numberExecutors != 0 && this.numberExecutors != numberExecutors) { + throw new IllegalArgumentException( + String.format( + "Number Executors can not be changed! Old %s , New %s .", + this.numberExecutors, numberExecutors)); + } + this.numberExecutors = numberExecutors; + + int executorId = row.getInt(1); + String hostname = row.getString(2).toString(); + int port = row.getInt(3); - executors.put(executorId, new InetSocketAddress(hostname, port)); + executors.put(executorId, new InetSocketAddress(hostname, port)); - if (executors.size() == numberExecutors) { - serviceManager.resetService( - PRIMARY_KEY_LOOKUP, executors.values().toArray(new InetSocketAddress[0])); + if (executors.size() == numberExecutors) { + serviceManager.resetService( + PRIMARY_KEY_LOOKUP, executors.values().toArray(new InetSocketAddress[0])); + } } - } - @Override - public void close() throws Exception { - super.close(); - serviceManager.deleteService(PRIMARY_KEY_LOOKUP); + @Override + public void flush(boolean endOfInput) {} + + @Override + public void close() { + serviceManager.deleteService(PRIMARY_KEY_LOOKUP); + } } } diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/service/QueryExecutorOperator.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/service/QueryExecutorOperator.java index 556c30839688..bf0521d55049 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/service/QueryExecutorOperator.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/service/QueryExecutorOperator.java @@ -23,6 +23,7 @@ import org.apache.paimon.data.GenericRow; import org.apache.paimon.data.InternalRow; import org.apache.paimon.disk.IOManager; +import org.apache.paimon.flink.utils.RuntimeContextUtils; import org.apache.paimon.io.DataFileMeta; import org.apache.paimon.io.DataFileMetaSerializer; import org.apache.paimon.service.network.NetworkUtils; @@ -77,8 +78,8 @@ public void initializeState(StateInitializationContext context) throws Exception this.query = ((FileStoreTable) table).newLocalTableQuery().withIOManager(ioManager); KvQueryServer server = new KvQueryServer( - getRuntimeContext().getIndexOfThisSubtask(), - getRuntimeContext().getNumberOfParallelSubtasks(), + RuntimeContextUtils.getIndexOfThisSubtask(getRuntimeContext()), + RuntimeContextUtils.getNumberOfParallelSubtasks(getRuntimeContext()), NetworkUtils.findHostAddress(), Collections.singletonList(0).iterator(), 1, @@ -96,8 +97,9 @@ public void initializeState(StateInitializationContext context) throws Exception this.output.collect( new StreamRecord<>( GenericRow.of( - getRuntimeContext().getNumberOfParallelSubtasks(), - getRuntimeContext().getIndexOfThisSubtask(), + RuntimeContextUtils.getNumberOfParallelSubtasks( + getRuntimeContext()), + RuntimeContextUtils.getIndexOfThisSubtask(getRuntimeContext()), BinaryString.fromString(address.getHostName()), address.getPort()))); } diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/service/QueryFileMonitor.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/service/QueryFileMonitor.java index 43cf654e91fe..6688503778a0 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/service/QueryFileMonitor.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/service/QueryFileMonitor.java @@ -21,6 +21,9 @@ import org.apache.paimon.CoreOptions; import org.apache.paimon.data.BinaryRow; import org.apache.paimon.data.InternalRow; +import org.apache.paimon.flink.source.AbstractNonCoordinatedSource; +import org.apache.paimon.flink.source.AbstractNonCoordinatedSourceReader; +import org.apache.paimon.flink.source.SimpleSourceSplit; import org.apache.paimon.flink.utils.InternalTypeInfo; import org.apache.paimon.options.Options; import org.apache.paimon.table.FileStoreTable; @@ -31,10 +34,14 @@ import org.apache.paimon.table.source.TableRead; import org.apache.paimon.table.system.FileMonitorTable; -import org.apache.flink.configuration.Configuration; +import org.apache.flink.api.common.eventtime.WatermarkStrategy; +import org.apache.flink.api.connector.source.Boundedness; +import org.apache.flink.api.connector.source.ReaderOutput; +import org.apache.flink.api.connector.source.SourceReader; +import org.apache.flink.api.connector.source.SourceReaderContext; +import org.apache.flink.core.io.InputStatus; import org.apache.flink.streaming.api.datastream.DataStream; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; -import org.apache.flink.streaming.api.functions.source.RichSourceFunction; import java.util.ArrayList; import java.util.List; @@ -49,19 +56,13 @@ *
  • Assigning them to downstream tasks for further processing. * */ -public class QueryFileMonitor extends RichSourceFunction { +public class QueryFileMonitor extends AbstractNonCoordinatedSource { private static final long serialVersionUID = 1L; private final Table table; private final long monitorInterval; - private transient SourceContext ctx; - private transient StreamTableScan scan; - private transient TableRead read; - - private volatile boolean isRunning = true; - public QueryFileMonitor(Table table) { this.table = table; this.monitorInterval = @@ -71,55 +72,53 @@ public QueryFileMonitor(Table table) { } @Override - public void open(Configuration parameters) throws Exception { - FileMonitorTable monitorTable = new FileMonitorTable((FileStoreTable) table); - ReadBuilder readBuilder = monitorTable.newReadBuilder(); - this.scan = readBuilder.newStreamScan(); - this.read = readBuilder.newRead(); + public Boundedness getBoundedness() { + return Boundedness.CONTINUOUS_UNBOUNDED; } @Override - public void run(SourceContext ctx) throws Exception { - this.ctx = ctx; - while (isRunning) { - boolean isEmpty; - synchronized (ctx.getCheckpointLock()) { - if (!isRunning) { - return; - } - isEmpty = doScan(); - } + public SourceReader createReader( + SourceReaderContext sourceReaderContext) throws Exception { + return new Reader(); + } + + private class Reader extends AbstractNonCoordinatedSourceReader { + private transient StreamTableScan scan; + private transient TableRead read; + + @Override + public void start() { + FileMonitorTable monitorTable = new FileMonitorTable((FileStoreTable) table); + ReadBuilder readBuilder = monitorTable.newReadBuilder().dropStats(); + this.scan = readBuilder.newStreamScan(); + this.read = readBuilder.newRead(); + } + + @Override + public InputStatus pollNext(ReaderOutput readerOutput) throws Exception { + boolean isEmpty = doScan(readerOutput); if (isEmpty) { Thread.sleep(monitorInterval); } + return InputStatus.MORE_AVAILABLE; } - } - private boolean doScan() throws Exception { - List records = new ArrayList<>(); - read.createReader(scan.plan()).forEachRemaining(records::add); - records.forEach(ctx::collect); - return records.isEmpty(); - } - - @Override - public void cancel() { - // this is to cover the case where cancel() is called before the run() - if (ctx != null) { - synchronized (ctx.getCheckpointLock()) { - isRunning = false; - } - } else { - isRunning = false; + private boolean doScan(ReaderOutput readerOutput) throws Exception { + List records = new ArrayList<>(); + read.createReader(scan.plan()).forEachRemaining(records::add); + records.forEach(readerOutput::collect); + return records.isEmpty(); } } public static DataStream build(StreamExecutionEnvironment env, Table table) { - return env.addSource( - new QueryFileMonitor(table), - "FileMonitor-" + table.name(), - InternalTypeInfo.fromRowType(FileMonitorTable.getRowType())); + return env.fromSource( + new QueryFileMonitor(table), + WatermarkStrategy.noWatermarks(), + "FileMonitor-" + table.name(), + InternalTypeInfo.fromRowType(FileMonitorTable.getRowType())) + .setParallelism(1); } public static ChannelComputer createChannelComputer() { diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/service/QueryService.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/service/QueryService.java index bd433fe0f00d..752d54cff5a0 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/service/QueryService.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/service/QueryService.java @@ -62,7 +62,7 @@ public static void build(StreamExecutionEnvironment env, Table table, int parall InternalTypeInfo.fromRowType(QueryExecutorOperator.outputType()), executorOperator) .setParallelism(parallelism) - .addSink(new QueryAddressRegister(table)) + .sinkTo(new QueryAddressRegister(table)) .setParallelism(1); sink.getTransformation().setMaxParallelism(1); diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/shuffle/RangeShuffle.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/shuffle/RangeShuffle.java index 54104130438b..8760f1dc5f80 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/shuffle/RangeShuffle.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/shuffle/RangeShuffle.java @@ -27,6 +27,7 @@ import org.apache.paimon.utils.SerializableSupplier; import org.apache.flink.annotation.Internal; +import org.apache.flink.api.common.functions.OpenContext; import org.apache.flink.api.common.functions.Partitioner; import org.apache.flink.api.common.functions.RichMapFunction; import org.apache.flink.api.common.typeinfo.BasicTypeInfo; @@ -182,9 +183,19 @@ public KeyAndSizeExtractor(RowType rowType, boolean isSortBySize) { this.isSortBySize = isSortBySize; } - @Override + /** + * Do not annotate with @override here to maintain compatibility with Flink + * 1.18-. + */ + public void open(OpenContext openContext) throws Exception { + open(new Configuration()); + } + + /** + * Do not annotate with @override here to maintain compatibility with Flink + * 2.0+. + */ public void open(Configuration parameters) throws Exception { - super.open(parameters); InternalRowToSizeVisitor internalRowToSizeVisitor = new InternalRowToSizeVisitor(); fieldSizeCalculator = rowType.getFieldTypes().stream() diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/AdaptiveParallelism.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/AdaptiveParallelism.java index 09b14779c766..0fd7cf9f3565 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/AdaptiveParallelism.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/AdaptiveParallelism.java @@ -18,7 +18,9 @@ package org.apache.paimon.flink.sink; +import org.apache.flink.api.common.ExecutionConfig; import org.apache.flink.configuration.BatchExecutionOptions; +import org.apache.flink.configuration.ReadableConfig; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; /** Get adaptive config from Flink. Only work for Flink 1.17+. */ @@ -27,4 +29,20 @@ public class AdaptiveParallelism { public static boolean isEnabled(StreamExecutionEnvironment env) { return env.getConfiguration().get(BatchExecutionOptions.ADAPTIVE_AUTO_PARALLELISM_ENABLED); } + + /** + * Get default max parallelism of AdaptiveBatchScheduler of Flink. See {@link + * org.apache.flink.runtime.scheduler.adaptivebatch.AdaptiveBatchSchedulerFactory#getDefaultMaxParallelism(Configuration, + * ExecutionConfig)}. + */ + public static int getDefaultMaxParallelism( + ReadableConfig configuration, ExecutionConfig executionConfig) { + return configuration + .getOptional(BatchExecutionOptions.ADAPTIVE_AUTO_PARALLELISM_MAX_PARALLELISM) + .orElse( + executionConfig.getParallelism() == ExecutionConfig.PARALLELISM_DEFAULT + ? BatchExecutionOptions.ADAPTIVE_AUTO_PARALLELISM_MAX_PARALLELISM + .defaultValue() + : executionConfig.getParallelism()); + } } diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/AppendBypassCompactWorkerOperator.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/AppendBypassCompactWorkerOperator.java index 92cd31ea8aa2..977511920a06 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/AppendBypassCompactWorkerOperator.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/AppendBypassCompactWorkerOperator.java @@ -21,7 +21,9 @@ import org.apache.paimon.append.UnawareAppendCompactionTask; import org.apache.paimon.table.FileStoreTable; -import org.apache.flink.streaming.api.operators.ChainingStrategy; +import org.apache.flink.streaming.api.operators.StreamOperator; +import org.apache.flink.streaming.api.operators.StreamOperatorFactory; +import org.apache.flink.streaming.api.operators.StreamOperatorParameters; import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; import org.apache.flink.types.Either; @@ -29,9 +31,11 @@ public class AppendBypassCompactWorkerOperator extends AppendCompactWorkerOperator> { - public AppendBypassCompactWorkerOperator(FileStoreTable table, String commitUser) { - super(table, commitUser); - this.chainingStrategy = ChainingStrategy.HEAD; + private AppendBypassCompactWorkerOperator( + StreamOperatorParameters parameters, + FileStoreTable table, + String commitUser) { + super(parameters, table, commitUser); } @Override @@ -49,4 +53,27 @@ public void processElement( unawareBucketCompactor.processElement(element.getValue().right()); } } + + /** {@link StreamOperatorFactory} of {@link AppendBypassCompactWorkerOperator}. */ + public static class Factory + extends AppendCompactWorkerOperator.Factory< + Either> { + + public Factory(FileStoreTable table, String initialCommitUser) { + super(table, initialCommitUser); + } + + @Override + @SuppressWarnings("unchecked") + public > T createStreamOperator( + StreamOperatorParameters parameters) { + return (T) new AppendBypassCompactWorkerOperator(parameters, table, commitUser); + } + + @Override + @SuppressWarnings("rawtypes") + public Class getStreamOperatorClass(ClassLoader classLoader) { + return AppendBypassCompactWorkerOperator.class; + } + } } diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/AppendCompactWorkerOperator.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/AppendCompactWorkerOperator.java index 52ab75de6b2c..7a3c0231eb65 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/AppendCompactWorkerOperator.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/AppendCompactWorkerOperator.java @@ -27,6 +27,8 @@ import org.apache.paimon.table.sink.CommitMessage; import org.apache.paimon.utils.ExecutorThreadFactory; +import org.apache.flink.streaming.api.operators.StreamOperatorFactory; +import org.apache.flink.streaming.api.operators.StreamOperatorParameters; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -53,8 +55,11 @@ public abstract class AppendCompactWorkerOperator private transient ExecutorService lazyCompactExecutor; - public AppendCompactWorkerOperator(FileStoreTable table, String commitUser) { - super(Options.fromMap(table.options())); + public AppendCompactWorkerOperator( + StreamOperatorParameters parameters, + FileStoreTable table, + String commitUser) { + super(parameters, Options.fromMap(table.options())); this.table = table; this.commitUser = commitUser; } @@ -101,4 +106,17 @@ public void close() throws Exception { this.unawareBucketCompactor.close(); } } + + /** {@link StreamOperatorFactory} of {@link AppendCompactWorkerOperator}. */ + protected abstract static class Factory + extends PrepareCommitOperator.Factory { + protected final FileStoreTable table; + protected final String commitUser; + + protected Factory(FileStoreTable table, String commitUser) { + super(Options.fromMap(table.options())); + this.table = table; + this.commitUser = commitUser; + } + } } diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/AppendOnlyMultiTableCompactionWorkerOperator.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/AppendOnlyMultiTableCompactionWorkerOperator.java index 15e7b9746fe6..83d51f302e51 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/AppendOnlyMultiTableCompactionWorkerOperator.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/AppendOnlyMultiTableCompactionWorkerOperator.java @@ -28,6 +28,9 @@ import org.apache.paimon.utils.ExceptionUtils; import org.apache.paimon.utils.ExecutorThreadFactory; +import org.apache.flink.streaming.api.operators.StreamOperator; +import org.apache.flink.streaming.api.operators.StreamOperatorFactory; +import org.apache.flink.streaming.api.operators.StreamOperatorParameters; import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -62,9 +65,12 @@ public class AppendOnlyMultiTableCompactionWorkerOperator private transient Catalog catalog; - public AppendOnlyMultiTableCompactionWorkerOperator( - Catalog.Loader catalogLoader, String commitUser, Options options) { - super(options); + private AppendOnlyMultiTableCompactionWorkerOperator( + StreamOperatorParameters parameters, + Catalog.Loader catalogLoader, + String commitUser, + Options options) { + super(parameters, options); this.commitUser = commitUser; this.catalogLoader = catalogLoader; } @@ -175,4 +181,34 @@ public void close() throws Exception { ExceptionUtils.throwMultiException(exceptions); } + + /** {@link StreamOperatorFactory} of {@link AppendOnlyMultiTableCompactionWorkerOperator}. */ + public static class Factory + extends PrepareCommitOperator.Factory< + MultiTableUnawareAppendCompactionTask, MultiTableCommittable> { + + private final String commitUser; + private final Catalog.Loader catalogLoader; + + public Factory(Catalog.Loader catalogLoader, String commitUser, Options options) { + super(options); + this.commitUser = commitUser; + this.catalogLoader = catalogLoader; + } + + @Override + @SuppressWarnings("unchecked") + public > T createStreamOperator( + StreamOperatorParameters parameters) { + return (T) + new AppendOnlyMultiTableCompactionWorkerOperator( + parameters, catalogLoader, commitUser, options); + } + + @Override + @SuppressWarnings("rawtypes") + public Class getStreamOperatorClass(ClassLoader classLoader) { + return AppendOnlyMultiTableCompactionWorkerOperator.class; + } + } } diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/AppendOnlySingleTableCompactionWorkerOperator.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/AppendOnlySingleTableCompactionWorkerOperator.java index 4d0201d32461..917a7f64f1a0 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/AppendOnlySingleTableCompactionWorkerOperator.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/AppendOnlySingleTableCompactionWorkerOperator.java @@ -22,6 +22,9 @@ import org.apache.paimon.flink.source.BucketUnawareCompactSource; import org.apache.paimon.table.FileStoreTable; +import org.apache.flink.streaming.api.operators.StreamOperator; +import org.apache.flink.streaming.api.operators.StreamOperatorFactory; +import org.apache.flink.streaming.api.operators.StreamOperatorParameters; import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; /** @@ -31,12 +34,39 @@ public class AppendOnlySingleTableCompactionWorkerOperator extends AppendCompactWorkerOperator { - public AppendOnlySingleTableCompactionWorkerOperator(FileStoreTable table, String commitUser) { - super(table, commitUser); + private AppendOnlySingleTableCompactionWorkerOperator( + StreamOperatorParameters parameters, + FileStoreTable table, + String commitUser) { + super(parameters, table, commitUser); } @Override public void processElement(StreamRecord element) throws Exception { this.unawareBucketCompactor.processElement(element.getValue()); } + + /** {@link StreamOperatorFactory} of {@link AppendOnlySingleTableCompactionWorkerOperator}. */ + public static class Factory + extends AppendCompactWorkerOperator.Factory { + + public Factory(FileStoreTable table, String initialCommitUser) { + super(table, initialCommitUser); + } + + @Override + @SuppressWarnings("unchecked") + public > T createStreamOperator( + StreamOperatorParameters parameters) { + return (T) + new AppendOnlySingleTableCompactionWorkerOperator( + parameters, table, commitUser); + } + + @Override + @SuppressWarnings("rawtypes") + public Class getStreamOperatorClass(ClassLoader classLoader) { + return AppendOnlySingleTableCompactionWorkerOperator.class; + } + } } diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/AutoTagForSavepointCommitterOperator.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/AutoTagForSavepointCommitterOperator.java index 6d27c6019483..0822f0461241 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/AutoTagForSavepointCommitterOperator.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/AutoTagForSavepointCommitterOperator.java @@ -32,18 +32,13 @@ import org.apache.flink.runtime.checkpoint.CheckpointOptions; import org.apache.flink.runtime.jobgraph.OperatorID; import org.apache.flink.runtime.state.CheckpointStreamFactory; -import org.apache.flink.streaming.api.graph.StreamConfig; import org.apache.flink.streaming.api.operators.BoundedOneInput; -import org.apache.flink.streaming.api.operators.ChainingStrategy; import org.apache.flink.streaming.api.operators.OneInputStreamOperator; import org.apache.flink.streaming.api.operators.OperatorSnapshotFutures; -import org.apache.flink.streaming.api.operators.Output; -import org.apache.flink.streaming.api.operators.SetupableStreamOperator; import org.apache.flink.streaming.api.operators.StreamTaskStateInitializer; import org.apache.flink.streaming.api.watermark.Watermark; import org.apache.flink.streaming.runtime.streamrecord.LatencyMarker; import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; -import org.apache.flink.streaming.runtime.tasks.StreamTask; import org.apache.flink.streaming.runtime.watermarkstatus.WatermarkStatus; import java.time.Duration; @@ -58,9 +53,7 @@ * time, tags are automatically created for each flink savepoint. */ public class AutoTagForSavepointCommitterOperator - implements OneInputStreamOperator, - SetupableStreamOperator, - BoundedOneInput { + implements OneInputStreamOperator, BoundedOneInput { public static final String SAVEPOINT_TAG_PREFIX = "savepoint-"; private static final long serialVersionUID = 1L; @@ -256,19 +249,4 @@ public void setKeyContextElement(StreamRecord record) throws Exception public void endInput() throws Exception { commitOperator.endInput(); } - - @Override - public void setup(StreamTask containingTask, StreamConfig config, Output output) { - commitOperator.setup(containingTask, config, output); - } - - @Override - public ChainingStrategy getChainingStrategy() { - return commitOperator.getChainingStrategy(); - } - - @Override - public void setChainingStrategy(ChainingStrategy strategy) { - commitOperator.setChainingStrategy(strategy); - } } diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/AutoTagForSavepointCommitterOperatorFactory.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/AutoTagForSavepointCommitterOperatorFactory.java new file mode 100644 index 000000000000..1787f8e7adce --- /dev/null +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/AutoTagForSavepointCommitterOperatorFactory.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.flink.sink; + +import org.apache.paimon.operation.TagDeletion; +import org.apache.paimon.table.sink.TagCallback; +import org.apache.paimon.utils.SerializableSupplier; +import org.apache.paimon.utils.SnapshotManager; +import org.apache.paimon.utils.TagManager; + +import org.apache.flink.streaming.api.operators.AbstractStreamOperatorFactory; +import org.apache.flink.streaming.api.operators.OneInputStreamOperatorFactory; +import org.apache.flink.streaming.api.operators.StreamOperator; +import org.apache.flink.streaming.api.operators.StreamOperatorParameters; + +import java.time.Duration; +import java.util.List; +import java.util.NavigableSet; +import java.util.TreeSet; + +/** + * {@link org.apache.flink.streaming.api.operators.StreamOperatorFactory} for {@link + * AutoTagForSavepointCommitterOperator}. + */ +public class AutoTagForSavepointCommitterOperatorFactory + extends AbstractStreamOperatorFactory + implements OneInputStreamOperatorFactory { + + private final CommitterOperatorFactory commitOperatorFactory; + + private final SerializableSupplier snapshotManagerFactory; + + private final SerializableSupplier tagManagerFactory; + + private final SerializableSupplier tagDeletionFactory; + + private final SerializableSupplier> callbacksSupplier; + + private final NavigableSet identifiersForTags; + + private final Duration tagTimeRetained; + + public AutoTagForSavepointCommitterOperatorFactory( + CommitterOperatorFactory commitOperatorFactory, + SerializableSupplier snapshotManagerFactory, + SerializableSupplier tagManagerFactory, + SerializableSupplier tagDeletionFactory, + SerializableSupplier> callbacksSupplier, + Duration tagTimeRetained) { + this.commitOperatorFactory = commitOperatorFactory; + this.tagManagerFactory = tagManagerFactory; + this.snapshotManagerFactory = snapshotManagerFactory; + this.tagDeletionFactory = tagDeletionFactory; + this.callbacksSupplier = callbacksSupplier; + this.identifiersForTags = new TreeSet<>(); + this.tagTimeRetained = tagTimeRetained; + } + + @Override + @SuppressWarnings("unchecked") + public > T createStreamOperator( + StreamOperatorParameters parameters) { + return (T) + new AutoTagForSavepointCommitterOperator<>( + commitOperatorFactory.createStreamOperator(parameters), + snapshotManagerFactory, + tagManagerFactory, + tagDeletionFactory, + callbacksSupplier, + tagTimeRetained); + } + + @Override + @SuppressWarnings("rawtypes") + public Class getStreamOperatorClass(ClassLoader classLoader) { + return AutoTagForSavepointCommitterOperator.class; + } +} diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/BatchWriteGeneratorTagOperator.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/BatchWriteGeneratorTagOperator.java index 23202b45077f..1cbcc4b2262f 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/BatchWriteGeneratorTagOperator.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/BatchWriteGeneratorTagOperator.java @@ -28,18 +28,13 @@ import org.apache.flink.runtime.checkpoint.CheckpointOptions; import org.apache.flink.runtime.jobgraph.OperatorID; import org.apache.flink.runtime.state.CheckpointStreamFactory; -import org.apache.flink.streaming.api.graph.StreamConfig; import org.apache.flink.streaming.api.operators.BoundedOneInput; -import org.apache.flink.streaming.api.operators.ChainingStrategy; import org.apache.flink.streaming.api.operators.OneInputStreamOperator; import org.apache.flink.streaming.api.operators.OperatorSnapshotFutures; -import org.apache.flink.streaming.api.operators.Output; -import org.apache.flink.streaming.api.operators.SetupableStreamOperator; import org.apache.flink.streaming.api.operators.StreamTaskStateInitializer; import org.apache.flink.streaming.api.watermark.Watermark; import org.apache.flink.streaming.runtime.streamrecord.LatencyMarker; import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; -import org.apache.flink.streaming.runtime.tasks.StreamTask; import org.apache.flink.streaming.runtime.watermarkstatus.WatermarkStatus; import java.time.Instant; @@ -53,9 +48,7 @@ * completed, the corresponding tag is generated. */ public class BatchWriteGeneratorTagOperator - implements OneInputStreamOperator, - SetupableStreamOperator, - BoundedOneInput { + implements OneInputStreamOperator, BoundedOneInput { private static final String BATCH_WRITE_TAG_PREFIX = "batch-write-"; @@ -250,19 +243,4 @@ public void setKeyContextElement(StreamRecord record) throws Exception public void endInput() throws Exception { commitOperator.endInput(); } - - @Override - public void setup(StreamTask containingTask, StreamConfig config, Output output) { - commitOperator.setup(containingTask, config, output); - } - - @Override - public ChainingStrategy getChainingStrategy() { - return commitOperator.getChainingStrategy(); - } - - @Override - public void setChainingStrategy(ChainingStrategy strategy) { - commitOperator.setChainingStrategy(strategy); - } } diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/BatchWriteGeneratorTagOperatorFactory.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/BatchWriteGeneratorTagOperatorFactory.java new file mode 100644 index 000000000000..e3c0e5c49168 --- /dev/null +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/BatchWriteGeneratorTagOperatorFactory.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.flink.sink; + +import org.apache.paimon.table.FileStoreTable; + +import org.apache.flink.streaming.api.operators.AbstractStreamOperatorFactory; +import org.apache.flink.streaming.api.operators.OneInputStreamOperatorFactory; +import org.apache.flink.streaming.api.operators.StreamOperator; +import org.apache.flink.streaming.api.operators.StreamOperatorParameters; + +/** + * {@link org.apache.flink.streaming.api.operators.StreamOperatorFactory} for {@link + * BatchWriteGeneratorTagOperator}. + */ +public class BatchWriteGeneratorTagOperatorFactory + extends AbstractStreamOperatorFactory + implements OneInputStreamOperatorFactory { + private final CommitterOperatorFactory commitOperatorFactory; + + protected final FileStoreTable table; + + public BatchWriteGeneratorTagOperatorFactory( + CommitterOperatorFactory commitOperatorFactory, + FileStoreTable table) { + this.table = table; + this.commitOperatorFactory = commitOperatorFactory; + } + + @Override + @SuppressWarnings("unchecked") + public > T createStreamOperator( + StreamOperatorParameters parameters) { + return (T) + new BatchWriteGeneratorTagOperator<>( + commitOperatorFactory.createStreamOperator(parameters), table); + } + + @Override + @SuppressWarnings("rawtypes") + public Class getStreamOperatorClass(ClassLoader classLoader) { + return BatchWriteGeneratorTagOperator.class; + } +} diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/CombinedTableCompactorSink.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/CombinedTableCompactorSink.java index 87a28091fa30..25f76ce97683 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/CombinedTableCompactorSink.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/CombinedTableCompactorSink.java @@ -32,8 +32,8 @@ import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator; import org.apache.flink.streaming.api.environment.CheckpointConfig; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; -import org.apache.flink.streaming.api.functions.sink.DiscardingSink; -import org.apache.flink.streaming.api.operators.OneInputStreamOperator; +import org.apache.flink.streaming.api.functions.sink.v2.DiscardingSink; +import org.apache.flink.streaming.api.operators.OneInputStreamOperatorFactory; import org.apache.flink.table.data.RowData; import java.io.Serializable; @@ -57,11 +57,15 @@ public class CombinedTableCompactorSink implements Serializable { private final Catalog.Loader catalogLoader; private final boolean ignorePreviousFiles; + private final boolean fullCompaction; + private final Options options; - public CombinedTableCompactorSink(Catalog.Loader catalogLoader, Options options) { + public CombinedTableCompactorSink( + Catalog.Loader catalogLoader, Options options, boolean fullCompaction) { this.catalogLoader = catalogLoader; this.ignorePreviousFiles = false; + this.fullCompaction = fullCompaction; this.options = options; } @@ -104,7 +108,10 @@ public DataStream doWrite( String.format("%s-%s", "Multi-Bucket-Table", WRITER_NAME), new MultiTableCommittableTypeInfo(), combinedMultiComacptionWriteOperator( - env.getCheckpointConfig(), isStreaming, commitUser)) + env.getCheckpointConfig(), + isStreaming, + fullCompaction, + commitUser)) .setParallelism(awareBucketTableSource.getParallelism()); SingleOutputStreamOperator unawareBucketTableRewriter = @@ -112,7 +119,7 @@ public DataStream doWrite( .transform( String.format("%s-%s", "Unaware-Bucket-Table", WRITER_NAME), new MultiTableCommittableTypeInfo(), - new AppendOnlyMultiTableCompactionWorkerOperator( + new AppendOnlyMultiTableCompactionWorkerOperator.Factory( catalogLoader, commitUser, options)) .setParallelism(unawareBucketTableSource.getParallelism()); @@ -153,28 +160,34 @@ protected DataStreamSink doCommit( .transform( GLOBAL_COMMITTER_NAME, new MultiTableCommittableTypeInfo(), - new CommitterOperator<>( + new CommitterOperatorFactory<>( streamingCheckpointEnabled, false, - options.get(SINK_COMMITTER_OPERATOR_CHAINING), commitUser, createCommitterFactory(isStreaming), createCommittableStateManager(), options.get(END_INPUT_WATERMARK))) .setParallelism(written.getParallelism()); - return committed.addSink(new DiscardingSink<>()).name("end").setParallelism(1); + if (!options.get(SINK_COMMITTER_OPERATOR_CHAINING)) { + committed = committed.startNewChain(); + } + return committed.sinkTo(new DiscardingSink<>()).name("end").setParallelism(1); } // TODO:refactor FlinkSink to adopt this sink - protected OneInputStreamOperator + protected OneInputStreamOperatorFactory combinedMultiComacptionWriteOperator( - CheckpointConfig checkpointConfig, boolean isStreaming, String commitUser) { - return new MultiTablesStoreCompactOperator( + CheckpointConfig checkpointConfig, + boolean isStreaming, + boolean fullCompaction, + String commitUser) { + return new MultiTablesStoreCompactOperator.Factory( catalogLoader, commitUser, checkpointConfig, isStreaming, ignorePreviousFiles, + fullCompaction, options); } diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/CommittableTypeInfo.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/CommittableTypeInfo.java index dcb87238b833..92e826a91379 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/CommittableTypeInfo.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/CommittableTypeInfo.java @@ -21,6 +21,7 @@ import org.apache.paimon.table.sink.CommitMessageSerializer; import org.apache.flink.api.common.ExecutionConfig; +import org.apache.flink.api.common.serialization.SerializerConfig; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.common.typeutils.TypeSerializer; @@ -57,7 +58,16 @@ public boolean isKeyType() { return false; } - @Override + /** + * Do not annotate with @override here to maintain compatibility with Flink 1.18-. + */ + public TypeSerializer createSerializer(SerializerConfig config) { + return this.createSerializer((ExecutionConfig) null); + } + + /** + * Do not annotate with @override here to maintain compatibility with Flink 2.0+. + */ public TypeSerializer createSerializer(ExecutionConfig config) { // no copy, so that data from writer is directly going into committer while chaining return new NoneCopyVersionedSerializerTypeSerializerProxy( diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/CommitterOperator.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/CommitterOperator.java index 2ec90b8c6c40..383cbcd6ebf7 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/CommitterOperator.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/CommitterOperator.java @@ -18,14 +18,15 @@ package org.apache.paimon.flink.sink; +import org.apache.paimon.flink.utils.RuntimeContextUtils; import org.apache.paimon.utils.Preconditions; import org.apache.flink.runtime.state.StateInitializationContext; import org.apache.flink.runtime.state.StateSnapshotContext; import org.apache.flink.streaming.api.operators.AbstractStreamOperator; import org.apache.flink.streaming.api.operators.BoundedOneInput; -import org.apache.flink.streaming.api.operators.ChainingStrategy; import org.apache.flink.streaming.api.operators.OneInputStreamOperator; +import org.apache.flink.streaming.api.operators.StreamOperatorParameters; import org.apache.flink.streaming.api.watermark.Watermark; import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; @@ -90,26 +91,9 @@ public class CommitterOperator extends AbstractStreamOpe private final Long endInputWatermark; public CommitterOperator( + StreamOperatorParameters parameters, boolean streamingCheckpointEnabled, boolean forceSingleParallelism, - boolean chaining, - String initialCommitUser, - Committer.Factory committerFactory, - CommittableStateManager committableStateManager) { - this( - streamingCheckpointEnabled, - forceSingleParallelism, - chaining, - initialCommitUser, - committerFactory, - committableStateManager, - null); - } - - public CommitterOperator( - boolean streamingCheckpointEnabled, - boolean forceSingleParallelism, - boolean chaining, String initialCommitUser, Committer.Factory committerFactory, CommittableStateManager committableStateManager, @@ -121,7 +105,10 @@ public CommitterOperator( this.committerFactory = checkNotNull(committerFactory); this.committableStateManager = committableStateManager; this.endInputWatermark = endInputWatermark; - setChainingStrategy(chaining ? ChainingStrategy.ALWAYS : ChainingStrategy.HEAD); + this.setup( + parameters.getContainingTask(), + parameters.getStreamConfig(), + parameters.getOutput()); } @Override @@ -129,7 +116,9 @@ public void initializeState(StateInitializationContext context) throws Exception super.initializeState(context); Preconditions.checkArgument( - !forceSingleParallelism || getRuntimeContext().getNumberOfParallelSubtasks() == 1, + !forceSingleParallelism + || RuntimeContextUtils.getNumberOfParallelSubtasks(getRuntimeContext()) + == 1, "Committer Operator parallelism in paimon MUST be one."); this.currentWatermark = Long.MIN_VALUE; diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/CommitterOperatorFactory.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/CommitterOperatorFactory.java new file mode 100644 index 000000000000..cce3d4e176bf --- /dev/null +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/CommitterOperatorFactory.java @@ -0,0 +1,115 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.flink.sink; + +import org.apache.flink.streaming.api.operators.AbstractStreamOperatorFactory; +import org.apache.flink.streaming.api.operators.OneInputStreamOperatorFactory; +import org.apache.flink.streaming.api.operators.StreamOperator; +import org.apache.flink.streaming.api.operators.StreamOperatorParameters; + +import java.util.NavigableMap; +import java.util.TreeMap; + +import static org.apache.paimon.utils.Preconditions.checkNotNull; + +/** + * {@link org.apache.flink.streaming.api.operators.StreamOperatorFactory} for {@link + * CommitterOperator}. + */ +public class CommitterOperatorFactory + extends AbstractStreamOperatorFactory + implements OneInputStreamOperatorFactory { + protected final boolean streamingCheckpointEnabled; + + /** Whether to check the parallelism while runtime. */ + protected final boolean forceSingleParallelism; + /** + * This commitUser is valid only for new jobs. After the job starts, this commitUser will be + * recorded into the states of write and commit operators. When the job restarts, commitUser + * will be recovered from states and this value is ignored. + */ + protected final String initialCommitUser; + + /** Group the committable by the checkpoint id. */ + protected final NavigableMap committablesPerCheckpoint; + + protected final Committer.Factory committerFactory; + + protected final CommittableStateManager committableStateManager; + + /** + * Aggregate committables to global committables and commit the global committables to the + * external system. + */ + protected Committer committer; + + protected final Long endInputWatermark; + + public CommitterOperatorFactory( + boolean streamingCheckpointEnabled, + boolean forceSingleParallelism, + String initialCommitUser, + Committer.Factory committerFactory, + CommittableStateManager committableStateManager) { + this( + streamingCheckpointEnabled, + forceSingleParallelism, + initialCommitUser, + committerFactory, + committableStateManager, + null); + } + + public CommitterOperatorFactory( + boolean streamingCheckpointEnabled, + boolean forceSingleParallelism, + String initialCommitUser, + Committer.Factory committerFactory, + CommittableStateManager committableStateManager, + Long endInputWatermark) { + this.streamingCheckpointEnabled = streamingCheckpointEnabled; + this.forceSingleParallelism = forceSingleParallelism; + this.initialCommitUser = initialCommitUser; + this.committablesPerCheckpoint = new TreeMap<>(); + this.committerFactory = checkNotNull(committerFactory); + this.committableStateManager = committableStateManager; + this.endInputWatermark = endInputWatermark; + } + + @Override + @SuppressWarnings("unchecked") + public > T createStreamOperator( + StreamOperatorParameters parameters) { + return (T) + new CommitterOperator<>( + parameters, + streamingCheckpointEnabled, + forceSingleParallelism, + initialCommitUser, + committerFactory, + committableStateManager, + endInputWatermark); + } + + @Override + @SuppressWarnings("rawtypes") + public Class getStreamOperatorClass(ClassLoader classLoader) { + return CommitterOperator.class; + } +} diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/CompactionTaskTypeInfo.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/CompactionTaskTypeInfo.java index 47defa61a971..6510a85b800a 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/CompactionTaskTypeInfo.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/CompactionTaskTypeInfo.java @@ -22,6 +22,7 @@ import org.apache.paimon.table.sink.CompactionTaskSerializer; import org.apache.flink.api.common.ExecutionConfig; +import org.apache.flink.api.common.serialization.SerializerConfig; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.common.typeutils.TypeSerializer; @@ -58,7 +59,16 @@ public boolean isKeyType() { return false; } - @Override + /** + * Do not annotate with @override here to maintain compatibility with Flink 1.18-. + */ + public TypeSerializer createSerializer(SerializerConfig config) { + return this.createSerializer((ExecutionConfig) null); + } + + /** + * Do not annotate with @override here to maintain compatibility with Flink 2.0+. + */ public TypeSerializer createSerializer(ExecutionConfig config) { // we don't need copy for task return new NoneCopyVersionedSerializerTypeSerializerProxy( diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/CompactorSink.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/CompactorSink.java index 7dc3ab1150b0..a9c6031dfa34 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/CompactorSink.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/CompactorSink.java @@ -21,7 +21,7 @@ import org.apache.paimon.manifest.ManifestCommittable; import org.apache.paimon.table.FileStoreTable; -import org.apache.flink.streaming.api.operators.OneInputStreamOperator; +import org.apache.flink.streaming.api.operators.OneInputStreamOperatorFactory; import org.apache.flink.table.data.RowData; /** {@link FlinkSink} for dedicated compact jobs. */ @@ -29,14 +29,17 @@ public class CompactorSink extends FlinkSink { private static final long serialVersionUID = 1L; - public CompactorSink(FileStoreTable table) { + private final boolean fullCompaction; + + public CompactorSink(FileStoreTable table, boolean fullCompaction) { super(table, false); + this.fullCompaction = fullCompaction; } @Override - protected OneInputStreamOperator createWriteOperator( + protected OneInputStreamOperatorFactory createWriteOperatorFactory( StoreSinkWrite.Provider writeProvider, String commitUser) { - return new StoreCompactOperator(table, writeProvider, commitUser); + return new StoreCompactOperator.Factory(table, writeProvider, commitUser, fullCompaction); } @Override diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/CompactorSinkBuilder.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/CompactorSinkBuilder.java index 926155cabf29..2d84ae6726fd 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/CompactorSinkBuilder.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/CompactorSinkBuilder.java @@ -37,8 +37,11 @@ public class CompactorSinkBuilder { private DataStream input; - public CompactorSinkBuilder(FileStoreTable table) { + private final boolean fullCompaction; + + public CompactorSinkBuilder(FileStoreTable table, boolean fullCompaction) { this.table = table; + this.fullCompaction = fullCompaction; } public CompactorSinkBuilder withInput(DataStream input) { @@ -66,6 +69,6 @@ private DataStreamSink buildForBucketAware() { .orElse(null); DataStream partitioned = partition(input, new BucketsRowChannelComputer(), parallelism); - return new CompactorSink(table).sinkFrom(partitioned); + return new CompactorSink(table, fullCompaction).sinkFrom(partitioned); } } diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/DynamicBucketRowWriteOperator.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/DynamicBucketRowWriteOperator.java index 53b9be457c3d..b31a1af05224 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/DynamicBucketRowWriteOperator.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/DynamicBucketRowWriteOperator.java @@ -22,6 +22,9 @@ import org.apache.paimon.table.FileStoreTable; import org.apache.flink.api.java.tuple.Tuple2; +import org.apache.flink.streaming.api.operators.StreamOperator; +import org.apache.flink.streaming.api.operators.StreamOperatorFactory; +import org.apache.flink.streaming.api.operators.StreamOperatorParameters; import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; /** @@ -32,11 +35,12 @@ public class DynamicBucketRowWriteOperator private static final long serialVersionUID = 1L; - public DynamicBucketRowWriteOperator( + private DynamicBucketRowWriteOperator( + StreamOperatorParameters parameters, FileStoreTable table, StoreSinkWrite.Provider storeSinkWriteProvider, String initialCommitUser) { - super(table, storeSinkWriteProvider, initialCommitUser); + super(parameters, table, storeSinkWriteProvider, initialCommitUser); } @Override @@ -49,4 +53,30 @@ public void processElement(StreamRecord> element) throws Exception { write.write(element.getValue().f0, element.getValue().f1); } + + /** {@link StreamOperatorFactory} of {@link DynamicBucketRowWriteOperator}. */ + public static class Factory extends TableWriteOperator.Factory> { + + public Factory( + FileStoreTable table, + StoreSinkWrite.Provider storeSinkWriteProvider, + String initialCommitUser) { + super(table, storeSinkWriteProvider, initialCommitUser); + } + + @Override + @SuppressWarnings("unchecked") + public > T createStreamOperator( + StreamOperatorParameters parameters) { + return (T) + new DynamicBucketRowWriteOperator( + parameters, table, storeSinkWriteProvider, initialCommitUser); + } + + @Override + @SuppressWarnings("rawtypes") + public Class getStreamOperatorClass(ClassLoader classLoader) { + return DynamicBucketRowWriteOperator.class; + } + } } diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/DynamicBucketSink.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/DynamicBucketSink.java index cf697108fd32..c2299a7e8699 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/DynamicBucketSink.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/DynamicBucketSink.java @@ -24,18 +24,22 @@ import org.apache.paimon.table.sink.ChannelComputer; import org.apache.paimon.table.sink.PartitionKeyExtractor; import org.apache.paimon.utils.SerializableFunction; +import org.apache.paimon.utils.StringUtils; import org.apache.flink.api.common.typeinfo.BasicTypeInfo; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.typeutils.TupleTypeInfo; import org.apache.flink.streaming.api.datastream.DataStream; import org.apache.flink.streaming.api.datastream.DataStreamSink; +import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator; import javax.annotation.Nullable; import java.util.Map; import static org.apache.paimon.CoreOptions.createCommitUser; +import static org.apache.paimon.flink.FlinkConnectorOptions.SINK_OPERATOR_UID_SUFFIX; +import static org.apache.paimon.flink.FlinkConnectorOptions.generateCustomUid; import static org.apache.paimon.flink.sink.FlinkStreamPartitioner.partition; /** Sink for dynamic bucket table. */ @@ -43,6 +47,8 @@ public abstract class DynamicBucketSink extends FlinkWriteSink overwritePartition) { super(table, overwritePartition); @@ -88,11 +94,20 @@ public DataStreamSink build(DataStream input, @Nullable Integer parallelis initialCommitUser, table, numAssigners, extractorFunction(), false); TupleTypeInfo> rowWithBucketType = new TupleTypeInfo<>(partitionByKeyHash.getType(), BasicTypeInfo.INT_TYPE_INFO); - DataStream> bucketAssigned = + SingleOutputStreamOperator> bucketAssigned = partitionByKeyHash - .transform("dynamic-bucket-assigner", rowWithBucketType, assignerOperator) + .transform( + DYNAMIC_BUCKET_ASSIGNER_NAME, rowWithBucketType, assignerOperator) .setParallelism(partitionByKeyHash.getParallelism()); + String uidSuffix = table.options().get(SINK_OPERATOR_UID_SUFFIX.key()); + if (!StringUtils.isNullOrWhitespaceOnly(uidSuffix)) { + bucketAssigned = + bucketAssigned.uid( + generateCustomUid( + DYNAMIC_BUCKET_ASSIGNER_NAME, table.name(), uidSuffix)); + } + // 3. shuffle by partition & bucket DataStream> partitionByBucket = partition(bucketAssigned, channelComputer2(), parallelism); diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/FixedBucketSink.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/FixedBucketSink.java index 613bf369b052..402abb4d5aac 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/FixedBucketSink.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/FixedBucketSink.java @@ -21,7 +21,7 @@ import org.apache.paimon.data.InternalRow; import org.apache.paimon.table.FileStoreTable; -import org.apache.flink.streaming.api.operators.OneInputStreamOperator; +import org.apache.flink.streaming.api.operators.OneInputStreamOperatorFactory; import javax.annotation.Nullable; @@ -43,8 +43,9 @@ public FixedBucketSink( } @Override - protected OneInputStreamOperator createWriteOperator( + protected OneInputStreamOperatorFactory createWriteOperatorFactory( StoreSinkWrite.Provider writeProvider, String commitUser) { - return new RowDataStoreWriteOperator(table, logSinkFunction, writeProvider, commitUser); + return new RowDataStoreWriteOperator.Factory( + table, logSinkFunction, writeProvider, commitUser); } } diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/FlinkSink.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/FlinkSink.java index e483e3c19f74..002f5887b5f0 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/FlinkSink.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/FlinkSink.java @@ -21,6 +21,9 @@ import org.apache.paimon.CoreOptions; import org.apache.paimon.CoreOptions.ChangelogProducer; import org.apache.paimon.CoreOptions.TagCreationMode; +import org.apache.paimon.flink.compact.changelog.ChangelogCompactCoordinateOperator; +import org.apache.paimon.flink.compact.changelog.ChangelogCompactWorkerOperator; +import org.apache.paimon.flink.compact.changelog.ChangelogTaskTypeInfo; import org.apache.paimon.manifest.ManifestCommittable; import org.apache.paimon.options.MemorySize; import org.apache.paimon.options.Options; @@ -31,6 +34,7 @@ import org.apache.flink.api.common.RuntimeExecutionMode; import org.apache.flink.api.common.operators.SlotSharingGroup; import org.apache.flink.api.dag.Transformation; +import org.apache.flink.api.java.typeutils.EitherTypeInfo; import org.apache.flink.configuration.ExecutionOptions; import org.apache.flink.configuration.ReadableConfig; import org.apache.flink.streaming.api.CheckpointingMode; @@ -38,10 +42,9 @@ import org.apache.flink.streaming.api.datastream.DataStreamSink; import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator; import org.apache.flink.streaming.api.environment.CheckpointConfig; -import org.apache.flink.streaming.api.environment.ExecutionCheckpointingOptions; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; -import org.apache.flink.streaming.api.functions.sink.DiscardingSink; -import org.apache.flink.streaming.api.operators.OneInputStreamOperator; +import org.apache.flink.streaming.api.functions.sink.v2.DiscardingSink; +import org.apache.flink.streaming.api.operators.OneInputStreamOperatorFactory; import org.apache.flink.table.api.config.ExecutionConfigOptions; import javax.annotation.Nullable; @@ -54,6 +57,7 @@ import static org.apache.paimon.CoreOptions.FULL_COMPACTION_DELTA_COMMITS; import static org.apache.paimon.CoreOptions.createCommitUser; +import static org.apache.paimon.flink.FlinkConnectorOptions.CHANGELOG_PRECOMMIT_COMPACT; import static org.apache.paimon.flink.FlinkConnectorOptions.CHANGELOG_PRODUCER_FULL_COMPACTION_TRIGGER_INTERVAL; import static org.apache.paimon.flink.FlinkConnectorOptions.END_INPUT_WATERMARK; import static org.apache.paimon.flink.FlinkConnectorOptions.SINK_AUTO_TAG_FOR_SAVEPOINT; @@ -61,7 +65,9 @@ import static org.apache.paimon.flink.FlinkConnectorOptions.SINK_COMMITTER_MEMORY; import static org.apache.paimon.flink.FlinkConnectorOptions.SINK_COMMITTER_OPERATOR_CHAINING; import static org.apache.paimon.flink.FlinkConnectorOptions.SINK_MANAGED_WRITER_BUFFER_MEMORY; +import static org.apache.paimon.flink.FlinkConnectorOptions.SINK_OPERATOR_UID_SUFFIX; import static org.apache.paimon.flink.FlinkConnectorOptions.SINK_USE_MANAGED_MEMORY; +import static org.apache.paimon.flink.FlinkConnectorOptions.generateCustomUid; import static org.apache.paimon.flink.utils.ManagedMemoryUtils.declareManagedMemory; import static org.apache.paimon.utils.Preconditions.checkArgument; @@ -214,7 +220,7 @@ public DataStream doWrite( + " : " + table.name(), new CommittableTypeInfo(), - createWriteOperator( + createWriteOperatorFactory( createWriteProvider( env.getCheckpointConfig(), isStreaming, @@ -223,9 +229,31 @@ public DataStream doWrite( .setParallelism(parallelism == null ? input.getParallelism() : parallelism); Options options = Options.fromMap(table.options()); + + String uidSuffix = options.get(SINK_OPERATOR_UID_SUFFIX); + if (options.get(SINK_OPERATOR_UID_SUFFIX) != null) { + written = written.uid(generateCustomUid(WRITER_NAME, table.name(), uidSuffix)); + } + if (options.get(SINK_USE_MANAGED_MEMORY)) { declareManagedMemory(written, options.get(SINK_MANAGED_WRITER_BUFFER_MEMORY)); } + + if (options.get(CHANGELOG_PRECOMMIT_COMPACT)) { + written = + written.transform( + "Changelog Compact Coordinator", + new EitherTypeInfo<>( + new CommittableTypeInfo(), new ChangelogTaskTypeInfo()), + new ChangelogCompactCoordinateOperator(table)) + .forceNonParallel() + .transform( + "Changelog Compact Worker", + new CommittableTypeInfo(), + new ChangelogCompactWorkerOperator(table)) + .setParallelism(written.getParallelism()); + } + return written; } @@ -240,11 +268,10 @@ protected DataStreamSink doCommit(DataStream written, String com } Options options = Options.fromMap(table.options()); - OneInputStreamOperator committerOperator = - new CommitterOperator<>( + OneInputStreamOperatorFactory committerOperator = + new CommitterOperatorFactory<>( streamingCheckpointEnabled, true, - options.get(SINK_COMMITTER_OPERATOR_CHAINING), commitUser, createCommitterFactory(), createCommittableStateManager(), @@ -252,8 +279,9 @@ protected DataStreamSink doCommit(DataStream written, String com if (options.get(SINK_AUTO_TAG_FOR_SAVEPOINT)) { committerOperator = - new AutoTagForSavepointCommitterOperator<>( - (CommitterOperator) committerOperator, + new AutoTagForSavepointCommitterOperatorFactory<>( + (CommitterOperatorFactory) + committerOperator, table::snapshotManager, table::tagManager, () -> table.store().newTagDeletion(), @@ -263,8 +291,9 @@ protected DataStreamSink doCommit(DataStream written, String com if (conf.get(ExecutionOptions.RUNTIME_MODE) == RuntimeExecutionMode.BATCH && table.coreOptions().tagCreationMode() == TagCreationMode.BATCH) { committerOperator = - new BatchWriteGeneratorTagOperator<>( - (CommitterOperator) committerOperator, + new BatchWriteGeneratorTagOperatorFactory<>( + (CommitterOperatorFactory) + committerOperator, table); } SingleOutputStreamOperator committed = @@ -274,9 +303,20 @@ protected DataStreamSink doCommit(DataStream written, String com committerOperator) .setParallelism(1) .setMaxParallelism(1); + if (options.get(SINK_OPERATOR_UID_SUFFIX) != null) { + committed = + committed.uid( + generateCustomUid( + GLOBAL_COMMITTER_NAME, + table.name(), + options.get(SINK_OPERATOR_UID_SUFFIX))); + } + if (!options.get(SINK_COMMITTER_OPERATOR_CHAINING)) { + committed = committed.startNewChain(); + } configureGlobalCommitter( committed, options.get(SINK_COMMITTER_CPU), options.get(SINK_COMMITTER_MEMORY)); - return committed.addSink(new DiscardingSink<>()).name("end").setParallelism(1); + return committed.sinkTo(new DiscardingSink<>()).name("end").setParallelism(1); } public static void configureGlobalCommitter( @@ -301,13 +341,11 @@ public static void assertStreamingConfiguration(StreamExecutionEnvironment env) checkArgument( !env.getCheckpointConfig().isUnalignedCheckpointsEnabled(), "Paimon sink currently does not support unaligned checkpoints. Please set " - + ExecutionCheckpointingOptions.ENABLE_UNALIGNED.key() - + " to false."); + + "execution.checkpointing.unaligned.enabled to false."); checkArgument( env.getCheckpointConfig().getCheckpointingMode() == CheckpointingMode.EXACTLY_ONCE, "Paimon sink currently only supports EXACTLY_ONCE checkpoint mode. Please set " - + ExecutionCheckpointingOptions.CHECKPOINTING_MODE.key() - + " to exactly-once"); + + "execution.checkpointing.mode to exactly-once"); } public static void assertBatchAdaptiveParallelism( @@ -328,7 +366,7 @@ public static void assertBatchAdaptiveParallelism( } } - protected abstract OneInputStreamOperator createWriteOperator( + protected abstract OneInputStreamOperatorFactory createWriteOperatorFactory( StoreSinkWrite.Provider writeProvider, String commitUser); protected abstract Committer.Factory createCommitterFactory(); diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/FlinkSinkBuilder.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/FlinkSinkBuilder.java index 1ff12ac93581..ecaa5678dd0b 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/FlinkSinkBuilder.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/FlinkSinkBuilder.java @@ -51,7 +51,6 @@ import java.util.List; import java.util.Map; -import static org.apache.flink.configuration.CoreOptions.DEFAULT_PARALLELISM; import static org.apache.paimon.flink.FlinkConnectorOptions.CLUSTERING_SAMPLE_FACTOR; import static org.apache.paimon.flink.FlinkConnectorOptions.CLUSTERING_STRATEGY; import static org.apache.paimon.flink.FlinkConnectorOptions.MIN_CLUSTERING_SAMPLE_FACTOR; @@ -223,7 +222,7 @@ public DataStreamSink build() { .transform( "local merge", input.getType(), - new LocalMergeOperator(table.schema())) + new LocalMergeOperator.Factory(table.schema())) .setParallelism(input.getParallelism()); } @@ -266,6 +265,16 @@ protected DataStreamSink buildDynamicBucketSink( } protected DataStreamSink buildForFixedBucket(DataStream input) { + int bucketNums = table.bucketSpec().getNumBuckets(); + if (parallelism == null + && bucketNums < input.getParallelism() + && table.partitionKeys().isEmpty()) { + // For non-partitioned table, if the bucketNums is less than job parallelism. + LOG.warn( + "For non-partitioned table, if bucketNums is less than the parallelism of inputOperator," + + " then the parallelism of writerOperator will be set to bucketNums."); + parallelism = bucketNums; + } DataStream partitioned = partition( input, @@ -318,11 +327,11 @@ private void setParallelismIfAdaptiveConflict() { parallelismSource = "input parallelism"; parallelism = input.getParallelism(); } else { - parallelismSource = DEFAULT_PARALLELISM.key(); + parallelismSource = "AdaptiveBatchScheduler's default max parallelism"; parallelism = - input.getExecutionEnvironment() - .getConfiguration() - .get(DEFAULT_PARALLELISM); + AdaptiveParallelism.getDefaultMaxParallelism( + input.getExecutionEnvironment().getConfiguration(), + input.getExecutionConfig()); } String msg = String.format( diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/FlinkStreamPartitioner.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/FlinkStreamPartitioner.java index 78e532053e15..f9b81760c30e 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/FlinkStreamPartitioner.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/FlinkStreamPartitioner.java @@ -24,6 +24,7 @@ import org.apache.flink.runtime.plugable.SerializationDelegate; import org.apache.flink.streaming.api.datastream.DataStream; import org.apache.flink.streaming.api.transformations.PartitionTransformation; +import org.apache.flink.streaming.runtime.partitioner.RebalancePartitioner; import org.apache.flink.streaming.runtime.partitioner.StreamPartitioner; import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; @@ -77,4 +78,14 @@ public static DataStream partition( } return new DataStream<>(input.getExecutionEnvironment(), partitioned); } + + public static DataStream rebalance(DataStream input, Integer parallelism) { + RebalancePartitioner partitioner = new RebalancePartitioner<>(); + PartitionTransformation partitioned = + new PartitionTransformation<>(input.getTransformation(), partitioner); + if (parallelism != null) { + partitioned.setParallelism(parallelism); + } + return new DataStream<>(input.getExecutionEnvironment(), partitioned); + } } diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/HashBucketAssignerOperator.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/HashBucketAssignerOperator.java index 70fac7a83e93..0c101c6d1e01 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/HashBucketAssignerOperator.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/HashBucketAssignerOperator.java @@ -19,6 +19,7 @@ package org.apache.paimon.flink.sink; import org.apache.paimon.flink.ProcessRecordAttributesUtil; +import org.apache.paimon.flink.utils.RuntimeContextUtils; import org.apache.paimon.index.BucketAssigner; import org.apache.paimon.index.HashBucketAssigner; import org.apache.paimon.index.SimpleHashBucketAssigner; @@ -76,8 +77,8 @@ public void initializeState(StateInitializationContext context) throws Exception StateUtils.getSingleValueFromState( context, "commit_user_state", String.class, initialCommitUser); - int numberTasks = getRuntimeContext().getNumberOfParallelSubtasks(); - int taskId = getRuntimeContext().getIndexOfThisSubtask(); + int numberTasks = RuntimeContextUtils.getNumberOfParallelSubtasks(getRuntimeContext()); + int taskId = RuntimeContextUtils.getIndexOfThisSubtask(getRuntimeContext()); long targetRowNum = table.coreOptions().dynamicBucketTargetRowNum(); this.assigner = overwrite diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/LocalMergeOperator.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/LocalMergeOperator.java index aba891e44100..070262147643 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/LocalMergeOperator.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/LocalMergeOperator.java @@ -20,13 +20,17 @@ import org.apache.paimon.CoreOptions; import org.apache.paimon.KeyValue; +import org.apache.paimon.annotation.VisibleForTesting; import org.apache.paimon.codegen.CodeGenUtils; import org.apache.paimon.codegen.Projection; -import org.apache.paimon.codegen.RecordComparator; +import org.apache.paimon.data.BinaryRow; import org.apache.paimon.data.InternalRow; import org.apache.paimon.memory.HeapMemorySegmentPool; import org.apache.paimon.mergetree.SortBufferWriteBuffer; import org.apache.paimon.mergetree.compact.MergeFunction; +import org.apache.paimon.mergetree.localmerge.HashMapLocalMerger; +import org.apache.paimon.mergetree.localmerge.LocalMerger; +import org.apache.paimon.mergetree.localmerge.SortBufferLocalMerger; import org.apache.paimon.options.MemorySize; import org.apache.paimon.schema.KeyValueFieldsExtractor; import org.apache.paimon.schema.TableSchema; @@ -40,9 +44,15 @@ import org.apache.paimon.utils.UserDefinedSeqComparator; import org.apache.flink.streaming.api.operators.AbstractStreamOperator; +import org.apache.flink.streaming.api.operators.AbstractStreamOperatorFactory; import org.apache.flink.streaming.api.operators.BoundedOneInput; import org.apache.flink.streaming.api.operators.ChainingStrategy; import org.apache.flink.streaming.api.operators.OneInputStreamOperator; +import org.apache.flink.streaming.api.operators.OneInputStreamOperatorFactory; +import org.apache.flink.streaming.api.operators.Output; +import org.apache.flink.streaming.api.operators.StreamOperator; +import org.apache.flink.streaming.api.operators.StreamOperatorFactory; +import org.apache.flink.streaming.api.operators.StreamOperatorParameters; import org.apache.flink.streaming.api.watermark.Watermark; import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; @@ -63,41 +73,36 @@ public class LocalMergeOperator extends AbstractStreamOperator private final boolean ignoreDelete; private transient Projection keyProjection; - private transient RecordComparator keyComparator; - private transient long recordCount; private transient RowKindGenerator rowKindGenerator; - private transient MergeFunction mergeFunction; - private transient SortBufferWriteBuffer buffer; + private transient LocalMerger merger; private transient long currentWatermark; private transient boolean endOfInput; - public LocalMergeOperator(TableSchema schema) { + private LocalMergeOperator( + StreamOperatorParameters parameters, TableSchema schema) { Preconditions.checkArgument( schema.primaryKeys().size() > 0, "LocalMergeOperator currently only support tables with primary keys"); this.schema = schema; this.ignoreDelete = CoreOptions.fromMap(schema.options()).ignoreDelete(); - setChainingStrategy(ChainingStrategy.ALWAYS); + setup(parameters.getContainingTask(), parameters.getStreamConfig(), parameters.getOutput()); } @Override public void open() throws Exception { super.open(); - RowType keyType = PrimaryKeyTableUtils.addKeyNamePrefix(schema.logicalPrimaryKeysType()); + List primaryKeys = schema.primaryKeys(); RowType valueType = schema.logicalRowType(); CoreOptions options = new CoreOptions(schema.options()); - keyProjection = - CodeGenUtils.newProjection(valueType, schema.projection(schema.primaryKeys())); - keyComparator = new KeyComparatorSupplier(keyType).get(); + keyProjection = CodeGenUtils.newProjection(valueType, schema.projection(primaryKeys)); - recordCount = 0; rowKindGenerator = RowKindGenerator.create(schema, options); - mergeFunction = + MergeFunction mergeFunction = PrimaryKeyTableUtils.createMergeFunctionFactory( schema, new KeyValueFieldsExtractor() { @@ -117,26 +122,51 @@ public List valueFields(TableSchema schema) { }) .create(); - buffer = - new SortBufferWriteBuffer( - keyType, - valueType, - UserDefinedSeqComparator.create(valueType, options), - new HeapMemorySegmentPool( - options.localMergeBufferSize(), options.pageSize()), - false, - MemorySize.MAX_VALUE, - options.localSortMaxNumFileHandles(), - options.spillCompressOptions(), - null); - currentWatermark = Long.MIN_VALUE; + boolean canHashMerger = true; + for (DataField field : valueType.getFields()) { + if (primaryKeys.contains(field.name())) { + continue; + } + + if (!BinaryRow.isInFixedLengthPart(field.type())) { + canHashMerger = false; + break; + } + } + HeapMemorySegmentPool pool = + new HeapMemorySegmentPool(options.localMergeBufferSize(), options.pageSize()); + UserDefinedSeqComparator udsComparator = + UserDefinedSeqComparator.create(valueType, options); + if (canHashMerger) { + merger = + new HashMapLocalMerger( + valueType, primaryKeys, pool, mergeFunction, udsComparator); + } else { + RowType keyType = + PrimaryKeyTableUtils.addKeyNamePrefix(schema.logicalPrimaryKeysType()); + SortBufferWriteBuffer sortBuffer = + new SortBufferWriteBuffer( + keyType, + valueType, + udsComparator, + pool, + false, + MemorySize.MAX_VALUE, + options.localSortMaxNumFileHandles(), + options.spillCompressOptions(), + null); + merger = + new SortBufferLocalMerger( + sortBuffer, new KeyComparatorSupplier(keyType).get(), mergeFunction); + } + + currentWatermark = Long.MIN_VALUE; endOfInput = false; } @Override public void processElement(StreamRecord record) throws Exception { - recordCount++; InternalRow row = record.getValue(); RowKind rowKind = RowKindGenerator.getRowKind(rowKindGenerator, row); @@ -147,10 +177,10 @@ public void processElement(StreamRecord record) throws Exception { // row kind must be INSERT when it is divided into key and value row.setRowKind(RowKind.INSERT); - InternalRow key = keyProjection.apply(row); - if (!buffer.put(recordCount, rowKind, key, row)) { + BinaryRow key = keyProjection.apply(row); + if (!merger.put(rowKind, key, row)) { flushBuffer(); - if (!buffer.put(recordCount, rowKind, key, row)) { + if (!merger.put(rowKind, key, row)) { // change row kind back row.setRowKind(rowKind); output.collect(record); @@ -180,28 +210,20 @@ public void endInput() throws Exception { @Override public void close() throws Exception { - if (buffer != null) { - buffer.clear(); + if (merger != null) { + merger.clear(); } super.close(); } private void flushBuffer() throws Exception { - if (buffer.size() == 0) { + if (merger.size() == 0) { return; } - buffer.forEach( - keyComparator, - mergeFunction, - null, - kv -> { - InternalRow row = kv.value(); - row.setRowKind(kv.valueKind()); - output.collect(new StreamRecord<>(row)); - }); - buffer.clear(); + merger.forEach(row -> output.collect(new StreamRecord<>(row))); + merger.clear(); if (currentWatermark != Long.MIN_VALUE) { super.processWatermark(new Watermark(currentWatermark)); @@ -209,4 +231,38 @@ private void flushBuffer() throws Exception { currentWatermark = Long.MIN_VALUE; } } + + @VisibleForTesting + LocalMerger merger() { + return merger; + } + + @VisibleForTesting + void setOutput(Output> output) { + this.output = output; + } + + /** {@link StreamOperatorFactory} of {@link LocalMergeOperator}. */ + public static class Factory extends AbstractStreamOperatorFactory + implements OneInputStreamOperatorFactory { + private final TableSchema schema; + + public Factory(TableSchema schema) { + this.chainingStrategy = ChainingStrategy.ALWAYS; + this.schema = schema; + } + + @Override + @SuppressWarnings("unchecked") + public > T createStreamOperator( + StreamOperatorParameters parameters) { + return (T) new LocalMergeOperator(parameters, schema); + } + + @Override + @SuppressWarnings("rawtypes") + public Class getStreamOperatorClass(ClassLoader classLoader) { + return LocalMergeOperator.class; + } + } } diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/MultiTableCommittableTypeInfo.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/MultiTableCommittableTypeInfo.java index f82f08209867..7da0ae0e2078 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/MultiTableCommittableTypeInfo.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/MultiTableCommittableTypeInfo.java @@ -21,6 +21,7 @@ import org.apache.paimon.table.sink.CommitMessageSerializer; import org.apache.flink.api.common.ExecutionConfig; +import org.apache.flink.api.common.serialization.SerializerConfig; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.common.typeutils.TypeSerializer; @@ -57,7 +58,16 @@ public boolean isKeyType() { return false; } - @Override + /** + * Do not annotate with @override here to maintain compatibility with Flink 1.18-. + */ + public TypeSerializer createSerializer(SerializerConfig config) { + return this.createSerializer((ExecutionConfig) null); + } + + /** + * Do not annotate with @override here to maintain compatibility with Flink 2.0+. + */ public TypeSerializer createSerializer(ExecutionConfig config) { // no copy, so that data from writer is directly going into committer while chaining return new NoneCopyVersionedSerializerTypeSerializerProxy( diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/MultiTableCompactionTaskTypeInfo.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/MultiTableCompactionTaskTypeInfo.java index f27f29f87fe7..0116ff198811 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/MultiTableCompactionTaskTypeInfo.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/MultiTableCompactionTaskTypeInfo.java @@ -23,6 +23,7 @@ import org.apache.paimon.table.sink.MultiTableCompactionTaskSerializer; import org.apache.flink.api.common.ExecutionConfig; +import org.apache.flink.api.common.serialization.SerializerConfig; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.common.typeutils.TypeSerializer; import org.apache.flink.core.io.SimpleVersionedSerializerTypeSerializerProxy; @@ -60,7 +61,17 @@ public boolean isKeyType() { return false; } - @Override + /** + * Do not annotate with @override here to maintain compatibility with Flink 1.18-. + */ + public TypeSerializer createSerializer( + SerializerConfig serializerConfig) { + return this.createSerializer((ExecutionConfig) null); + } + + /** + * Do not annotate with @override here to maintain compatibility with Flink 2.0+. + */ public TypeSerializer createSerializer( ExecutionConfig executionConfig) { return new SimpleVersionedSerializerTypeSerializerProxy< diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/MultiTablesStoreCompactOperator.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/MultiTablesStoreCompactOperator.java index 7cb5d30c2f8e..58f6a3834096 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/MultiTablesStoreCompactOperator.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/MultiTablesStoreCompactOperator.java @@ -22,6 +22,7 @@ import org.apache.paimon.catalog.Catalog; import org.apache.paimon.catalog.Identifier; import org.apache.paimon.data.BinaryRow; +import org.apache.paimon.flink.utils.RuntimeContextUtils; import org.apache.paimon.io.DataFileMeta; import org.apache.paimon.io.DataFileMetaSerializer; import org.apache.paimon.options.Options; @@ -32,6 +33,9 @@ import org.apache.flink.runtime.state.StateInitializationContext; import org.apache.flink.runtime.state.StateSnapshotContext; import org.apache.flink.streaming.api.environment.CheckpointConfig; +import org.apache.flink.streaming.api.operators.StreamOperator; +import org.apache.flink.streaming.api.operators.StreamOperatorFactory; +import org.apache.flink.streaming.api.operators.StreamOperatorParameters; import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; import org.apache.flink.table.data.RowData; @@ -62,6 +66,7 @@ public class MultiTablesStoreCompactOperator private final CheckpointConfig checkpointConfig; private final boolean isStreaming; private final boolean ignorePreviousFiles; + private final boolean fullCompaction; private final String initialCommitUser; private transient StoreSinkWriteState state; @@ -74,19 +79,22 @@ public class MultiTablesStoreCompactOperator protected Map writes; protected String commitUser; - public MultiTablesStoreCompactOperator( + private MultiTablesStoreCompactOperator( + StreamOperatorParameters parameters, Catalog.Loader catalogLoader, String initialCommitUser, CheckpointConfig checkpointConfig, boolean isStreaming, boolean ignorePreviousFiles, + boolean fullCompaction, Options options) { - super(options); + super(parameters, options); this.catalogLoader = catalogLoader; this.initialCommitUser = initialCommitUser; this.checkpointConfig = checkpointConfig; this.isStreaming = isStreaming; this.ignorePreviousFiles = ignorePreviousFiles; + this.fullCompaction = fullCompaction; } @Override @@ -109,8 +117,10 @@ public void initializeState(StateInitializationContext context) throws Exception ChannelComputer.select( partition, bucket, - getRuntimeContext().getNumberOfParallelSubtasks()) - == getRuntimeContext().getIndexOfThisSubtask()); + RuntimeContextUtils.getNumberOfParallelSubtasks( + getRuntimeContext())) + == RuntimeContextUtils.getIndexOfThisSubtask( + getRuntimeContext())); tables = new HashMap<>(); writes = new HashMap<>(); @@ -159,13 +169,14 @@ public void processElement(StreamRecord element) throws Exception { if (write.streamingMode()) { write.notifyNewFiles(snapshotId, partition, bucket, files); + // The full compact is not supported in streaming mode. write.compact(partition, bucket, false); } else { Preconditions.checkArgument( files.isEmpty(), "Batch compact job does not concern what files are compacted. " + "They only need to know what buckets are compacted."); - write.compact(partition, bucket, true); + write.compact(partition, bucket, fullCompaction); } } @@ -309,4 +320,54 @@ private StoreSinkWrite.Provider createWriteProvider( memoryPool, metricGroup); } + + /** {@link StreamOperatorFactory} of {@link MultiTablesStoreCompactOperator}. */ + public static class Factory + extends PrepareCommitOperator.Factory { + private final Catalog.Loader catalogLoader; + private final CheckpointConfig checkpointConfig; + private final boolean isStreaming; + private final boolean ignorePreviousFiles; + private final boolean fullCompaction; + private final String initialCommitUser; + + public Factory( + Catalog.Loader catalogLoader, + String initialCommitUser, + CheckpointConfig checkpointConfig, + boolean isStreaming, + boolean ignorePreviousFiles, + boolean fullCompaction, + Options options) { + super(options); + this.catalogLoader = catalogLoader; + this.initialCommitUser = initialCommitUser; + this.checkpointConfig = checkpointConfig; + this.isStreaming = isStreaming; + this.ignorePreviousFiles = ignorePreviousFiles; + this.fullCompaction = fullCompaction; + } + + @Override + @SuppressWarnings("unchecked") + public > T createStreamOperator( + StreamOperatorParameters parameters) { + return (T) + new MultiTablesStoreCompactOperator( + parameters, + catalogLoader, + initialCommitUser, + checkpointConfig, + isStreaming, + ignorePreviousFiles, + fullCompaction, + options); + } + + @Override + @SuppressWarnings("rawtypes") + public Class getStreamOperatorClass(ClassLoader classLoader) { + return MultiTablesStoreCompactOperator.class; + } + } } diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/PrepareCommitOperator.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/PrepareCommitOperator.java index 3668386ddc2d..8b114d3e492f 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/PrepareCommitOperator.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/PrepareCommitOperator.java @@ -26,10 +26,14 @@ import org.apache.flink.runtime.memory.MemoryManager; import org.apache.flink.streaming.api.graph.StreamConfig; import org.apache.flink.streaming.api.operators.AbstractStreamOperator; +import org.apache.flink.streaming.api.operators.AbstractStreamOperatorFactory; import org.apache.flink.streaming.api.operators.BoundedOneInput; import org.apache.flink.streaming.api.operators.ChainingStrategy; import org.apache.flink.streaming.api.operators.OneInputStreamOperator; +import org.apache.flink.streaming.api.operators.OneInputStreamOperatorFactory; import org.apache.flink.streaming.api.operators.Output; +import org.apache.flink.streaming.api.operators.StreamOperatorFactory; +import org.apache.flink.streaming.api.operators.StreamOperatorParameters; import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; import org.apache.flink.streaming.runtime.tasks.StreamTask; @@ -52,9 +56,9 @@ public abstract class PrepareCommitOperator extends AbstractStreamOpera private final Options options; private boolean endOfInput = false; - public PrepareCommitOperator(Options options) { + public PrepareCommitOperator(StreamOperatorParameters parameters, Options options) { this.options = options; - setChainingStrategy(ChainingStrategy.ALWAYS); + setup(parameters.getContainingTask(), parameters.getStreamConfig(), parameters.getOutput()); } @Override @@ -103,4 +107,15 @@ private void emitCommittables(boolean waitCompaction, long checkpointId) throws protected abstract List prepareCommit(boolean waitCompaction, long checkpointId) throws IOException; + + /** {@link StreamOperatorFactory} of {@link PrepareCommitOperator}. */ + protected abstract static class Factory extends AbstractStreamOperatorFactory + implements OneInputStreamOperatorFactory { + protected final Options options; + + protected Factory(Options options) { + this.options = options; + this.chainingStrategy = ChainingStrategy.ALWAYS; + } + } } diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/RewriteFileIndexSink.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/RewriteFileIndexSink.java index 39dcca03c6aa..d9f863c6b919 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/RewriteFileIndexSink.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/RewriteFileIndexSink.java @@ -45,11 +45,10 @@ import org.apache.paimon.utils.FileStorePathFactory; import org.apache.paimon.utils.Pair; -import org.apache.flink.streaming.api.graph.StreamConfig; -import org.apache.flink.streaming.api.operators.OneInputStreamOperator; -import org.apache.flink.streaming.api.operators.Output; +import org.apache.flink.streaming.api.operators.OneInputStreamOperatorFactory; +import org.apache.flink.streaming.api.operators.StreamOperator; +import org.apache.flink.streaming.api.operators.StreamOperatorParameters; import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; -import org.apache.flink.streaming.runtime.tasks.StreamTask; import javax.annotation.Nullable; @@ -76,34 +75,49 @@ public RewriteFileIndexSink(FileStoreTable table) { } @Override - protected OneInputStreamOperator createWriteOperator( + protected OneInputStreamOperatorFactory createWriteOperatorFactory( StoreSinkWrite.Provider writeProvider, String commitUser) { - return new FileIndexModificationOperator(table.coreOptions().toConfiguration(), table); + return new FileIndexModificationOperatorFactory( + table.coreOptions().toConfiguration(), table); } - /** File index modification operator to rewrite file index. */ - private static class FileIndexModificationOperator - extends PrepareCommitOperator { - - private static final long serialVersionUID = 1L; - + private static class FileIndexModificationOperatorFactory + extends PrepareCommitOperator.Factory { private final FileStoreTable table; - private transient FileIndexProcessor fileIndexProcessor; - private transient List messages; - - public FileIndexModificationOperator(Options options, FileStoreTable table) { + public FileIndexModificationOperatorFactory(Options options, FileStoreTable table) { super(options); this.table = table; } @Override - public void setup( - StreamTask containingTask, - StreamConfig config, - Output> output) { - super.setup(containingTask, config, output); + @SuppressWarnings("unchecked") + public > T createStreamOperator( + StreamOperatorParameters parameters) { + return (T) new FileIndexModificationOperator(parameters, options, table); + } + + @Override + @SuppressWarnings("rawtypes") + public Class getStreamOperatorClass(ClassLoader classLoader) { + return FileIndexModificationOperator.class; + } + } + + /** File index modification operator to rewrite file index. */ + private static class FileIndexModificationOperator + extends PrepareCommitOperator { + + private static final long serialVersionUID = 1L; + + private final transient FileIndexProcessor fileIndexProcessor; + private final transient List messages; + private FileIndexModificationOperator( + StreamOperatorParameters parameters, + Options options, + FileStoreTable table) { + super(parameters, options); this.fileIndexProcessor = new FileIndexProcessor(table); this.messages = new ArrayList<>(); } diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/RowDataStoreWriteOperator.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/RowDataStoreWriteOperator.java index 07fe275543a1..8009bec9677f 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/RowDataStoreWriteOperator.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/RowDataStoreWriteOperator.java @@ -23,6 +23,8 @@ import org.apache.paimon.table.FileStoreTable; import org.apache.paimon.table.sink.SinkRecord; +import org.apache.flink.api.common.functions.Function; +import org.apache.flink.api.common.functions.OpenContext; import org.apache.flink.api.common.functions.RichFunction; import org.apache.flink.api.common.functions.util.FunctionUtils; import org.apache.flink.api.common.state.CheckpointListener; @@ -30,18 +32,20 @@ import org.apache.flink.runtime.state.StateInitializationContext; import org.apache.flink.runtime.state.StateSnapshotContext; import org.apache.flink.streaming.api.functions.sink.SinkFunction; -import org.apache.flink.streaming.api.graph.StreamConfig; import org.apache.flink.streaming.api.operators.InternalTimerService; -import org.apache.flink.streaming.api.operators.Output; +import org.apache.flink.streaming.api.operators.StreamOperator; +import org.apache.flink.streaming.api.operators.StreamOperatorFactory; +import org.apache.flink.streaming.api.operators.StreamOperatorParameters; import org.apache.flink.streaming.api.watermark.Watermark; import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; import org.apache.flink.streaming.runtime.tasks.ProcessingTimeService; -import org.apache.flink.streaming.runtime.tasks.StreamTask; import org.apache.flink.streaming.util.functions.StreamingFunctionUtils; import javax.annotation.Nullable; import java.io.IOException; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; import java.util.List; import java.util.Objects; @@ -57,21 +61,14 @@ public class RowDataStoreWriteOperator extends TableWriteOperator { /** We listen to this ourselves because we don't have an {@link InternalTimerService}. */ private long currentWatermark = Long.MIN_VALUE; - public RowDataStoreWriteOperator( + protected RowDataStoreWriteOperator( + StreamOperatorParameters parameters, FileStoreTable table, @Nullable LogSinkFunction logSinkFunction, StoreSinkWrite.Provider storeSinkWriteProvider, String initialCommitUser) { - super(table, storeSinkWriteProvider, initialCommitUser); + super(parameters, table, storeSinkWriteProvider, initialCommitUser); this.logSinkFunction = logSinkFunction; - } - - @Override - public void setup( - StreamTask containingTask, - StreamConfig config, - Output> output) { - super.setup(containingTask, config, output); if (logSinkFunction != null) { FunctionUtils.setFunctionRuntimeContext(logSinkFunction, getRuntimeContext()); } @@ -97,17 +94,29 @@ public void open() throws Exception { this.sinkContext = new SimpleContext(getProcessingTimeService()); if (logSinkFunction != null) { - // to stay compatible with Flink 1.18- - if (logSinkFunction instanceof RichFunction) { - RichFunction richFunction = (RichFunction) logSinkFunction; - richFunction.open(new Configuration()); - } - + openFunction(logSinkFunction); logCallback = new LogWriteCallback(); logSinkFunction.setWriteCallback(logCallback); } } + private static void openFunction(Function function) throws Exception { + if (function instanceof RichFunction) { + RichFunction richFunction = (RichFunction) function; + + try { + Method method = RichFunction.class.getDeclaredMethod("open", OpenContext.class); + method.invoke(richFunction, new OpenContext() {}); + return; + } catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException e) { + // to stay compatible with Flink 1.18- + } + + Method method = RichFunction.class.getDeclaredMethod("open", Configuration.class); + method.invoke(richFunction, new Configuration()); + } + } + @Override public void processWatermark(Watermark mark) throws Exception { super.processWatermark(mark); @@ -233,4 +242,38 @@ public Long timestamp() { return timestamp; } } + + /** {@link StreamOperatorFactory} of {@link RowDataStoreWriteOperator}. */ + public static class Factory extends TableWriteOperator.Factory { + + @Nullable private final LogSinkFunction logSinkFunction; + + public Factory( + FileStoreTable table, + @Nullable LogSinkFunction logSinkFunction, + StoreSinkWrite.Provider storeSinkWriteProvider, + String initialCommitUser) { + super(table, storeSinkWriteProvider, initialCommitUser); + this.logSinkFunction = logSinkFunction; + } + + @Override + @SuppressWarnings("unchecked") + public > T createStreamOperator( + StreamOperatorParameters parameters) { + return (T) + new RowDataStoreWriteOperator( + parameters, + table, + logSinkFunction, + storeSinkWriteProvider, + initialCommitUser); + } + + @Override + @SuppressWarnings("rawtypes") + public Class getStreamOperatorClass(ClassLoader classLoader) { + return RowDataStoreWriteOperator.class; + } + } } diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/RowDynamicBucketSink.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/RowDynamicBucketSink.java index bf6c70f0aa29..1f7e62d74916 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/RowDynamicBucketSink.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/RowDynamicBucketSink.java @@ -27,7 +27,7 @@ import org.apache.paimon.utils.SerializableFunction; import org.apache.flink.api.java.tuple.Tuple2; -import org.apache.flink.streaming.api.operators.OneInputStreamOperator; +import org.apache.flink.streaming.api.operators.OneInputStreamOperatorFactory; import javax.annotation.Nullable; @@ -60,8 +60,8 @@ protected ChannelComputer> channelComputer2() { } @Override - protected OneInputStreamOperator, Committable> createWriteOperator( - StoreSinkWrite.Provider writeProvider, String commitUser) { - return new DynamicBucketRowWriteOperator(table, writeProvider, commitUser); + protected OneInputStreamOperatorFactory, Committable> + createWriteOperatorFactory(StoreSinkWrite.Provider writeProvider, String commitUser) { + return new DynamicBucketRowWriteOperator.Factory(table, writeProvider, commitUser); } } diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/RowUnawareBucketSink.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/RowUnawareBucketSink.java index 1cd10390c1a0..fea8a382a954 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/RowUnawareBucketSink.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/RowUnawareBucketSink.java @@ -22,7 +22,9 @@ import org.apache.paimon.table.FileStoreTable; import org.apache.flink.runtime.state.StateInitializationContext; -import org.apache.flink.streaming.api.operators.OneInputStreamOperator; +import org.apache.flink.streaming.api.operators.OneInputStreamOperatorFactory; +import org.apache.flink.streaming.api.operators.StreamOperator; +import org.apache.flink.streaming.api.operators.StreamOperatorParameters; import java.util.Map; @@ -38,25 +40,35 @@ public RowUnawareBucketSink( } @Override - protected OneInputStreamOperator createWriteOperator( + protected OneInputStreamOperatorFactory createWriteOperatorFactory( StoreSinkWrite.Provider writeProvider, String commitUser) { - return new RowDataStoreWriteOperator(table, logSinkFunction, writeProvider, commitUser) { - + return new RowDataStoreWriteOperator.Factory( + table, logSinkFunction, writeProvider, commitUser) { @Override - protected StoreSinkWriteState createState( - StateInitializationContext context, - StoreSinkWriteState.StateValueFilter stateFilter) - throws Exception { - // No conflicts will occur in append only unaware bucket writer, so no state is - // needed. - return new NoopStoreSinkWriteState(stateFilter); - } + public StreamOperator createStreamOperator(StreamOperatorParameters parameters) { + return new RowDataStoreWriteOperator( + parameters, table, logSinkFunction, writeProvider, commitUser) { - @Override - protected String getCommitUser(StateInitializationContext context) throws Exception { - // No conflicts will occur in append only unaware bucket writer, so commitUser does - // not matter. - return commitUser; + @Override + protected StoreSinkWriteState createState( + StateInitializationContext context, + StoreSinkWriteState.StateValueFilter stateFilter) + throws Exception { + // No conflicts will occur in append only unaware bucket writer, so no state + // is + // needed. + return new NoopStoreSinkWriteState(stateFilter); + } + + @Override + protected String getCommitUser(StateInitializationContext context) + throws Exception { + // No conflicts will occur in append only unaware bucket writer, so + // commitUser does + // not matter. + return commitUser; + } + }; } }; } diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/StoreCommitter.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/StoreCommitter.java index b3e74a3f6125..4908b99317ba 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/StoreCommitter.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/StoreCommitter.java @@ -20,9 +20,10 @@ import org.apache.paimon.annotation.VisibleForTesting; import org.apache.paimon.flink.metrics.FlinkMetricRegistry; -import org.apache.paimon.flink.sink.partition.PartitionMarkDone; +import org.apache.paimon.flink.sink.partition.PartitionListeners; import org.apache.paimon.io.DataFileMeta; import org.apache.paimon.manifest.ManifestCommittable; +import org.apache.paimon.table.BucketMode; import org.apache.paimon.table.FileStoreTable; import org.apache.paimon.table.sink.CommitMessage; import org.apache.paimon.table.sink.CommitMessageImpl; @@ -43,7 +44,8 @@ public class StoreCommitter implements Committer committables) throws IOException, InterruptedException { commit.commitMultiple(committables, false); calcNumBytesAndRecordsOut(committables); - if (partitionMarkDone != null) { - partitionMarkDone.notifyCommittable(committables); - } + partitionListeners.notifyCommittable(committables); } @Override public int filterAndCommit( List globalCommittables, boolean checkAppendFiles) { int committed = commit.filterAndCommitMultiple(globalCommittables, checkAppendFiles); - if (partitionMarkDone != null) { - partitionMarkDone.notifyCommittable(globalCommittables); - } + partitionListeners.notifyCommittable(globalCommittables); return committed; } @Override public Map> groupByCheckpoint(Collection committables) { - if (partitionMarkDone != null) { - try { - partitionMarkDone.snapshotState(); - } catch (Exception e) { - throw new RuntimeException(e); - } + try { + partitionListeners.snapshotState(); + } catch (Exception e) { + throw new RuntimeException(e); } Map> grouped = new HashMap<>(); @@ -146,6 +139,11 @@ public Map> groupByCheckpoint(Collection co @Override public void close() throws Exception { commit.close(); + partitionListeners.close(); + } + + public boolean allowLogOffsetDuplicate() { + return allowLogOffsetDuplicate; } private void calcNumBytesAndRecordsOut(List committables) { diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/StoreCompactOperator.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/StoreCompactOperator.java index bc7bb350df21..1870a0493c2f 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/StoreCompactOperator.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/StoreCompactOperator.java @@ -20,6 +20,7 @@ import org.apache.paimon.CoreOptions; import org.apache.paimon.data.BinaryRow; +import org.apache.paimon.flink.utils.RuntimeContextUtils; import org.apache.paimon.io.DataFileMeta; import org.apache.paimon.io.DataFileMetaSerializer; import org.apache.paimon.options.Options; @@ -30,6 +31,9 @@ import org.apache.flink.runtime.state.StateInitializationContext; import org.apache.flink.runtime.state.StateSnapshotContext; +import org.apache.flink.streaming.api.operators.StreamOperator; +import org.apache.flink.streaming.api.operators.StreamOperatorFactory; +import org.apache.flink.streaming.api.operators.StreamOperatorParameters; import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; import org.apache.flink.table.data.RowData; @@ -52,23 +56,27 @@ public class StoreCompactOperator extends PrepareCommitOperator> waitToCompact; - public StoreCompactOperator( + private StoreCompactOperator( + StreamOperatorParameters parameters, FileStoreTable table, StoreSinkWrite.Provider storeSinkWriteProvider, - String initialCommitUser) { - super(Options.fromMap(table.options())); + String initialCommitUser, + boolean fullCompaction) { + super(parameters, Options.fromMap(table.options())); Preconditions.checkArgument( !table.coreOptions().writeOnly(), CoreOptions.WRITE_ONLY.key() + " should not be true for StoreCompactOperator."); this.table = table; this.storeSinkWriteProvider = storeSinkWriteProvider; this.initialCommitUser = initialCommitUser; + this.fullCompaction = fullCompaction; } @Override @@ -89,8 +97,10 @@ public void initializeState(StateInitializationContext context) throws Exception ChannelComputer.select( partition, bucket, - getRuntimeContext().getNumberOfParallelSubtasks()) - == getRuntimeContext().getIndexOfThisSubtask()); + RuntimeContextUtils.getNumberOfParallelSubtasks( + getRuntimeContext())) + == RuntimeContextUtils.getIndexOfThisSubtask( + getRuntimeContext())); write = storeSinkWriteProvider.provide( table, @@ -136,10 +146,7 @@ protected List prepareCommit(boolean waitCompaction, long checkpoin try { for (Pair partitionBucket : waitToCompact) { - write.compact( - partitionBucket.getKey(), - partitionBucket.getRight(), - !write.streamingMode()); + write.compact(partitionBucket.getKey(), partitionBucket.getRight(), fullCompaction); } } catch (Exception e) { throw new RuntimeException("Exception happens while executing compaction.", e); @@ -160,4 +167,46 @@ public void close() throws Exception { super.close(); write.close(); } + + /** {@link StreamOperatorFactory} of {@link StoreCompactOperator}. */ + public static class Factory extends PrepareCommitOperator.Factory { + private final FileStoreTable table; + private final StoreSinkWrite.Provider storeSinkWriteProvider; + private final String initialCommitUser; + private final boolean fullCompaction; + + public Factory( + FileStoreTable table, + StoreSinkWrite.Provider storeSinkWriteProvider, + String initialCommitUser, + boolean fullCompaction) { + super(Options.fromMap(table.options())); + Preconditions.checkArgument( + !table.coreOptions().writeOnly(), + CoreOptions.WRITE_ONLY.key() + " should not be true for StoreCompactOperator."); + this.table = table; + this.storeSinkWriteProvider = storeSinkWriteProvider; + this.initialCommitUser = initialCommitUser; + this.fullCompaction = fullCompaction; + } + + @Override + @SuppressWarnings("unchecked") + public > T createStreamOperator( + StreamOperatorParameters parameters) { + return (T) + new StoreCompactOperator( + parameters, + table, + storeSinkWriteProvider, + initialCommitUser, + fullCompaction); + } + + @Override + @SuppressWarnings("rawtypes") + public Class getStreamOperatorClass(ClassLoader classLoader) { + return StoreCompactOperator.class; + } + } } diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/StoreMultiCommitter.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/StoreMultiCommitter.java index aeb3e1857b9b..537a98f97fb0 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/StoreMultiCommitter.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/StoreMultiCommitter.java @@ -92,11 +92,11 @@ public WrappedManifestCommittable combine( WrappedManifestCommittable wrappedManifestCommittable, List committables) { for (MultiTableCommittable committable : committables) { + Identifier identifier = + Identifier.create(committable.getDatabase(), committable.getTable()); ManifestCommittable manifestCommittable = wrappedManifestCommittable.computeCommittableIfAbsent( - Identifier.create(committable.getDatabase(), committable.getTable()), - checkpointId, - watermark); + identifier, checkpointId, watermark); switch (committable.kind()) { case FILE: @@ -106,7 +106,9 @@ public WrappedManifestCommittable combine( case LOG_OFFSET: LogOffsetCommittable offset = (LogOffsetCommittable) committable.wrappedCommittable(); - manifestCommittable.addLogOffset(offset.bucket(), offset.offset()); + StoreCommitter committer = tableCommitters.get(identifier); + manifestCommittable.addLogOffset( + offset.bucket(), offset.offset(), committer.allowLogOffsetDuplicate()); break; } } diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/StoreSinkWriteImpl.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/StoreSinkWriteImpl.java index bdaf7bc327be..ef8042820947 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/StoreSinkWriteImpl.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/StoreSinkWriteImpl.java @@ -65,6 +65,8 @@ public class StoreSinkWriteImpl implements StoreSinkWrite { @Nullable private final MetricGroup metricGroup; + @Nullable private Boolean insertOnly; + public StoreSinkWriteImpl( FileStoreTable table, String commitUser, @@ -154,15 +156,21 @@ private TableWriteImpl newTableWrite(FileStoreTable table) { } if (memoryPoolFactory != null) { - return tableWrite.withMemoryPoolFactory(memoryPoolFactory); + tableWrite.withMemoryPoolFactory(memoryPoolFactory); } else { - return tableWrite.withMemoryPool( + tableWrite.withMemoryPool( memoryPool != null ? memoryPool : new HeapMemorySegmentPool( table.coreOptions().writeBufferSize(), table.coreOptions().pageSize())); } + + if (insertOnly != null) { + tableWrite.withInsertOnly(insertOnly); + } + + return tableWrite; } public void withCompactExecutor(ExecutorService compactExecutor) { @@ -171,6 +179,7 @@ public void withCompactExecutor(ExecutorService compactExecutor) { @Override public void withInsertOnly(boolean insertOnly) { + this.insertOnly = insertOnly; write.withInsertOnly(insertOnly); } diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/TableWriteOperator.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/TableWriteOperator.java index 67b4720e2964..fd876698c094 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/TableWriteOperator.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/TableWriteOperator.java @@ -21,12 +21,15 @@ import org.apache.paimon.annotation.VisibleForTesting; import org.apache.paimon.flink.ProcessRecordAttributesUtil; import org.apache.paimon.flink.sink.StoreSinkWriteState.StateValueFilter; +import org.apache.paimon.flink.utils.RuntimeContextUtils; import org.apache.paimon.options.Options; import org.apache.paimon.table.FileStoreTable; import org.apache.paimon.table.sink.ChannelComputer; import org.apache.flink.runtime.state.StateInitializationContext; import org.apache.flink.runtime.state.StateSnapshotContext; +import org.apache.flink.streaming.api.operators.StreamOperatorFactory; +import org.apache.flink.streaming.api.operators.StreamOperatorParameters; import org.apache.flink.streaming.runtime.streamrecord.RecordAttributes; import java.io.IOException; @@ -44,10 +47,11 @@ public abstract class TableWriteOperator extends PrepareCommitOperator parameters, FileStoreTable table, StoreSinkWrite.Provider storeSinkWriteProvider, String initialCommitUser) { - super(Options.fromMap(table.options())); + super(parameters, Options.fromMap(table.options())); this.table = table; this.storeSinkWriteProvider = storeSinkWriteProvider; this.initialCommitUser = initialCommitUser; @@ -58,14 +62,14 @@ public void initializeState(StateInitializationContext context) throws Exception super.initializeState(context); boolean containLogSystem = containLogSystem(); - int numTasks = getRuntimeContext().getNumberOfParallelSubtasks(); + int numTasks = RuntimeContextUtils.getNumberOfParallelSubtasks(getRuntimeContext()); StateValueFilter stateFilter = (tableName, partition, bucket) -> { int task = containLogSystem ? ChannelComputer.select(bucket, numTasks) : ChannelComputer.select(partition, bucket, numTasks); - return task == getRuntimeContext().getIndexOfThisSubtask(); + return task == RuntimeContextUtils.getIndexOfThisSubtask(getRuntimeContext()); }; state = createState(context, stateFilter); @@ -127,4 +131,22 @@ protected List prepareCommit(boolean waitCompaction, long checkpoin public StoreSinkWrite getWrite() { return write; } + + /** {@link StreamOperatorFactory} of {@link TableWriteOperator}. */ + protected abstract static class Factory + extends PrepareCommitOperator.Factory { + protected final FileStoreTable table; + protected final StoreSinkWrite.Provider storeSinkWriteProvider; + protected final String initialCommitUser; + + protected Factory( + FileStoreTable table, + StoreSinkWrite.Provider storeSinkWriteProvider, + String initialCommitUser) { + super(Options.fromMap(table.options())); + this.table = table; + this.storeSinkWriteProvider = storeSinkWriteProvider; + this.initialCommitUser = initialCommitUser; + } + } } diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/UnawareBucketCompactionSink.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/UnawareBucketCompactionSink.java index da966d5e5156..7a4095f896cc 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/UnawareBucketCompactionSink.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/UnawareBucketCompactionSink.java @@ -24,7 +24,7 @@ import org.apache.flink.streaming.api.datastream.DataStream; import org.apache.flink.streaming.api.datastream.DataStreamSink; -import org.apache.flink.streaming.api.operators.OneInputStreamOperator; +import org.apache.flink.streaming.api.operators.OneInputStreamOperatorFactory; /** Compaction Sink for unaware-bucket table. */ public class UnawareBucketCompactionSink extends FlinkSink { @@ -42,9 +42,9 @@ public static DataStreamSink sink( } @Override - protected OneInputStreamOperator createWriteOperator( - StoreSinkWrite.Provider writeProvider, String commitUser) { - return new AppendOnlySingleTableCompactionWorkerOperator(table, commitUser); + protected OneInputStreamOperatorFactory + createWriteOperatorFactory(StoreSinkWrite.Provider writeProvider, String commitUser) { + return new AppendOnlySingleTableCompactionWorkerOperator.Factory(table, commitUser); } @Override diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/UnawareBucketSink.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/UnawareBucketSink.java index 98b58aa8e96d..7bc40d4c2080 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/UnawareBucketSink.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/UnawareBucketSink.java @@ -74,11 +74,14 @@ public DataStream doWrite( new CommittableTypeInfo(), new CompactionTaskTypeInfo()), new AppendBypassCoordinateOperatorFactory<>(table)) + .startNewChain() .forceNonParallel() .transform( "Compact Worker: " + table.name(), new CommittableTypeInfo(), - new AppendBypassCompactWorkerOperator(table, initialCommitUser)) + new AppendBypassCompactWorkerOperator.Factory( + table, initialCommitUser)) + .startNewChain() .setParallelism(written.getParallelism()); } diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/index/GlobalDynamicBucketSink.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/index/GlobalDynamicBucketSink.java index 26e080c32e83..7022002a43ba 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/index/GlobalDynamicBucketSink.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/index/GlobalDynamicBucketSink.java @@ -39,7 +39,7 @@ import org.apache.flink.api.java.typeutils.TupleTypeInfo; import org.apache.flink.streaming.api.datastream.DataStream; import org.apache.flink.streaming.api.datastream.DataStreamSink; -import org.apache.flink.streaming.api.operators.OneInputStreamOperator; +import org.apache.flink.streaming.api.operators.OneInputStreamOperatorFactory; import javax.annotation.Nullable; @@ -63,9 +63,9 @@ public GlobalDynamicBucketSink( } @Override - protected OneInputStreamOperator, Committable> createWriteOperator( - StoreSinkWrite.Provider writeProvider, String commitUser) { - return new DynamicBucketRowWriteOperator(table, writeProvider, commitUser); + protected OneInputStreamOperatorFactory, Committable> + createWriteOperatorFactory(StoreSinkWrite.Provider writeProvider, String commitUser) { + return new DynamicBucketRowWriteOperator.Factory(table, writeProvider, commitUser); } public DataStreamSink build(DataStream input, @Nullable Integer parallelism) { @@ -89,7 +89,8 @@ public DataStreamSink build(DataStream input, @Nullable Integer new InternalTypeInfo<>( new KeyWithRowSerializer<>( bootstrapSerializer, rowSerializer)), - new IndexBootstrapOperator<>(new IndexBootstrap(table), r -> r)) + new IndexBootstrapOperator.Factory<>( + new IndexBootstrap(table), r -> r)) .setParallelism(input.getParallelism()); // 1. shuffle by key hash diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/index/GlobalIndexAssignerOperator.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/index/GlobalIndexAssignerOperator.java index 7fee3f45f3db..99cce07fdc57 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/index/GlobalIndexAssignerOperator.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/index/GlobalIndexAssignerOperator.java @@ -22,6 +22,7 @@ import org.apache.paimon.crosspartition.KeyPartOrRow; import org.apache.paimon.data.InternalRow; import org.apache.paimon.disk.IOManager; +import org.apache.paimon.flink.utils.RuntimeContextUtils; import org.apache.paimon.table.Table; import org.apache.flink.api.java.tuple.Tuple2; @@ -59,8 +60,8 @@ public void initializeState(StateInitializationContext context) throws Exception assigner.open( computeManagedMemory(this), ioManager, - getRuntimeContext().getNumberOfParallelSubtasks(), - getRuntimeContext().getIndexOfThisSubtask(), + RuntimeContextUtils.getNumberOfParallelSubtasks(getRuntimeContext()), + RuntimeContextUtils.getIndexOfThisSubtask(getRuntimeContext()), this::collect); } diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/index/IndexBootstrapOperator.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/index/IndexBootstrapOperator.java index 501e35dff46c..8136565f98cf 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/index/IndexBootstrapOperator.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/index/IndexBootstrapOperator.java @@ -21,13 +21,19 @@ import org.apache.paimon.crosspartition.IndexBootstrap; import org.apache.paimon.crosspartition.KeyPartOrRow; import org.apache.paimon.data.InternalRow; +import org.apache.paimon.flink.utils.RuntimeContextUtils; import org.apache.paimon.utils.SerializableFunction; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.runtime.state.StateInitializationContext; import org.apache.flink.streaming.api.operators.AbstractStreamOperator; +import org.apache.flink.streaming.api.operators.AbstractStreamOperatorFactory; import org.apache.flink.streaming.api.operators.ChainingStrategy; import org.apache.flink.streaming.api.operators.OneInputStreamOperator; +import org.apache.flink.streaming.api.operators.OneInputStreamOperatorFactory; +import org.apache.flink.streaming.api.operators.StreamOperator; +import org.apache.flink.streaming.api.operators.StreamOperatorFactory; +import org.apache.flink.streaming.api.operators.StreamOperatorParameters; import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; /** Operator for {@link IndexBootstrap}. */ @@ -39,19 +45,21 @@ public class IndexBootstrapOperator extends AbstractStreamOperator converter; - public IndexBootstrapOperator( - IndexBootstrap bootstrap, SerializableFunction converter) { + private IndexBootstrapOperator( + StreamOperatorParameters> parameters, + IndexBootstrap bootstrap, + SerializableFunction converter) { this.bootstrap = bootstrap; this.converter = converter; - setChainingStrategy(ChainingStrategy.ALWAYS); + setup(parameters.getContainingTask(), parameters.getStreamConfig(), parameters.getOutput()); } @Override public void initializeState(StateInitializationContext context) throws Exception { super.initializeState(context); bootstrap.bootstrap( - getRuntimeContext().getNumberOfParallelSubtasks(), - getRuntimeContext().getIndexOfThisSubtask(), + RuntimeContextUtils.getNumberOfParallelSubtasks(getRuntimeContext()), + RuntimeContextUtils.getIndexOfThisSubtask(getRuntimeContext()), this::collect); } @@ -64,4 +72,30 @@ private void collect(InternalRow row) { output.collect( new StreamRecord<>(new Tuple2<>(KeyPartOrRow.KEY_PART, converter.apply(row)))); } + + /** {@link StreamOperatorFactory} of {@link IndexBootstrapOperator}. */ + public static class Factory extends AbstractStreamOperatorFactory> + implements OneInputStreamOperatorFactory> { + private final IndexBootstrap bootstrap; + private final SerializableFunction converter; + + public Factory(IndexBootstrap bootstrap, SerializableFunction converter) { + this.chainingStrategy = ChainingStrategy.ALWAYS; + this.bootstrap = bootstrap; + this.converter = converter; + } + + @Override + @SuppressWarnings("unchecked") + public >> OP createStreamOperator( + StreamOperatorParameters> parameters) { + return (OP) new IndexBootstrapOperator<>(parameters, bootstrap, converter); + } + + @Override + @SuppressWarnings("rawtypes") + public Class getStreamOperatorClass(ClassLoader classLoader) { + return IndexBootstrapOperator.class; + } + } } diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/partition/HmsReporter.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/partition/HmsReporter.java new file mode 100644 index 000000000000..853dc52c20bf --- /dev/null +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/partition/HmsReporter.java @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.flink.sink.partition; + +import org.apache.paimon.Snapshot; +import org.apache.paimon.fs.Path; +import org.apache.paimon.io.DataFileMeta; +import org.apache.paimon.metastore.MetastoreClient; +import org.apache.paimon.table.FileStoreTable; +import org.apache.paimon.table.source.DataSplit; +import org.apache.paimon.table.source.ScanMode; +import org.apache.paimon.table.source.snapshot.SnapshotReader; +import org.apache.paimon.utils.Preconditions; +import org.apache.paimon.utils.SnapshotManager; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.Closeable; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import static org.apache.paimon.catalog.Catalog.HIVE_LAST_UPDATE_TIME_PROP; +import static org.apache.paimon.catalog.Catalog.NUM_FILES_PROP; +import static org.apache.paimon.catalog.Catalog.NUM_ROWS_PROP; +import static org.apache.paimon.catalog.Catalog.TOTAL_SIZE_PROP; +import static org.apache.paimon.utils.PartitionPathUtils.extractPartitionSpecFromPath; + +/** Action to report the table statistic from the latest snapshot to HMS. */ +public class HmsReporter implements Closeable { + + private static final Logger LOG = LoggerFactory.getLogger(HmsReporter.class); + + private final MetastoreClient metastoreClient; + private final SnapshotReader snapshotReader; + private final SnapshotManager snapshotManager; + + public HmsReporter(FileStoreTable table, MetastoreClient client) { + this.metastoreClient = + Preconditions.checkNotNull(client, "the metastore client factory is null"); + this.snapshotReader = table.newSnapshotReader(); + this.snapshotManager = table.snapshotManager(); + } + + public void report(String partition, long modifyTime) throws Exception { + Snapshot snapshot = snapshotManager.latestSnapshot(); + if (snapshot != null) { + LinkedHashMap partitionSpec = + extractPartitionSpecFromPath(new Path(partition)); + List splits = + new ArrayList<>( + snapshotReader + .withMode(ScanMode.ALL) + .withPartitionFilter(partitionSpec) + .withSnapshot(snapshot) + .read() + .dataSplits()); + long rowCount = 0; + long totalSize = 0; + long fileCount = 0; + for (DataSplit split : splits) { + List fileMetas = split.dataFiles(); + rowCount += split.rowCount(); + fileCount += fileMetas.size(); + for (DataFileMeta fileMeta : fileMetas) { + totalSize += fileMeta.fileSize(); + } + } + Map statistic = new HashMap<>(); + statistic.put(NUM_FILES_PROP, String.valueOf(fileCount)); + statistic.put(TOTAL_SIZE_PROP, String.valueOf(totalSize)); + statistic.put(NUM_ROWS_PROP, String.valueOf(rowCount)); + statistic.put(HIVE_LAST_UPDATE_TIME_PROP, String.valueOf(modifyTime / 1000)); + + LOG.info("alter partition {} with statistic {}.", partitionSpec, statistic); + metastoreClient.alterPartition(partitionSpec, statistic, modifyTime, true); + } + } + + @Override + public void close() throws IOException { + try { + metastoreClient.close(); + } catch (Exception e) { + throw new RuntimeException(e); + } + } +} diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/partition/PartitionListener.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/partition/PartitionListener.java new file mode 100644 index 000000000000..65d25fbc0271 --- /dev/null +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/partition/PartitionListener.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.flink.sink.partition; + +import org.apache.paimon.manifest.ManifestCommittable; + +import java.io.Closeable; +import java.util.List; + +/** The partition listener. */ +public interface PartitionListener extends Closeable { + + void notifyCommittable(List committables); + + void snapshotState() throws Exception; +} diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/partition/PartitionListeners.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/partition/PartitionListeners.java new file mode 100644 index 000000000000..dbdf77601480 --- /dev/null +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/partition/PartitionListeners.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.flink.sink.partition; + +import org.apache.paimon.flink.sink.Committer; +import org.apache.paimon.manifest.ManifestCommittable; +import org.apache.paimon.table.FileStoreTable; +import org.apache.paimon.utils.IOUtils; + +import java.io.Closeable; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/** Partition listeners. */ +public class PartitionListeners implements Closeable { + + private final List listeners; + + private PartitionListeners(List listeners) { + this.listeners = listeners; + } + + public void notifyCommittable(List committables) { + for (PartitionListener trigger : listeners) { + trigger.notifyCommittable(committables); + } + } + + public void snapshotState() throws Exception { + for (PartitionListener trigger : listeners) { + trigger.snapshotState(); + } + } + + @Override + public void close() throws IOException { + IOUtils.closeAllQuietly(listeners); + } + + public static PartitionListeners create(Committer.Context context, FileStoreTable table) + throws Exception { + List listeners = new ArrayList<>(); + + ReportHmsListener.create(context.isRestored(), context.stateStore(), table) + .ifPresent(listeners::add); + PartitionMarkDone.create( + context.streamingCheckpointEnabled(), + context.isRestored(), + context.stateStore(), + table) + .ifPresent(listeners::add); + + return new PartitionListeners(listeners); + } +} diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/partition/PartitionMarkDone.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/partition/PartitionMarkDone.java index 39438a101b04..8714e0006e7b 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/partition/PartitionMarkDone.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/partition/PartitionMarkDone.java @@ -33,28 +33,25 @@ import org.apache.flink.api.common.state.OperatorStateStore; -import javax.annotation.Nullable; - -import java.io.Closeable; import java.io.IOException; import java.time.Duration; import java.util.HashSet; import java.util.List; +import java.util.Optional; import java.util.Set; +import static org.apache.paimon.CoreOptions.PARTITION_MARK_DONE_WHEN_END_INPUT; import static org.apache.paimon.flink.FlinkConnectorOptions.PARTITION_IDLE_TIME_TO_DONE; -import static org.apache.paimon.flink.FlinkConnectorOptions.PARTITION_MARK_DONE_WHEN_END_INPUT; /** Mark partition done. */ -public class PartitionMarkDone implements Closeable { +public class PartitionMarkDone implements PartitionListener { private final InternalRowPartitionComputer partitionComputer; private final PartitionMarkDoneTrigger trigger; private final List actions; private final boolean waitCompaction; - @Nullable - public static PartitionMarkDone create( + public static Optional create( boolean isStreaming, boolean isRestored, OperatorStateStore stateStore, @@ -64,14 +61,15 @@ public static PartitionMarkDone create( Options options = coreOptions.toConfiguration(); if (disablePartitionMarkDone(isStreaming, table, options)) { - return null; + return Optional.empty(); } InternalRowPartitionComputer partitionComputer = new InternalRowPartitionComputer( coreOptions.partitionDefaultName(), table.schema().logicalPartitionType(), - table.partitionKeys().toArray(new String[0])); + table.partitionKeys().toArray(new String[0]), + coreOptions.legacyPartitionName()); PartitionMarkDoneTrigger trigger = PartitionMarkDoneTrigger.create(coreOptions, isRestored, stateStore); @@ -86,7 +84,8 @@ public static PartitionMarkDone create( && (coreOptions.deletionVectorsEnabled() || coreOptions.mergeEngine() == MergeEngine.FIRST_ROW); - return new PartitionMarkDone(partitionComputer, trigger, actions, waitCompaction); + return Optional.of( + new PartitionMarkDone(partitionComputer, trigger, actions, waitCompaction)); } private static boolean disablePartitionMarkDone( @@ -115,6 +114,7 @@ public PartitionMarkDone( this.waitCompaction = waitCompaction; } + @Override public void notifyCommittable(List committables) { Set partitions = new HashSet<>(); boolean endInput = false; @@ -152,6 +152,7 @@ public static void markDone(List partitions, List restore() throws Exception { List pendingPartitions = new ArrayList<>(); if (isRestored) { - pendingPartitions.addAll(pendingPartitionsState.get().iterator().next()); + Iterator> state = pendingPartitionsState.get().iterator(); + if (state.hasNext()) { + pendingPartitions.addAll(state.next()); + } } return pendingPartitions; } diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/partition/ReportHmsListener.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/partition/ReportHmsListener.java new file mode 100644 index 000000000000..842dd012e88e --- /dev/null +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/partition/ReportHmsListener.java @@ -0,0 +1,189 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.flink.sink.partition; + +import org.apache.paimon.CoreOptions; +import org.apache.paimon.flink.FlinkConnectorOptions; +import org.apache.paimon.manifest.ManifestCommittable; +import org.apache.paimon.options.Options; +import org.apache.paimon.table.FileStoreTable; +import org.apache.paimon.table.sink.CommitMessage; +import org.apache.paimon.table.sink.CommitMessageImpl; +import org.apache.paimon.utils.InternalRowPartitionComputer; +import org.apache.paimon.utils.PartitionPathUtils; + +import org.apache.flink.api.common.state.ListState; +import org.apache.flink.api.common.state.ListStateDescriptor; +import org.apache.flink.api.common.state.OperatorStateStore; +import org.apache.flink.api.common.typeutils.base.LongSerializer; +import org.apache.flink.api.common.typeutils.base.MapSerializer; +import org.apache.flink.api.common.typeutils.base.StringSerializer; + +import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +/** + * This listener will collect data from the newly touched partition and then decide when to trigger + * a report based on the partition's idle time. + */ +public class ReportHmsListener implements PartitionListener { + + @SuppressWarnings("unchecked") + private static final ListStateDescriptor> PENDING_REPORT_STATE_DESC = + new ListStateDescriptor<>( + "pending-report-hms-partition", + new MapSerializer<>(StringSerializer.INSTANCE, LongSerializer.INSTANCE)); + + private final InternalRowPartitionComputer partitionComputer; + private final HmsReporter hmsReporter; + private final ListState> pendingPartitionsState; + private final Map pendingPartitions; + private final long idleTime; + + private ReportHmsListener( + InternalRowPartitionComputer partitionComputer, + HmsReporter hmsReporter, + OperatorStateStore store, + boolean isRestored, + long idleTime) + throws Exception { + this.partitionComputer = partitionComputer; + this.hmsReporter = hmsReporter; + this.pendingPartitionsState = store.getListState(PENDING_REPORT_STATE_DESC); + this.pendingPartitions = new HashMap<>(); + if (isRestored) { + Iterator> it = pendingPartitionsState.get().iterator(); + if (it.hasNext()) { + Map state = it.next(); + pendingPartitions.putAll(state); + } + } + this.idleTime = idleTime; + } + + public void notifyCommittable(List committables) { + Set partition = new HashSet<>(); + boolean endInput = false; + for (ManifestCommittable committable : committables) { + for (CommitMessage commitMessage : committable.fileCommittables()) { + CommitMessageImpl message = (CommitMessageImpl) commitMessage; + if (!message.newFilesIncrement().isEmpty() + || !message.compactIncrement().isEmpty()) { + partition.add( + PartitionPathUtils.generatePartitionPath( + partitionComputer.generatePartValues(message.partition()))); + } + } + if (committable.identifier() == Long.MAX_VALUE) { + endInput = true; + } + } + // append to map + long current = System.currentTimeMillis(); + partition.forEach(p -> pendingPartitions.put(p, current)); + + try { + Map partitions = reportPartition(endInput); + for (Map.Entry entry : partitions.entrySet()) { + hmsReporter.report(entry.getKey(), entry.getValue()); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + private Map reportPartition(boolean endInput) { + if (endInput) { + return pendingPartitions; + } + + Iterator> iterator = pendingPartitions.entrySet().iterator(); + Map result = new HashMap<>(); + long current = System.currentTimeMillis(); + while (iterator.hasNext()) { + Map.Entry entry = iterator.next(); + if (current - entry.getValue() > idleTime) { + result.put(entry.getKey(), entry.getValue()); + iterator.remove(); + } + } + + return result; + } + + public void snapshotState() throws Exception { + pendingPartitionsState.update(Collections.singletonList(pendingPartitions)); + } + + public static Optional create( + boolean isRestored, OperatorStateStore stateStore, FileStoreTable table) + throws Exception { + + CoreOptions coreOptions = table.coreOptions(); + Options options = coreOptions.toConfiguration(); + if (options.get(FlinkConnectorOptions.PARTITION_IDLE_TIME_TO_REPORT_STATISTIC).toMillis() + <= 0) { + return Optional.empty(); + } + + if ((table.partitionKeys().isEmpty())) { + return Optional.empty(); + } + + if (!coreOptions.partitionedTableInMetastore()) { + return Optional.empty(); + } + + if (table.catalogEnvironment().metastoreClientFactory() == null) { + return Optional.empty(); + } + + InternalRowPartitionComputer partitionComputer = + new InternalRowPartitionComputer( + coreOptions.partitionDefaultName(), + table.schema().logicalPartitionType(), + table.partitionKeys().toArray(new String[0]), + coreOptions.legacyPartitionName()); + + return Optional.of( + new ReportHmsListener( + partitionComputer, + new HmsReporter( + table, + table.catalogEnvironment().metastoreClientFactory().create()), + stateStore, + isRestored, + options.get(FlinkConnectorOptions.PARTITION_IDLE_TIME_TO_REPORT_STATISTIC) + .toMillis())); + } + + @Override + public void close() throws IOException { + if (hmsReporter != null) { + hmsReporter.close(); + } + } +} diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sorter/SortOperator.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sorter/SortOperator.java index 52ad896892a7..b6847125fbc6 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sorter/SortOperator.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sorter/SortOperator.java @@ -23,6 +23,7 @@ import org.apache.paimon.data.BinaryRow; import org.apache.paimon.data.InternalRow; import org.apache.paimon.disk.IOManager; +import org.apache.paimon.flink.utils.RuntimeContextUtils; import org.apache.paimon.options.MemorySize; import org.apache.paimon.sort.BinaryExternalSortBuffer; import org.apache.paimon.types.RowType; @@ -48,6 +49,7 @@ public class SortOperator extends TableStreamOperator private final CompressOptions spillCompression; private final int sinkParallelism; private final MemorySize maxDiskSize; + private final boolean sequenceOrder; private transient BinaryExternalSortBuffer buffer; private transient IOManager ioManager; @@ -60,7 +62,8 @@ public SortOperator( int spillSortMaxNumFiles, CompressOptions spillCompression, int sinkParallelism, - MemorySize maxDiskSize) { + MemorySize maxDiskSize, + boolean sequenceOrder) { this.keyType = keyType; this.rowType = rowType; this.maxMemory = maxMemory; @@ -70,13 +73,15 @@ public SortOperator( this.spillCompression = spillCompression; this.sinkParallelism = sinkParallelism; this.maxDiskSize = maxDiskSize; + this.sequenceOrder = sequenceOrder; } @Override public void open() throws Exception { super.open(); initBuffer(); - if (sinkParallelism != getRuntimeContext().getNumberOfParallelSubtasks()) { + if (sinkParallelism + != RuntimeContextUtils.getNumberOfParallelSubtasks(getRuntimeContext())) { throw new IllegalArgumentException( "Please ensure that the runtime parallelism of the sink matches the initial configuration " + "to avoid potential issues with skewed range partitioning."); @@ -100,7 +105,8 @@ void initBuffer() { pageSize, spillSortMaxNumFiles, spillCompression, - maxDiskSize); + maxDiskSize, + sequenceOrder); } @Override diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sorter/SortUtils.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sorter/SortUtils.java index e163ac364026..b30e14551296 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sorter/SortUtils.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sorter/SortUtils.java @@ -31,6 +31,7 @@ import org.apache.paimon.utils.KeyProjectedRow; import org.apache.paimon.utils.SerializableSupplier; +import org.apache.flink.api.common.functions.OpenContext; import org.apache.flink.api.common.functions.RichMapFunction; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.java.tuple.Tuple2; @@ -119,9 +120,19 @@ public static DataStream sortStreamByKey( .map( new RichMapFunction>() { - @Override + /** + * Do not annotate with @override here to maintain + * compatibility with Flink 1.18-. + */ + public void open(OpenContext openContext) throws Exception { + open(new Configuration()); + } + + /** + * Do not annotate with @override here to maintain + * compatibility with Flink 2.0+. + */ public void open(Configuration parameters) throws Exception { - super.open(parameters); shuffleKeyAbstract.open(); } @@ -163,7 +174,8 @@ public Tuple2 map(RowData value) { options.localSortMaxNumFileHandles(), options.spillCompressOptions(), sinkParallelism, - options.writeBufferSpillDiskSize())) + options.writeBufferSpillDiskSize(), + options.sequenceFieldSortOrderIsAscending())) .setParallelism(sinkParallelism) // remove the key column from every row .map( @@ -171,7 +183,18 @@ public Tuple2 map(RowData value) { private transient KeyProjectedRow keyProjectedRow; - @Override + /** + * Do not annotate with @override here to maintain + * compatibility with Flink 1.18-. + */ + public void open(OpenContext openContext) { + open(new Configuration()); + } + + /** + * Do not annotate with @override here to maintain + * compatibility with Flink 2.0+. + */ public void open(Configuration parameters) { keyProjectedRow = new KeyProjectedRow(valueProjectionMap); } diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/AbstractNonCoordinatedSource.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/AbstractNonCoordinatedSource.java new file mode 100644 index 000000000000..a9a389e837a2 --- /dev/null +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/AbstractNonCoordinatedSource.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.flink.source; + +import org.apache.flink.api.connector.source.Source; +import org.apache.flink.api.connector.source.SplitEnumerator; +import org.apache.flink.api.connector.source.SplitEnumeratorContext; +import org.apache.flink.core.io.SimpleVersionedSerializer; + +/** {@link Source} that does not require coordination between JobManager and TaskManagers. */ +public abstract class AbstractNonCoordinatedSource + implements Source { + @Override + public SplitEnumerator createEnumerator( + SplitEnumeratorContext enumContext) { + return new NoOpEnumerator<>(); + } + + @Override + public SplitEnumerator restoreEnumerator( + SplitEnumeratorContext enumContext, NoOpEnumState checkpoint) { + return new NoOpEnumerator<>(); + } + + @Override + public SimpleVersionedSerializer getSplitSerializer() { + return new SimpleSourceSplitSerializer(); + } + + @Override + public SimpleVersionedSerializer getEnumeratorCheckpointSerializer() { + return new NoOpEnumStateSerializer(); + } +} diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/AbstractNonCoordinatedSourceReader.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/AbstractNonCoordinatedSourceReader.java new file mode 100644 index 000000000000..18c278868ffa --- /dev/null +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/AbstractNonCoordinatedSourceReader.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.flink.source; + +import org.apache.flink.api.connector.source.SourceReader; + +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CompletableFuture; + +/** Abstract {@link SourceReader} for {@link AbstractNonCoordinatedSource}. */ +public abstract class AbstractNonCoordinatedSourceReader + implements SourceReader { + @Override + public void start() {} + + @Override + public List snapshotState(long l) { + return Collections.emptyList(); + } + + @Override + public CompletableFuture isAvailable() { + return CompletableFuture.completedFuture(null); + } + + @Override + public void addSplits(List list) {} + + @Override + public void notifyNoMoreSplits() {} + + @Override + public void close() throws Exception {} +} diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/AppendBypassCoordinateOperator.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/AppendBypassCoordinateOperator.java index 668aa24c145d..b8b0d61e10a9 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/AppendBypassCoordinateOperator.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/AppendBypassCoordinateOperator.java @@ -20,13 +20,14 @@ import org.apache.paimon.append.UnawareAppendCompactionTask; import org.apache.paimon.append.UnawareAppendTableCompactionCoordinator; +import org.apache.paimon.flink.utils.RuntimeContextUtils; import org.apache.paimon.table.FileStoreTable; import org.apache.paimon.utils.ExecutorUtils; import org.apache.flink.api.common.operators.ProcessingTimeService.ProcessingTimeCallback; import org.apache.flink.streaming.api.operators.AbstractStreamOperator; -import org.apache.flink.streaming.api.operators.ChainingStrategy; import org.apache.flink.streaming.api.operators.OneInputStreamOperator; +import org.apache.flink.streaming.api.operators.StreamOperatorParameters; import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; import org.apache.flink.streaming.runtime.tasks.ProcessingTimeService; import org.apache.flink.types.Either; @@ -57,17 +58,19 @@ public class AppendBypassCoordinateOperator private transient LinkedBlockingQueue compactTasks; public AppendBypassCoordinateOperator( - FileStoreTable table, ProcessingTimeService processingTimeService) { + StreamOperatorParameters> parameters, + FileStoreTable table, + ProcessingTimeService processingTimeService) { this.table = table; this.processingTimeService = processingTimeService; - this.chainingStrategy = ChainingStrategy.HEAD; + setup(parameters.getContainingTask(), parameters.getStreamConfig(), parameters.getOutput()); } @Override public void open() throws Exception { super.open(); checkArgument( - getRuntimeContext().getNumberOfParallelSubtasks() == 1, + RuntimeContextUtils.getNumberOfParallelSubtasks(getRuntimeContext()) == 1, "Compaction Coordinator parallelism in paimon MUST be one."); long intervalMs = table.coreOptions().continuousDiscoveryInterval().toMillis(); this.compactTasks = new LinkedBlockingQueue<>(); diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/AppendBypassCoordinateOperatorFactory.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/AppendBypassCoordinateOperatorFactory.java index 7c53e01b47e6..a4c51e5b5a9b 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/AppendBypassCoordinateOperatorFactory.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/AppendBypassCoordinateOperatorFactory.java @@ -45,11 +45,7 @@ T createStreamOperator( StreamOperatorParameters> parameters) { AppendBypassCoordinateOperator operator = - new AppendBypassCoordinateOperator<>(table, processingTimeService); - operator.setup( - parameters.getContainingTask(), - parameters.getStreamConfig(), - parameters.getOutput()); + new AppendBypassCoordinateOperator<>(parameters, table, processingTimeService); return (T) operator; } diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/BaseDataTableSource.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/BaseDataTableSource.java index 9458f7817415..a94d799773bc 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/BaseDataTableSource.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/BaseDataTableSource.java @@ -28,12 +28,15 @@ import org.apache.paimon.flink.log.LogStoreTableFactory; import org.apache.paimon.flink.lookup.FileStoreLookupFunction; import org.apache.paimon.flink.lookup.LookupRuntimeProviderFactory; -import org.apache.paimon.manifest.PartitionEntry; +import org.apache.paimon.options.ConfigOption; import org.apache.paimon.options.Options; import org.apache.paimon.predicate.Predicate; +import org.apache.paimon.schema.TableSchema; import org.apache.paimon.table.DataTable; +import org.apache.paimon.table.FileStoreTable; import org.apache.paimon.table.Table; -import org.apache.paimon.table.source.TableScan; +import org.apache.paimon.table.source.DataSplit; +import org.apache.paimon.table.source.Split; import org.apache.paimon.utils.Projection; import org.apache.flink.api.common.eventtime.WatermarkStrategy; @@ -41,10 +44,6 @@ import org.apache.flink.table.catalog.ObjectIdentifier; import org.apache.flink.table.connector.ChangelogMode; import org.apache.flink.table.connector.source.LookupTableSource; -import org.apache.flink.table.connector.source.LookupTableSource.LookupContext; -import org.apache.flink.table.connector.source.LookupTableSource.LookupRuntimeProvider; -import org.apache.flink.table.connector.source.ScanTableSource.ScanContext; -import org.apache.flink.table.connector.source.ScanTableSource.ScanRuntimeProvider; import org.apache.flink.table.connector.source.SourceProvider; import org.apache.flink.table.connector.source.abilities.SupportsAggregatePushDown; import org.apache.flink.table.connector.source.abilities.SupportsWatermarkPushDown; @@ -56,7 +55,10 @@ import javax.annotation.Nullable; import java.time.Duration; +import java.util.Arrays; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.stream.IntStream; @@ -72,6 +74,7 @@ import static org.apache.paimon.flink.FlinkConnectorOptions.SCAN_WATERMARK_ALIGNMENT_UPDATE_INTERVAL; import static org.apache.paimon.flink.FlinkConnectorOptions.SCAN_WATERMARK_EMIT_STRATEGY; import static org.apache.paimon.flink.FlinkConnectorOptions.SCAN_WATERMARK_IDLE_TIMEOUT; +import static org.apache.paimon.utils.Preconditions.checkNotNull; /** * Table source to create {@link StaticFileStoreSource} or {@link ContinuousFileStoreSource} under @@ -80,13 +83,23 @@ public abstract class BaseDataTableSource extends FlinkTableSource implements LookupTableSource, SupportsWatermarkPushDown, SupportsAggregatePushDown { + private static final List> TIME_TRAVEL_OPTIONS = + Arrays.asList( + CoreOptions.SCAN_TIMESTAMP, + CoreOptions.SCAN_TIMESTAMP_MILLIS, + CoreOptions.SCAN_WATERMARK, + CoreOptions.SCAN_FILE_CREATION_TIME_MILLIS, + CoreOptions.SCAN_SNAPSHOT_ID, + CoreOptions.SCAN_TAG_NAME, + CoreOptions.SCAN_VERSION); + protected final ObjectIdentifier tableIdentifier; protected final boolean streaming; protected final DynamicTableFactory.Context context; @Nullable protected final LogStoreTableFactory logStoreTableFactory; @Nullable protected WatermarkStrategy watermarkStrategy; - protected boolean isBatchCountStar; + @Nullable protected Long countPushed; public BaseDataTableSource( ObjectIdentifier tableIdentifier, @@ -98,7 +111,7 @@ public BaseDataTableSource( @Nullable int[][] projectFields, @Nullable Long limit, @Nullable WatermarkStrategy watermarkStrategy, - boolean isBatchCountStar) { + @Nullable Long countPushed) { super(table, predicate, projectFields, limit); this.tableIdentifier = tableIdentifier; this.streaming = streaming; @@ -108,7 +121,7 @@ public BaseDataTableSource( this.projectFields = projectFields; this.limit = limit; this.watermarkStrategy = watermarkStrategy; - this.isBatchCountStar = isBatchCountStar; + this.countPushed = countPushed; } @Override @@ -147,7 +160,7 @@ public ChangelogMode getChangelogMode() { @Override public ScanRuntimeProvider getScanRuntimeProvider(ScanContext scanContext) { - if (isBatchCountStar) { + if (countPushed != null) { return createCountStarScan(); } @@ -200,10 +213,8 @@ public ScanRuntimeProvider getScanRuntimeProvider(ScanContext scanContext) { } private ScanRuntimeProvider createCountStarScan() { - TableScan scan = table.newReadBuilder().withFilter(predicate).newScan(); - List partitionEntries = scan.listPartitionEntries(); - long rowCount = partitionEntries.stream().mapToLong(PartitionEntry::recordCount).sum(); - NumberSequenceRowSource source = new NumberSequenceRowSource(rowCount, rowCount); + checkNotNull(countPushed); + NumberSequenceRowSource source = new NumberSequenceRowSource(countPushed, countPushed); return new SourceProvider() { @Override public Source createSource() { @@ -231,6 +242,12 @@ public void applyWatermark(WatermarkStrategy watermarkStrategy) { @Override public LookupRuntimeProvider getLookupRuntimeProvider(LookupContext context) { + if (!(table instanceof FileStoreTable)) { + throw new UnsupportedOperationException( + "Currently, lookup dim table only support FileStoreTable but is " + + table.getClass().getName()); + } + if (limit != null) { throw new RuntimeException( "Limit push down should not happen in Lookup source, but it is " + limit); @@ -244,11 +261,34 @@ public LookupRuntimeProvider getLookupRuntimeProvider(LookupContext context) { boolean enableAsync = options.get(LOOKUP_ASYNC); int asyncThreadNumber = options.get(LOOKUP_ASYNC_THREAD_NUMBER); return LookupRuntimeProviderFactory.create( - new FileStoreLookupFunction(table, projection, joinKey, predicate), + getFileStoreLookupFunction( + context, + timeTravelDisabledTable((FileStoreTable) table), + projection, + joinKey), enableAsync, asyncThreadNumber); } + protected FileStoreLookupFunction getFileStoreLookupFunction( + LookupContext context, Table table, int[] projection, int[] joinKey) { + return new FileStoreLookupFunction(table, projection, joinKey, predicate); + } + + private FileStoreTable timeTravelDisabledTable(FileStoreTable table) { + Map newOptions = new HashMap<>(table.options()); + TIME_TRAVEL_OPTIONS.stream().map(ConfigOption::key).forEach(newOptions::remove); + + CoreOptions.StartupMode startupMode = CoreOptions.fromMap(newOptions).startupMode(); + if (startupMode != CoreOptions.StartupMode.COMPACTED_FULL) { + startupMode = CoreOptions.StartupMode.LATEST_FULL; + } + newOptions.put(CoreOptions.SCAN_MODE.key(), startupMode.toString()); + + TableSchema newSchema = table.schema().copy(newOptions); + return table.copy(newSchema); + } + @Override public boolean applyAggregates( List groupingSets, @@ -262,15 +302,6 @@ public boolean applyAggregates( return false; } - if (!table.primaryKeys().isEmpty()) { - return false; - } - - CoreOptions options = ((DataTable) table).coreOptions(); - if (options.deletionVectorsEnabled()) { - return false; - } - if (groupingSets.size() != 1) { return false; } @@ -293,7 +324,22 @@ public boolean applyAggregates( return false; } - isBatchCountStar = true; + List splits = + table.newReadBuilder().dropStats().withFilter(predicate).newScan().plan().splits(); + long countPushed = 0; + for (Split s : splits) { + if (!(s instanceof DataSplit)) { + return false; + } + DataSplit split = (DataSplit) s; + if (!split.mergedRowCountAvailable()) { + return false; + } + + countPushed += split.mergedRowCount(); + } + + this.countPushed = countPushed; return true; } diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/BucketUnawareCompactSource.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/BucketUnawareCompactSource.java index d306c7d8e1e5..7954aad2df0a 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/BucketUnawareCompactSource.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/BucketUnawareCompactSource.java @@ -24,14 +24,16 @@ import org.apache.paimon.predicate.Predicate; import org.apache.paimon.table.FileStoreTable; import org.apache.paimon.table.source.EndOfScanException; -import org.apache.paimon.utils.Preconditions; +import org.apache.flink.api.common.eventtime.WatermarkStrategy; import org.apache.flink.api.connector.source.Boundedness; -import org.apache.flink.configuration.Configuration; +import org.apache.flink.api.connector.source.ReaderOutput; +import org.apache.flink.api.connector.source.SourceReader; +import org.apache.flink.api.connector.source.SourceReaderContext; +import org.apache.flink.core.io.InputStatus; import org.apache.flink.streaming.api.datastream.DataStreamSource; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; -import org.apache.flink.streaming.api.functions.source.RichSourceFunction; -import org.apache.flink.streaming.api.operators.StreamSource; +import org.apache.flink.util.Preconditions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -40,15 +42,16 @@ import java.util.List; /** - * Source Function for unaware-bucket Compaction. + * Source for unaware-bucket Compaction. * - *

    Note: The function is the source function of unaware-bucket compactor coordinator. It will - * read the latest snapshot continuously by compactionCoordinator, and generate new compaction - * tasks. The source function is used in unaware-bucket compaction job (both stand-alone and - * write-combined). Besides, we don't need to save state in this function, it will invoke a full - * scan when starting up, and scan continuously for the following snapshot. + *

    Note: The function is the source of unaware-bucket compactor coordinator. It will read the + * latest snapshot continuously by compactionCoordinator, and generate new compaction tasks. The + * source is used in unaware-bucket compaction job (both stand-alone and write-combined). Besides, + * we don't need to save state in this source, it will invoke a full scan when starting up, and scan + * continuously for the following snapshot. */ -public class BucketUnawareCompactSource extends RichSourceFunction { +public class BucketUnawareCompactSource + extends AbstractNonCoordinatedSource { private static final Logger LOG = LoggerFactory.getLogger(BucketUnawareCompactSource.class); private static final String COMPACTION_COORDINATOR_NAME = "Compaction Coordinator"; @@ -57,9 +60,6 @@ public class BucketUnawareCompactSource extends RichSourceFunction ctx; - private volatile boolean isRunning = true; public BucketUnawareCompactSource( FileStoreTable table, @@ -73,66 +73,63 @@ public BucketUnawareCompactSource( } @Override - public void open(Configuration parameters) throws Exception { - compactionCoordinator = - new UnawareAppendTableCompactionCoordinator(table, streaming, filter); + public Boundedness getBoundedness() { + return streaming ? Boundedness.CONTINUOUS_UNBOUNDED : Boundedness.BOUNDED; + } + + @Override + public SourceReader createReader( + SourceReaderContext readerContext) throws Exception { Preconditions.checkArgument( - this.getRuntimeContext().getNumberOfParallelSubtasks() == 1, + readerContext.currentParallelism() == 1, "Compaction Operator parallelism in paimon MUST be one."); + return new BucketUnawareCompactSourceReader(table, streaming, filter, scanInterval); } - @Override - public void run(SourceContext sourceContext) throws Exception { - this.ctx = sourceContext; - while (isRunning) { + /** BucketUnawareCompactSourceReader. */ + public static class BucketUnawareCompactSourceReader + extends AbstractNonCoordinatedSourceReader { + private final UnawareAppendTableCompactionCoordinator compactionCoordinator; + private final long scanInterval; + + public BucketUnawareCompactSourceReader( + FileStoreTable table, boolean streaming, Predicate filter, long scanInterval) { + this.scanInterval = scanInterval; + compactionCoordinator = + new UnawareAppendTableCompactionCoordinator(table, streaming, filter); + } + + @Override + public InputStatus pollNext(ReaderOutput readerOutput) + throws Exception { boolean isEmpty; - synchronized (ctx.getCheckpointLock()) { - if (!isRunning) { - return; - } - try { - // do scan and plan action, emit append-only compaction tasks. - List tasks = compactionCoordinator.run(); - isEmpty = tasks.isEmpty(); - tasks.forEach(ctx::collect); - } catch (EndOfScanException esf) { - LOG.info("Catching EndOfStreamException, the stream is finished."); - return; - } + try { + // do scan and plan action, emit append-only compaction tasks. + List tasks = compactionCoordinator.run(); + isEmpty = tasks.isEmpty(); + tasks.forEach(readerOutput::collect); + } catch (EndOfScanException esf) { + LOG.info("Catching EndOfStreamException, the stream is finished."); + return InputStatus.END_OF_INPUT; } if (isEmpty) { Thread.sleep(scanInterval); } - } - } - - @Override - public void cancel() { - if (ctx != null) { - synchronized (ctx.getCheckpointLock()) { - isRunning = false; - } - } else { - isRunning = false; + return InputStatus.MORE_AVAILABLE; } } public static DataStreamSource buildSource( StreamExecutionEnvironment env, BucketUnawareCompactSource source, - boolean streaming, String tableIdentifier) { - final StreamSource sourceOperator = - new StreamSource<>(source); return (DataStreamSource) - new DataStreamSource<>( - env, - new CompactionTaskTypeInfo(), - sourceOperator, - false, + env.fromSource( + source, + WatermarkStrategy.noWatermarks(), COMPACTION_COORDINATOR_NAME + " : " + tableIdentifier, - streaming ? Boundedness.CONTINUOUS_UNBOUNDED : Boundedness.BOUNDED) + new CompactionTaskTypeInfo()) .setParallelism(1) .setMaxParallelism(1); } diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/CombinedTableCompactorSourceBuilder.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/CombinedTableCompactorSourceBuilder.java index e5cbbe845ceb..415eddb037df 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/CombinedTableCompactorSourceBuilder.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/CombinedTableCompactorSourceBuilder.java @@ -21,10 +21,10 @@ import org.apache.paimon.append.MultiTableUnawareAppendCompactionTask; import org.apache.paimon.catalog.Catalog; import org.apache.paimon.flink.LogicalTypeConversion; -import org.apache.paimon.flink.source.operator.CombinedAwareBatchSourceFunction; -import org.apache.paimon.flink.source.operator.CombinedAwareStreamingSourceFunction; -import org.apache.paimon.flink.source.operator.CombinedUnawareBatchSourceFunction; -import org.apache.paimon.flink.source.operator.CombinedUnawareStreamingSourceFunction; +import org.apache.paimon.flink.source.operator.CombinedAwareBatchSource; +import org.apache.paimon.flink.source.operator.CombinedAwareStreamingSource; +import org.apache.paimon.flink.source.operator.CombinedUnawareBatchSource; +import org.apache.paimon.flink.source.operator.CombinedUnawareStreamingSource; import org.apache.paimon.table.system.CompactBucketsTable; import org.apache.paimon.types.RowType; import org.apache.paimon.utils.Preconditions; @@ -87,7 +87,7 @@ public DataStream buildAwareBucketTableSource() { Preconditions.checkArgument(env != null, "StreamExecutionEnvironment should not be null."); RowType produceType = CompactBucketsTable.getRowType(); if (isContinuous) { - return CombinedAwareStreamingSourceFunction.buildSource( + return CombinedAwareStreamingSource.buildSource( env, "Combine-MultiBucketTables--StreamingCompactorSource", InternalTypeInfo.of(LogicalTypeConversion.toLogicalType(produceType)), @@ -97,7 +97,7 @@ public DataStream buildAwareBucketTableSource() { databasePattern, monitorInterval); } else { - return CombinedAwareBatchSourceFunction.buildSource( + return CombinedAwareBatchSource.buildSource( env, "Combine-MultiBucketTables-BatchCompactorSource", InternalTypeInfo.of(LogicalTypeConversion.toLogicalType(produceType)), @@ -112,7 +112,7 @@ public DataStream buildAwareBucketTableSource() { public DataStream buildForUnawareBucketsTableSource() { Preconditions.checkArgument(env != null, "StreamExecutionEnvironment should not be null."); if (isContinuous) { - return CombinedUnawareStreamingSourceFunction.buildSource( + return CombinedUnawareStreamingSource.buildSource( env, "Combined-UnawareBucketTables-StreamingCompactorSource", catalogLoader, @@ -121,7 +121,7 @@ public DataStream buildForUnawareBucketsT databasePattern, monitorInterval); } else { - return CombinedUnawareBatchSourceFunction.buildSource( + return CombinedUnawareBatchSource.buildSource( env, "Combined-UnawareBucketTables-BatchCompactorSource", catalogLoader, diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/DataTableSource.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/DataTableSource.java index ad5123205d4a..2b470cb4383a 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/DataTableSource.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/DataTableSource.java @@ -20,6 +20,8 @@ import org.apache.paimon.flink.log.LogStoreTableFactory; import org.apache.paimon.predicate.Predicate; +import org.apache.paimon.stats.ColStats; +import org.apache.paimon.stats.Statistics; import org.apache.paimon.table.Table; import org.apache.flink.api.common.eventtime.WatermarkStrategy; @@ -28,12 +30,17 @@ import org.apache.flink.table.connector.source.abilities.SupportsStatisticReport; import org.apache.flink.table.data.RowData; import org.apache.flink.table.factories.DynamicTableFactory; +import org.apache.flink.table.plan.stats.ColumnStats; import org.apache.flink.table.plan.stats.TableStats; import javax.annotation.Nullable; +import java.util.AbstractMap; import java.util.Collections; import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; import static org.apache.paimon.utils.Preconditions.checkState; @@ -63,7 +70,7 @@ public DataTableSource( null, null, null, - false); + null); } public DataTableSource( @@ -77,7 +84,7 @@ public DataTableSource( @Nullable Long limit, @Nullable WatermarkStrategy watermarkStrategy, @Nullable List dynamicPartitionFilteringFields, - boolean isBatchCountStar) { + @Nullable Long countPushed) { super( tableIdentifier, table, @@ -88,7 +95,7 @@ public DataTableSource( projectFields, limit, watermarkStrategy, - isBatchCountStar); + countPushed); this.dynamicPartitionFilteringFields = dynamicPartitionFilteringFields; } @@ -105,7 +112,7 @@ public DataTableSource copy() { limit, watermarkStrategy, dynamicPartitionFilteringFields, - isBatchCountStar); + countPushed); } @Override @@ -113,7 +120,21 @@ public TableStats reportStatistics() { if (streaming) { return TableStats.UNKNOWN; } - + Optional optionStatistics = table.statistics(); + if (optionStatistics.isPresent()) { + Statistics statistics = optionStatistics.get(); + if (statistics.mergedRecordCount().isPresent()) { + Map flinkColStats = + statistics.colStats().entrySet().stream() + .map( + entry -> + new AbstractMap.SimpleEntry<>( + entry.getKey(), + toFlinkColumnStats(entry.getValue()))) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + return new TableStats(statistics.mergedRecordCount().getAsLong(), flinkColStats); + } + } scanSplitsForInference(); return new TableStats(splitStatistics.totalRowCount()); } @@ -143,4 +164,23 @@ public void applyDynamicFiltering(List candidateFilterFields) { protected List dynamicPartitionFilteringFields() { return dynamicPartitionFilteringFields; } + + private ColumnStats toFlinkColumnStats(ColStats colStats) { + return ColumnStats.Builder.builder() + .setNdv( + colStats.distinctCount().isPresent() + ? colStats.distinctCount().getAsLong() + : null) + .setNullCount( + colStats.nullCount().isPresent() ? colStats.nullCount().getAsLong() : null) + .setAvgLen( + colStats.avgLen().isPresent() + ? (double) colStats.avgLen().getAsLong() + : null) + .setMaxLen( + colStats.maxLen().isPresent() ? (int) colStats.maxLen().getAsLong() : null) + .setMax(colStats.max().isPresent() ? colStats.max().get() : null) + .setMin(colStats.min().isPresent() ? colStats.min().get() : null) + .build(); + } } diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/FileStoreSourceReader.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/FileStoreSourceReader.java index 92adf5e04998..8fc78c868ba5 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/FileStoreSourceReader.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/FileStoreSourceReader.java @@ -25,9 +25,7 @@ import org.apache.flink.api.connector.source.SourceReader; import org.apache.flink.api.connector.source.SourceReaderContext; -import org.apache.flink.connector.base.source.reader.RecordsWithSplitIds; import org.apache.flink.connector.base.source.reader.SingleThreadMultiplexSourceReaderBase; -import org.apache.flink.connector.base.source.reader.synchronization.FutureCompletingBlockingQueue; import org.apache.flink.connector.file.src.reader.BulkFormat.RecordIterator; import org.apache.flink.table.data.RowData; @@ -64,27 +62,6 @@ public FileStoreSourceReader( this.ioManager = ioManager; } - public FileStoreSourceReader( - SourceReaderContext readerContext, - TableRead tableRead, - FileStoreSourceReaderMetrics metrics, - IOManager ioManager, - @Nullable Long limit, - FutureCompletingBlockingQueue>> - elementsQueue) { - super( - elementsQueue, - () -> - new FileStoreSourceSplitReader( - tableRead, RecordLimiter.create(limit), metrics), - (element, output, state) -> - FlinkRecordsWithSplitIds.emitRecord( - readerContext, element, output, state, metrics), - readerContext.getConfiguration(), - readerContext); - this.ioManager = ioManager; - } - @Override public void start() { // we request a split only if we did not get splits during the checkpoint restore diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/FlinkSourceBuilder.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/FlinkSourceBuilder.java index 3131ae0e0afa..b85d5274b241 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/FlinkSourceBuilder.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/FlinkSourceBuilder.java @@ -26,7 +26,7 @@ import org.apache.paimon.flink.log.LogSourceProvider; import org.apache.paimon.flink.sink.FlinkSink; import org.apache.paimon.flink.source.align.AlignedContinuousFileStoreSource; -import org.apache.paimon.flink.source.operator.MonitorFunction; +import org.apache.paimon.flink.source.operator.MonitorSource; import org.apache.paimon.flink.utils.TableScanUtils; import org.apache.paimon.options.Options; import org.apache.paimon.predicate.Predicate; @@ -34,6 +34,7 @@ import org.apache.paimon.table.FileStoreTable; import org.apache.paimon.table.Table; import org.apache.paimon.table.source.ReadBuilder; +import org.apache.paimon.utils.StringUtils; import org.apache.flink.api.common.eventtime.WatermarkStrategy; import org.apache.flink.api.common.functions.MapFunction; @@ -45,7 +46,6 @@ import org.apache.flink.streaming.api.datastream.DataStream; import org.apache.flink.streaming.api.datastream.DataStreamSource; import org.apache.flink.streaming.api.environment.CheckpointConfig; -import org.apache.flink.streaming.api.environment.ExecutionCheckpointingOptions; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.table.data.RowData; import org.apache.flink.table.data.util.DataFormatConverters; @@ -63,6 +63,8 @@ import static org.apache.flink.table.types.utils.TypeConversions.fromLogicalToDataType; import static org.apache.paimon.CoreOptions.StreamingReadMode.FILE; +import static org.apache.paimon.flink.FlinkConnectorOptions.SOURCE_OPERATOR_UID_SUFFIX; +import static org.apache.paimon.flink.FlinkConnectorOptions.generateCustomUid; import static org.apache.paimon.flink.LogicalTypeConversion.toLogicalType; import static org.apache.paimon.utils.Preconditions.checkArgument; import static org.apache.paimon.utils.Preconditions.checkState; @@ -73,6 +75,7 @@ * @since 0.8 */ public class FlinkSourceBuilder { + private static final String SOURCE_NAME = "Source"; private final Table table; private final Options conf; @@ -174,7 +177,7 @@ private ReadBuilder createReadBuilder() { if (limit != null) { readBuilder.withLimit(limit.intValue()); } - return readBuilder; + return readBuilder.dropStats(); } private DataStream buildStaticFileSource() { @@ -210,6 +213,14 @@ private DataStream toDataStream(Source source) { : watermarkStrategy, sourceName, produceTypeInfo()); + + String uidSuffix = table.options().get(SOURCE_OPERATOR_UID_SUFFIX.key()); + if (!StringUtils.isNullOrWhitespaceOnly(uidSuffix)) { + dataStream = + (DataStreamSource) + dataStream.uid(generateCustomUid(SOURCE_NAME, table.name(), uidSuffix)); + } + if (parallelism != null) { dataStream.setParallelism(parallelism); } @@ -247,7 +258,9 @@ public DataStream build() { if (conf.contains(CoreOptions.CONSUMER_ID) && !conf.contains(CoreOptions.CONSUMER_EXPIRATION_TIME)) { throw new IllegalArgumentException( - "consumer.expiration-time should be specified when using consumer-id."); + "You need to configure 'consumer.expiration-time' (ALTER TABLE) and restart your write job for it" + + " to take effect, when you need consumer-id feature. This is to prevent consumers from leaving" + + " too many snapshots that could pose a risk to the file system."); } if (sourceBounded) { @@ -293,7 +306,7 @@ private DataStream buildContinuousStreamOperator() { "Cannot limit streaming source, please use batch execution mode."); } dataStream = - MonitorFunction.buildSource( + MonitorSource.buildSource( env, sourceName, produceTypeInfo(), @@ -317,30 +330,25 @@ private void assertStreamingConfigurationForAlignMode(StreamExecutionEnvironment checkArgument( checkpointConfig.isCheckpointingEnabled(), "The align mode of paimon source is only supported when checkpoint enabled. Please set " - + ExecutionCheckpointingOptions.CHECKPOINTING_INTERVAL.key() - + "larger than 0"); + + "execution.checkpointing.interval larger than 0"); checkArgument( checkpointConfig.getMaxConcurrentCheckpoints() == 1, "The align mode of paimon source supports at most one ongoing checkpoint at the same time. Please set " - + ExecutionCheckpointingOptions.MAX_CONCURRENT_CHECKPOINTS.key() - + " to 1"); + + "execution.checkpointing.max-concurrent-checkpoints to 1"); checkArgument( checkpointConfig.getCheckpointTimeout() > conf.get(FlinkConnectorOptions.SOURCE_CHECKPOINT_ALIGN_TIMEOUT) .toMillis(), "The align mode of paimon source requires that the timeout of checkpoint is greater than the timeout of the source's snapshot alignment. Please increase " - + ExecutionCheckpointingOptions.CHECKPOINTING_TIMEOUT.key() - + " or decrease " + + "execution.checkpointing.timeout or decrease " + FlinkConnectorOptions.SOURCE_CHECKPOINT_ALIGN_TIMEOUT.key()); checkArgument( !env.getCheckpointConfig().isUnalignedCheckpointsEnabled(), "The align mode of paimon source currently does not support unaligned checkpoints. Please set " - + ExecutionCheckpointingOptions.ENABLE_UNALIGNED.key() - + " to false."); + + "execution.checkpointing.unaligned.enabled to false."); checkArgument( env.getCheckpointConfig().getCheckpointingMode() == CheckpointingMode.EXACTLY_ONCE, "The align mode of paimon source currently only supports EXACTLY_ONCE checkpoint mode. Please set " - + ExecutionCheckpointingOptions.CHECKPOINTING_MODE.key() - + " to exactly-once"); + + "execution.checkpointing.mode to exactly-once"); } } diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/FlinkTableSource.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/FlinkTableSource.java index 2be0248f3ce8..12b579589d0f 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/FlinkTableSource.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/FlinkTableSource.java @@ -31,6 +31,7 @@ import org.apache.paimon.table.DataTable; import org.apache.paimon.table.Table; import org.apache.paimon.table.source.Split; +import org.apache.paimon.table.source.TableScan; import org.apache.flink.configuration.Configuration; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; @@ -172,11 +173,7 @@ protected Integer inferSourceParallelism(StreamExecutionEnvironment env) { protected void scanSplitsForInference() { if (splitStatistics == null) { if (table instanceof DataTable) { - List partitionEntries = - table.newReadBuilder() - .withFilter(predicate) - .newScan() - .listPartitionEntries(); + List partitionEntries = newTableScan().listPartitionEntries(); long totalSize = 0; long rowCount = 0; for (PartitionEntry entry : partitionEntries) { @@ -187,8 +184,7 @@ protected void scanSplitsForInference() { splitStatistics = new SplitStatistics((int) (totalSize / splitTargetSize + 1), rowCount); } else { - List splits = - table.newReadBuilder().withFilter(predicate).newScan().plan().splits(); + List splits = newTableScan().plan().splits(); splitStatistics = new SplitStatistics( splits.size(), splits.stream().mapToLong(Split::rowCount).sum()); @@ -196,6 +192,10 @@ protected void scanSplitsForInference() { } } + private TableScan newTableScan() { + return table.newReadBuilder().dropStats().withFilter(predicate).newScan(); + } + /** Split statistics for inferring row count and parallelism size. */ protected static class SplitStatistics { diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/NoOpEnumState.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/NoOpEnumState.java new file mode 100644 index 000000000000..f07317c155aa --- /dev/null +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/NoOpEnumState.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.flink.source; + +/** The enumerator state class for {@link NoOpEnumerator}. */ +public class NoOpEnumState {} diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/NoOpEnumStateSerializer.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/NoOpEnumStateSerializer.java new file mode 100644 index 000000000000..89c0ad6ac1f1 --- /dev/null +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/NoOpEnumStateSerializer.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.flink.source; + +import org.apache.flink.core.io.SimpleVersionedSerializer; + +import java.io.IOException; + +/** {@link SimpleVersionedSerializer} for {@link NoOpEnumState}. */ +public class NoOpEnumStateSerializer implements SimpleVersionedSerializer { + @Override + public int getVersion() { + return 0; + } + + @Override + public byte[] serialize(NoOpEnumState obj) throws IOException { + return new byte[0]; + } + + @Override + public NoOpEnumState deserialize(int version, byte[] serialized) throws IOException { + return new NoOpEnumState(); + } +} diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/NoOpEnumerator.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/NoOpEnumerator.java new file mode 100644 index 000000000000..f29c6d6db76d --- /dev/null +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/NoOpEnumerator.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.flink.source; + +import org.apache.flink.api.connector.source.SourceSplit; +import org.apache.flink.api.connector.source.SplitEnumerator; + +import javax.annotation.Nullable; + +import java.io.IOException; +import java.util.List; + +/** + * A {@link SplitEnumerator} that provides no functionality. It is basically used for sources that + * does not require a coordinator. + */ +public class NoOpEnumerator + implements SplitEnumerator { + @Override + public void start() {} + + @Override + public void handleSplitRequest(int subtaskId, @Nullable String requesterHostname) {} + + @Override + public void addSplitsBack(List splits, int subtaskId) {} + + @Override + public void addReader(int subtaskId) {} + + @Override + public NoOpEnumState snapshotState(long checkpointId) throws Exception { + return new NoOpEnumState(); + } + + @Override + public void close() throws IOException {} +} diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/SimpleSourceSplit.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/SimpleSourceSplit.java new file mode 100644 index 000000000000..2db0868f8e34 --- /dev/null +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/SimpleSourceSplit.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.flink.source; + +import org.apache.flink.api.connector.source.SourceSplit; + +import java.util.UUID; + +/** A {@link SourceSplit} that provides basic information through splitId. */ +public class SimpleSourceSplit implements SourceSplit { + private final String splitId; + private final String value; + + public SimpleSourceSplit() { + this(""); + } + + public SimpleSourceSplit(String value) { + this(UUID.randomUUID().toString(), value); + } + + public SimpleSourceSplit(String splitId, String value) { + this.splitId = splitId; + this.value = value; + } + + @Override + public String splitId() { + return splitId; + } + + public String value() { + return value; + } +} diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/SimpleSourceSplitSerializer.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/SimpleSourceSplitSerializer.java new file mode 100644 index 000000000000..3387afed1c2a --- /dev/null +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/SimpleSourceSplitSerializer.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.flink.source; + +import org.apache.flink.core.io.SimpleVersionedSerializer; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; + +/** {@link SimpleVersionedSerializer} for {@link SimpleSourceSplit}. */ +public class SimpleSourceSplitSerializer implements SimpleVersionedSerializer { + + @Override + public int getVersion() { + return 0; + } + + @Override + public byte[] serialize(SimpleSourceSplit split) throws IOException { + if (split.splitId() == null) { + return new byte[0]; + } + + try (final ByteArrayOutputStream baos = new ByteArrayOutputStream(); + final DataOutputStream out = new DataOutputStream(baos)) { + writeString(out, split.splitId()); + writeString(out, split.value()); + return baos.toByteArray(); + } + } + + @Override + public SimpleSourceSplit deserialize(int version, byte[] serialized) throws IOException { + if (serialized.length == 0) { + return new SimpleSourceSplit(); + } + + try (final ByteArrayInputStream bais = new ByteArrayInputStream(serialized); + final DataInputStream in = new DataInputStream(bais)) { + String splitId = readString(in); + String value = readString(in); + return new SimpleSourceSplit(splitId, value); + } + } + + private void writeString(DataOutputStream out, String str) throws IOException { + byte[] bytes = str.getBytes(); + out.writeInt(bytes.length); + out.write(str.getBytes()); + } + + private String readString(DataInputStream in) throws IOException { + int length = in.readInt(); + byte[] bytes = new byte[length]; + in.readFully(bytes); + return new String(bytes); + } +} diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/SplitListState.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/SplitListState.java new file mode 100644 index 000000000000..0049bdf284e3 --- /dev/null +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/SplitListState.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.flink.source; + +import org.apache.paimon.utils.Preconditions; + +import org.apache.flink.api.common.state.ListState; + +import java.util.ArrayList; +import java.util.List; +import java.util.function.Function; +import java.util.stream.Collectors; + +/** + * Utility class to provide {@link ListState}-like experience for sources that use {@link + * SimpleSourceSplit}. + */ +public class SplitListState implements ListState { + private final String splitPrefix; + private final List values; + private final Function serializer; + private final Function deserializer; + + public SplitListState( + String identifier, Function serializer, Function deserializer) { + Preconditions.checkArgument( + !Character.isDigit(identifier.charAt(0)), + String.format("Identifier %s should not start with digits.", identifier)); + this.splitPrefix = identifier.length() + identifier; + this.serializer = serializer; + this.deserializer = deserializer; + this.values = new ArrayList<>(); + } + + @Override + public void add(T value) { + values.add(value); + } + + @Override + public List get() { + return new ArrayList<>(values); + } + + @Override + public void update(List values) { + this.values.clear(); + this.values.addAll(values); + } + + @Override + public void addAll(List values) throws Exception { + this.values.addAll(values); + } + + @Override + public void clear() { + values.clear(); + } + + public List snapshotState() { + return values.stream() + .map(x -> new SimpleSourceSplit(splitPrefix + serializer.apply(x))) + .collect(Collectors.toList()); + } + + public void restoreState(List splits) { + values.clear(); + splits.stream() + .map(SimpleSourceSplit::value) + .filter(x -> x.startsWith(splitPrefix)) + .map(x -> x.substring(splitPrefix.length())) + .map(this.deserializer) + .forEach(values::add); + } +} diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/StaticFileStoreSplitEnumerator.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/StaticFileStoreSplitEnumerator.java index abd12aa37736..7bd8ff990f86 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/StaticFileStoreSplitEnumerator.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/StaticFileStoreSplitEnumerator.java @@ -21,12 +21,15 @@ import org.apache.paimon.Snapshot; import org.apache.paimon.annotation.VisibleForTesting; import org.apache.paimon.flink.source.assigners.DynamicPartitionPruningAssigner; +import org.apache.paimon.flink.source.assigners.PreAssignSplitAssigner; import org.apache.paimon.flink.source.assigners.SplitAssigner; import org.apache.flink.api.connector.source.SourceEvent; import org.apache.flink.api.connector.source.SplitEnumerator; import org.apache.flink.api.connector.source.SplitEnumeratorContext; import org.apache.flink.api.connector.source.SplitsAssignment; +import org.apache.flink.api.connector.source.SupportsHandleExecutionAttemptSourceEvent; +import org.apache.flink.table.connector.source.DynamicFilteringEvent; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -39,7 +42,8 @@ /** A {@link SplitEnumerator} implementation for {@link StaticFileStoreSource} input. */ public class StaticFileStoreSplitEnumerator - implements SplitEnumerator { + implements SplitEnumerator, + SupportsHandleExecutionAttemptSourceEvent { private static final Logger LOG = LoggerFactory.getLogger(StaticFileStoreSplitEnumerator.class); @@ -116,6 +120,17 @@ public Snapshot snapshot() { return snapshot; } + @Override + public void handleSourceEvent(int subtaskId, int attemptNumber, SourceEvent sourceEvent) { + // Only recognize events that don't care attemptNumber. + handleSourceEvent(subtaskId, sourceEvent); + } + + /** + * When to support a new kind of event, pay attention that whether the new event can be sent + * multiple times from different attempts of one subtask. If so, it should be handled via method + * {@link #handleSourceEvent(int, int, SourceEvent)} + */ @Override public void handleSourceEvent(int subtaskId, SourceEvent sourceEvent) { if (sourceEvent instanceof ReaderConsumeProgressEvent) { @@ -128,13 +143,23 @@ public void handleSourceEvent(int subtaskId, SourceEvent sourceEvent) { checkNotNull( dynamicPartitionFilteringInfo, "Cannot apply dynamic filtering because dynamicPartitionFilteringInfo hasn't been set."); - this.splitAssigner = - DynamicPartitionPruningAssigner.createDynamicPartitionPruningAssignerIfNeeded( - subtaskId, - splitAssigner, - dynamicPartitionFilteringInfo.getPartitionRowProjection(), - sourceEvent, - LOG); + + if (splitAssigner instanceof PreAssignSplitAssigner) { + this.splitAssigner = + ((PreAssignSplitAssigner) splitAssigner) + .ofDynamicPartitionPruning( + dynamicPartitionFilteringInfo.getPartitionRowProjection(), + ((DynamicFilteringEvent) sourceEvent).getData()); + } else { + this.splitAssigner = + DynamicPartitionPruningAssigner + .createDynamicPartitionPruningAssignerIfNeeded( + subtaskId, + splitAssigner, + dynamicPartitionFilteringInfo.getPartitionRowProjection(), + sourceEvent, + LOG); + } } else { LOG.error("Received unrecognized event: {}", sourceEvent); } diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/align/AlignedContinuousFileStoreSource.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/align/AlignedContinuousFileStoreSource.java index d6b7060763ac..705e1d9a7a4c 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/align/AlignedContinuousFileStoreSource.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/align/AlignedContinuousFileStoreSource.java @@ -73,7 +73,7 @@ public SourceReader createReader(SourceReaderCont limit, new FutureCompletingBlockingQueue<>( context.getConfiguration() - .getInteger(SourceReaderOptions.ELEMENT_QUEUE_CAPACITY))); + .get(SourceReaderOptions.ELEMENT_QUEUE_CAPACITY))); } @Override diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/align/AlignedSourceReader.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/align/AlignedSourceReader.java index 1f0bbca314b6..a8ffe3de561f 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/align/AlignedSourceReader.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/align/AlignedSourceReader.java @@ -58,7 +58,7 @@ public AlignedSourceReader( @Nullable Long limit, FutureCompletingBlockingQueue>> elementsQueue) { - super(readerContext, tableRead, metrics, ioManager, limit, elementsQueue); + super(readerContext, tableRead, metrics, ioManager, limit); this.elementsQueue = elementsQueue; this.nextCheckpointId = null; } diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/assigners/PreAssignSplitAssigner.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/assigners/PreAssignSplitAssigner.java index 400a2e5c54eb..fbb31bd1080a 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/assigners/PreAssignSplitAssigner.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/assigners/PreAssignSplitAssigner.java @@ -18,10 +18,15 @@ package org.apache.paimon.flink.source.assigners; +import org.apache.paimon.codegen.Projection; +import org.apache.paimon.data.BinaryRow; +import org.apache.paimon.flink.FlinkRowData; import org.apache.paimon.flink.source.FileStoreSourceSplit; +import org.apache.paimon.table.source.DataSplit; import org.apache.paimon.utils.BinPacking; import org.apache.flink.api.connector.source.SplitEnumeratorContext; +import org.apache.flink.table.connector.source.DynamicFilteringData; import javax.annotation.Nullable; @@ -35,29 +40,53 @@ import java.util.Optional; import java.util.Queue; import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; import static org.apache.paimon.flink.utils.TableScanUtils.getSnapshotId; /** - * Pre-calculate which splits each task should process according to the weight, and then distribute - * the splits fairly. + * Pre-calculate which splits each task should process according to the weight or given + * DynamicFilteringData, and then distribute the splits fairly. */ public class PreAssignSplitAssigner implements SplitAssigner { /** Default batch splits size to avoid exceed `akka.framesize`. */ private final int splitBatchSize; + private final int parallelism; + private final Map> pendingSplitAssignment; private final AtomicInteger numberOfPendingSplits; + private final Collection splits; public PreAssignSplitAssigner( int splitBatchSize, SplitEnumeratorContext context, Collection splits) { + this(splitBatchSize, context.currentParallelism(), splits); + } + + public PreAssignSplitAssigner( + int splitBatchSize, + int parallelism, + Collection splits, + Projection partitionRowProjection, + DynamicFilteringData dynamicFilteringData) { + this( + splitBatchSize, + parallelism, + splits.stream() + .filter(s -> filter(partitionRowProjection, dynamicFilteringData, s)) + .collect(Collectors.toList())); + } + + public PreAssignSplitAssigner( + int splitBatchSize, int parallelism, Collection splits) { this.splitBatchSize = splitBatchSize; - this.pendingSplitAssignment = - createBatchFairSplitAssignment(splits, context.currentParallelism()); + this.parallelism = parallelism; + this.splits = splits; + this.pendingSplitAssignment = createBatchFairSplitAssignment(splits, parallelism); this.numberOfPendingSplits = new AtomicInteger(splits.size()); } @@ -127,4 +156,20 @@ public Optional getNextSnapshotId(int subtask) { public int numberOfRemainingSplits() { return numberOfPendingSplits.get(); } + + public SplitAssigner ofDynamicPartitionPruning( + Projection partitionRowProjection, DynamicFilteringData dynamicFilteringData) { + return new PreAssignSplitAssigner( + splitBatchSize, parallelism, splits, partitionRowProjection, dynamicFilteringData); + } + + private static boolean filter( + Projection partitionRowProjection, + DynamicFilteringData dynamicFilteringData, + FileStoreSourceSplit sourceSplit) { + DataSplit dataSplit = (DataSplit) sourceSplit.split(); + BinaryRow partition = dataSplit.partition(); + FlinkRowData projected = new FlinkRowData(partitionRowProjection.apply(partition)); + return dynamicFilteringData.contains(projected); + } } diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/metrics/FileStoreSourceReaderMetrics.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/metrics/FileStoreSourceReaderMetrics.java index 2e1e94777949..a270e0eceecd 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/metrics/FileStoreSourceReaderMetrics.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/metrics/FileStoreSourceReaderMetrics.java @@ -29,6 +29,7 @@ public class FileStoreSourceReaderMetrics { private long lastSplitUpdateTime = UNDEFINED; public static final long UNDEFINED = -1L; + public static final long ACTIVE = Long.MAX_VALUE; public FileStoreSourceReaderMetrics(MetricGroup sourceReaderMetricGroup) { sourceReaderMetricGroup.gauge( diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/operator/CombinedAwareBatchSourceFunction.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/operator/CombinedAwareBatchSource.java similarity index 66% rename from paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/operator/CombinedAwareBatchSourceFunction.java rename to paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/operator/CombinedAwareBatchSource.java index cee6081aa29f..c3a1258bb176 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/operator/CombinedAwareBatchSourceFunction.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/operator/CombinedAwareBatchSource.java @@ -21,20 +21,23 @@ import org.apache.paimon.catalog.Catalog; import org.apache.paimon.flink.compact.MultiAwareBucketTableScan; import org.apache.paimon.flink.compact.MultiTableScanBase; +import org.apache.paimon.flink.source.AbstractNonCoordinatedSourceReader; +import org.apache.paimon.flink.source.SimpleSourceSplit; import org.apache.paimon.flink.utils.JavaTypeInfo; import org.apache.paimon.table.source.DataSplit; import org.apache.paimon.table.source.Split; +import org.apache.flink.api.common.eventtime.WatermarkStrategy; import org.apache.flink.api.common.typeinfo.BasicTypeInfo; import org.apache.flink.api.common.typeinfo.TypeInformation; -import org.apache.flink.api.connector.source.Boundedness; +import org.apache.flink.api.connector.source.ReaderOutput; +import org.apache.flink.api.connector.source.SourceReader; +import org.apache.flink.api.connector.source.SourceReaderContext; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.typeutils.TupleTypeInfo; -import org.apache.flink.configuration.Configuration; +import org.apache.flink.core.io.InputStatus; import org.apache.flink.streaming.api.datastream.DataStream; -import org.apache.flink.streaming.api.datastream.DataStreamSource; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; -import org.apache.flink.streaming.api.operators.StreamSource; import org.apache.flink.table.data.RowData; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -46,15 +49,11 @@ import static org.apache.paimon.flink.compact.MultiTableScanBase.ScanResult.IS_EMPTY; /** It is responsible for monitoring compactor source of aware bucket table in batch mode. */ -public class CombinedAwareBatchSourceFunction - extends CombinedCompactorSourceFunction> { +public class CombinedAwareBatchSource extends CombinedCompactorSource> { - private static final Logger LOGGER = - LoggerFactory.getLogger(CombinedAwareBatchSourceFunction.class); + private static final Logger LOGGER = LoggerFactory.getLogger(CombinedAwareBatchSource.class); - private MultiTableScanBase> tableScan; - - public CombinedAwareBatchSourceFunction( + public CombinedAwareBatchSource( Catalog.Loader catalogLoader, Pattern includingPattern, Pattern excludingPattern, @@ -63,24 +62,32 @@ public CombinedAwareBatchSourceFunction( } @Override - public void open(Configuration parameters) throws Exception { - super.open(parameters); - tableScan = - new MultiAwareBucketTableScan( - catalogLoader, - includingPattern, - excludingPattern, - databasePattern, - isStreaming, - isRunning); + public SourceReader, SimpleSourceSplit> createReader( + SourceReaderContext sourceReaderContext) throws Exception { + return new Reader(); } - @Override - void scanTable() throws Exception { - if (isRunning.get()) { - MultiTableScanBase.ScanResult scanResult = tableScan.scanTable(ctx); + private class Reader extends AbstractNonCoordinatedSourceReader> { + private MultiTableScanBase> tableScan; + + @Override + public void start() { + super.start(); + tableScan = + new MultiAwareBucketTableScan( + catalogLoader, + includingPattern, + excludingPattern, + databasePattern, + isStreaming); + } + + @Override + public InputStatus pollNext(ReaderOutput> readerOutput) + throws Exception { + MultiTableScanBase.ScanResult scanResult = tableScan.scanTable(readerOutput); if (scanResult == FINISHED) { - return; + return InputStatus.END_OF_INPUT; } if (scanResult == IS_EMPTY) { // Currently, in the combined mode, there are two scan tasks for the table of two @@ -89,6 +96,15 @@ void scanTable() throws Exception { // should not be thrown exception here. LOGGER.info("No file were collected for the table of aware-bucket"); } + return InputStatus.END_OF_INPUT; + } + + @Override + public void close() throws Exception { + super.close(); + if (tableScan != null) { + tableScan.close(); + } } } @@ -101,15 +117,14 @@ public static DataStream buildSource( Pattern excludingPattern, Pattern databasePattern, Duration partitionIdleTime) { - CombinedAwareBatchSourceFunction function = - new CombinedAwareBatchSourceFunction( + CombinedAwareBatchSource source = + new CombinedAwareBatchSource( catalogLoader, includingPattern, excludingPattern, databasePattern); - StreamSource, ?> sourceOperator = new StreamSource<>(function); TupleTypeInfo> tupleTypeInfo = new TupleTypeInfo<>( new JavaTypeInfo<>(Split.class), BasicTypeInfo.STRING_TYPE_INFO); - return new DataStreamSource<>( - env, tupleTypeInfo, sourceOperator, false, name, Boundedness.BOUNDED) + + return env.fromSource(source, WatermarkStrategy.noWatermarks(), name, tupleTypeInfo) .forceNonParallel() .partitionCustom( (key, numPartitions) -> key % numPartitions, @@ -119,12 +134,4 @@ public static DataStream buildSource( typeInfo, new MultiTablesReadOperator(catalogLoader, false, partitionIdleTime)); } - - @Override - public void close() throws Exception { - super.close(); - if (tableScan != null) { - tableScan.close(); - } - } } diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/operator/CombinedAwareStreamingSourceFunction.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/operator/CombinedAwareStreamingSource.java similarity index 65% rename from paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/operator/CombinedAwareStreamingSourceFunction.java rename to paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/operator/CombinedAwareStreamingSource.java index bff690ea30c2..9bd4a84f571c 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/operator/CombinedAwareStreamingSourceFunction.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/operator/CombinedAwareStreamingSource.java @@ -21,20 +21,23 @@ import org.apache.paimon.catalog.Catalog; import org.apache.paimon.flink.compact.MultiAwareBucketTableScan; import org.apache.paimon.flink.compact.MultiTableScanBase; +import org.apache.paimon.flink.source.AbstractNonCoordinatedSourceReader; +import org.apache.paimon.flink.source.SimpleSourceSplit; import org.apache.paimon.flink.utils.JavaTypeInfo; import org.apache.paimon.table.source.DataSplit; import org.apache.paimon.table.source.Split; +import org.apache.flink.api.common.eventtime.WatermarkStrategy; import org.apache.flink.api.common.typeinfo.BasicTypeInfo; import org.apache.flink.api.common.typeinfo.TypeInformation; -import org.apache.flink.api.connector.source.Boundedness; +import org.apache.flink.api.connector.source.ReaderOutput; +import org.apache.flink.api.connector.source.SourceReader; +import org.apache.flink.api.connector.source.SourceReaderContext; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.typeutils.TupleTypeInfo; -import org.apache.flink.configuration.Configuration; +import org.apache.flink.core.io.InputStatus; import org.apache.flink.streaming.api.datastream.DataStream; -import org.apache.flink.streaming.api.datastream.DataStreamSource; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; -import org.apache.flink.streaming.api.operators.StreamSource; import org.apache.flink.table.data.RowData; import java.util.regex.Pattern; @@ -43,13 +46,11 @@ import static org.apache.paimon.flink.compact.MultiTableScanBase.ScanResult.IS_EMPTY; /** It is responsible for monitoring compactor source of multi bucket table in stream mode. */ -public class CombinedAwareStreamingSourceFunction - extends CombinedCompactorSourceFunction> { +public class CombinedAwareStreamingSource extends CombinedCompactorSource> { private final long monitorInterval; - private transient MultiTableScanBase> tableScan; - public CombinedAwareStreamingSourceFunction( + public CombinedAwareStreamingSource( Catalog.Loader catalogLoader, Pattern includingPattern, Pattern excludingPattern, @@ -60,29 +61,45 @@ public CombinedAwareStreamingSourceFunction( } @Override - public void open(Configuration parameters) throws Exception { - super.open(parameters); - tableScan = - new MultiAwareBucketTableScan( - catalogLoader, - includingPattern, - excludingPattern, - databasePattern, - isStreaming, - isRunning); + public SourceReader, SimpleSourceSplit> createReader( + SourceReaderContext sourceReaderContext) throws Exception { + return new Reader(); } - @SuppressWarnings("BusyWait") - @Override - void scanTable() throws Exception { - while (isRunning.get()) { - MultiTableScanBase.ScanResult scanResult = tableScan.scanTable(ctx); + private class Reader extends AbstractNonCoordinatedSourceReader> { + private transient MultiTableScanBase> tableScan; + + @Override + public void start() { + super.start(); + tableScan = + new MultiAwareBucketTableScan( + catalogLoader, + includingPattern, + excludingPattern, + databasePattern, + isStreaming); + } + + @Override + public InputStatus pollNext(ReaderOutput> readerOutput) + throws Exception { + MultiTableScanBase.ScanResult scanResult = tableScan.scanTable(readerOutput); if (scanResult == FINISHED) { - return; + return InputStatus.END_OF_INPUT; } if (scanResult == IS_EMPTY) { Thread.sleep(monitorInterval); } + return InputStatus.MORE_AVAILABLE; + } + + @Override + public void close() throws Exception { + super.close(); + if (tableScan != null) { + tableScan.close(); + } } } @@ -96,37 +113,22 @@ public static DataStream buildSource( Pattern databasePattern, long monitorInterval) { - CombinedAwareStreamingSourceFunction function = - new CombinedAwareStreamingSourceFunction( + CombinedAwareStreamingSource source = + new CombinedAwareStreamingSource( catalogLoader, includingPattern, excludingPattern, databasePattern, monitorInterval); - StreamSource, ?> sourceOperator = new StreamSource<>(function); - boolean isParallel = false; TupleTypeInfo> tupleTypeInfo = new TupleTypeInfo<>( new JavaTypeInfo<>(Split.class), BasicTypeInfo.STRING_TYPE_INFO); - return new DataStreamSource<>( - env, - tupleTypeInfo, - sourceOperator, - isParallel, - name, - Boundedness.CONTINUOUS_UNBOUNDED) + + return env.fromSource(source, WatermarkStrategy.noWatermarks(), name, tupleTypeInfo) .forceNonParallel() .partitionCustom( (key, numPartitions) -> key % numPartitions, split -> ((DataSplit) split.f0).bucket()) .transform(name, typeInfo, new MultiTablesReadOperator(catalogLoader, true)); } - - @Override - public void close() throws Exception { - super.close(); - if (tableScan != null) { - tableScan.close(); - } - } } diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/operator/CombinedCompactorSourceFunction.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/operator/CombinedCompactorSource.java similarity index 70% rename from paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/operator/CombinedCompactorSourceFunction.java rename to paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/operator/CombinedCompactorSource.java index 1964927b5cdd..f58d86cdd65e 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/operator/CombinedCompactorSourceFunction.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/operator/CombinedCompactorSource.java @@ -20,12 +20,11 @@ import org.apache.paimon.append.UnawareAppendCompactionTask; import org.apache.paimon.catalog.Catalog; +import org.apache.paimon.flink.source.AbstractNonCoordinatedSource; import org.apache.paimon.table.source.Split; -import org.apache.flink.configuration.Configuration; -import org.apache.flink.streaming.api.functions.source.RichSourceFunction; +import org.apache.flink.api.connector.source.Boundedness; -import java.util.concurrent.atomic.AtomicBoolean; import java.util.regex.Pattern; /** @@ -44,8 +43,7 @@ *

    Currently, only dedicated compaction job for multi-tables rely on this monitor. This is the * single (non-parallel) monitoring task, it is responsible for the new Paimon table. */ -public abstract class CombinedCompactorSourceFunction extends RichSourceFunction { - +public abstract class CombinedCompactorSource extends AbstractNonCoordinatedSource { private static final long serialVersionUID = 2L; protected final Catalog.Loader catalogLoader; @@ -54,10 +52,7 @@ public abstract class CombinedCompactorSourceFunction extends RichSourceFunct protected final Pattern databasePattern; protected final boolean isStreaming; - protected transient AtomicBoolean isRunning; - protected transient SourceContext ctx; - - public CombinedCompactorSourceFunction( + public CombinedCompactorSource( Catalog.Loader catalogLoader, Pattern includingPattern, Pattern excludingPattern, @@ -71,27 +66,7 @@ public CombinedCompactorSourceFunction( } @Override - public void open(Configuration parameters) throws Exception { - isRunning = new AtomicBoolean(true); - } - - @Override - public void run(SourceContext sourceContext) throws Exception { - this.ctx = sourceContext; - scanTable(); + public Boundedness getBoundedness() { + return isStreaming ? Boundedness.CONTINUOUS_UNBOUNDED : Boundedness.BOUNDED; } - - @Override - public void cancel() { - // this is to cover the case where cancel() is called before the run() - if (ctx != null) { - synchronized (ctx.getCheckpointLock()) { - isRunning.set(false); - } - } else { - isRunning.set(false); - } - } - - abstract void scanTable() throws Exception; } diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/operator/CombinedUnawareBatchSourceFunction.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/operator/CombinedUnawareBatchSource.java similarity index 71% rename from paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/operator/CombinedUnawareBatchSourceFunction.java rename to paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/operator/CombinedUnawareBatchSource.java index b0b5a7784a38..64f0c38f5a11 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/operator/CombinedUnawareBatchSourceFunction.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/operator/CombinedUnawareBatchSource.java @@ -25,19 +25,20 @@ import org.apache.paimon.flink.compact.MultiTableScanBase; import org.apache.paimon.flink.compact.MultiUnawareBucketTableScan; import org.apache.paimon.flink.sink.MultiTableCompactionTaskTypeInfo; +import org.apache.paimon.flink.source.AbstractNonCoordinatedSourceReader; +import org.apache.paimon.flink.source.SimpleSourceSplit; import org.apache.paimon.manifest.PartitionEntry; import org.apache.paimon.table.FileStoreTable; import org.apache.paimon.table.Table; -import org.apache.flink.api.connector.source.Boundedness; -import org.apache.flink.configuration.Configuration; +import org.apache.flink.api.common.eventtime.WatermarkStrategy; +import org.apache.flink.api.connector.source.ReaderOutput; +import org.apache.flink.api.connector.source.SourceReader; +import org.apache.flink.api.connector.source.SourceReaderContext; +import org.apache.flink.core.io.InputStatus; import org.apache.flink.streaming.api.datastream.DataStream; -import org.apache.flink.streaming.api.datastream.DataStreamSource; import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; -import org.apache.flink.streaming.api.operators.StreamSource; -import org.apache.flink.streaming.api.transformations.PartitionTransformation; -import org.apache.flink.streaming.runtime.partitioner.RebalancePartitioner; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -56,14 +57,12 @@ * It is responsible for the batch compactor source of the table with unaware bucket in combined * mode. */ -public class CombinedUnawareBatchSourceFunction - extends CombinedCompactorSourceFunction { +public class CombinedUnawareBatchSource + extends CombinedCompactorSource { - private static final Logger LOGGER = - LoggerFactory.getLogger(CombinedUnawareBatchSourceFunction.class); - private transient MultiTableScanBase tableScan; + private static final Logger LOGGER = LoggerFactory.getLogger(CombinedUnawareBatchSource.class); - public CombinedUnawareBatchSourceFunction( + public CombinedUnawareBatchSource( Catalog.Loader catalogLoader, Pattern includingPattern, Pattern excludingPattern, @@ -72,24 +71,33 @@ public CombinedUnawareBatchSourceFunction( } @Override - public void open(Configuration parameters) throws Exception { - super.open(parameters); - tableScan = - new MultiUnawareBucketTableScan( - catalogLoader, - includingPattern, - excludingPattern, - databasePattern, - isStreaming, - isRunning); + public SourceReader createReader( + SourceReaderContext sourceReaderContext) throws Exception { + return new Reader(); } - @Override - void scanTable() throws Exception { - if (isRunning.get()) { - MultiTableScanBase.ScanResult scanResult = tableScan.scanTable(ctx); + private class Reader + extends AbstractNonCoordinatedSourceReader { + private transient MultiTableScanBase tableScan; + + @Override + public void start() { + super.start(); + tableScan = + new MultiUnawareBucketTableScan( + catalogLoader, + includingPattern, + excludingPattern, + databasePattern, + isStreaming); + } + + @Override + public InputStatus pollNext( + ReaderOutput readerOutput) throws Exception { + MultiTableScanBase.ScanResult scanResult = tableScan.scanTable(readerOutput); if (scanResult == FINISHED) { - return; + return InputStatus.END_OF_INPUT; } if (scanResult == IS_EMPTY) { // Currently, in the combined mode, there are two scan tasks for the table of two @@ -98,6 +106,15 @@ void scanTable() throws Exception { // should not be thrown exception here. LOGGER.info("No file were collected for the table of unaware-bucket"); } + return InputStatus.END_OF_INPUT; + } + + @Override + public void close() throws Exception { + super.close(); + if (tableScan != null) { + tableScan.close(); + } } } @@ -109,22 +126,18 @@ public static DataStream buildSource( Pattern excludingPattern, Pattern databasePattern, @Nullable Duration partitionIdleTime) { - CombinedUnawareBatchSourceFunction function = - new CombinedUnawareBatchSourceFunction( + CombinedUnawareBatchSource combinedUnawareBatchSource = + new CombinedUnawareBatchSource( catalogLoader, includingPattern, excludingPattern, databasePattern); - StreamSource - sourceOperator = new StreamSource<>(function); MultiTableCompactionTaskTypeInfo compactionTaskTypeInfo = new MultiTableCompactionTaskTypeInfo(); SingleOutputStreamOperator source = - new DataStreamSource<>( - env, - compactionTaskTypeInfo, - sourceOperator, - false, + env.fromSource( + combinedUnawareBatchSource, + WatermarkStrategy.noWatermarks(), name, - Boundedness.BOUNDED) + compactionTaskTypeInfo) .forceNonParallel(); if (partitionIdleTime != null) { @@ -135,11 +148,7 @@ public static DataStream buildSource( new MultiUnawareTablesReadOperator(catalogLoader, partitionIdleTime)); } - PartitionTransformation transformation = - new PartitionTransformation<>( - source.getTransformation(), new RebalancePartitioner<>()); - - return new DataStream<>(env, transformation); + return source; } private static Long getPartitionInfo( @@ -173,12 +182,4 @@ private static Long getPartitionInfo( } return partitionInfo.get(partition); } - - @Override - public void close() throws Exception { - super.close(); - if (tableScan != null) { - tableScan.close(); - } - } } diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/operator/CombinedUnawareStreamingSourceFunction.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/operator/CombinedUnawareStreamingSource.java similarity index 58% rename from paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/operator/CombinedUnawareStreamingSourceFunction.java rename to paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/operator/CombinedUnawareStreamingSource.java index 54a90ac3670e..6ea1ead4db30 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/operator/CombinedUnawareStreamingSourceFunction.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/operator/CombinedUnawareStreamingSource.java @@ -23,13 +23,16 @@ import org.apache.paimon.flink.compact.MultiTableScanBase; import org.apache.paimon.flink.compact.MultiUnawareBucketTableScan; import org.apache.paimon.flink.sink.MultiTableCompactionTaskTypeInfo; +import org.apache.paimon.flink.source.AbstractNonCoordinatedSourceReader; +import org.apache.paimon.flink.source.SimpleSourceSplit; -import org.apache.flink.api.connector.source.Boundedness; -import org.apache.flink.configuration.Configuration; +import org.apache.flink.api.common.eventtime.WatermarkStrategy; +import org.apache.flink.api.connector.source.ReaderOutput; +import org.apache.flink.api.connector.source.SourceReader; +import org.apache.flink.api.connector.source.SourceReaderContext; +import org.apache.flink.core.io.InputStatus; import org.apache.flink.streaming.api.datastream.DataStream; -import org.apache.flink.streaming.api.datastream.DataStreamSource; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; -import org.apache.flink.streaming.api.operators.StreamSource; import java.util.regex.Pattern; @@ -39,13 +42,12 @@ /** * It is responsible for monitoring compactor source in stream mode for the table of unaware bucket. */ -public class CombinedUnawareStreamingSourceFunction - extends CombinedCompactorSourceFunction { +public class CombinedUnawareStreamingSource + extends CombinedCompactorSource { private final long monitorInterval; - private MultiTableScanBase tableScan; - public CombinedUnawareStreamingSourceFunction( + public CombinedUnawareStreamingSource( Catalog.Loader catalogLoader, Pattern includingPattern, Pattern excludingPattern, @@ -56,29 +58,46 @@ public CombinedUnawareStreamingSourceFunction( } @Override - public void open(Configuration parameters) throws Exception { - super.open(parameters); - tableScan = - new MultiUnawareBucketTableScan( - catalogLoader, - includingPattern, - excludingPattern, - databasePattern, - isStreaming, - isRunning); + public SourceReader createReader( + SourceReaderContext sourceReaderContext) throws Exception { + return new Reader(); } - @SuppressWarnings("BusyWait") - @Override - void scanTable() throws Exception { - while (isRunning.get()) { - MultiTableScanBase.ScanResult scanResult = tableScan.scanTable(ctx); + private class Reader + extends AbstractNonCoordinatedSourceReader { + private MultiTableScanBase tableScan; + + @Override + public void start() { + super.start(); + tableScan = + new MultiUnawareBucketTableScan( + catalogLoader, + includingPattern, + excludingPattern, + databasePattern, + isStreaming); + } + + @Override + public InputStatus pollNext( + ReaderOutput readerOutput) throws Exception { + MultiTableScanBase.ScanResult scanResult = tableScan.scanTable(readerOutput); if (scanResult == FINISHED) { - return; + return InputStatus.END_OF_INPUT; } if (scanResult == IS_EMPTY) { Thread.sleep(monitorInterval); } + return InputStatus.MORE_AVAILABLE; + } + + @Override + public void close() throws Exception { + super.close(); + if (tableScan != null) { + tableScan.close(); + } } } @@ -91,34 +110,18 @@ public static DataStream buildSource( Pattern databasePattern, long monitorInterval) { - CombinedUnawareStreamingSourceFunction function = - new CombinedUnawareStreamingSourceFunction( + CombinedUnawareStreamingSource source = + new CombinedUnawareStreamingSource( catalogLoader, includingPattern, excludingPattern, databasePattern, monitorInterval); - StreamSource - sourceOperator = new StreamSource<>(function); - boolean isParallel = false; MultiTableCompactionTaskTypeInfo compactionTaskTypeInfo = new MultiTableCompactionTaskTypeInfo(); - return new DataStreamSource<>( - env, - compactionTaskTypeInfo, - sourceOperator, - isParallel, - name, - Boundedness.CONTINUOUS_UNBOUNDED) - .forceNonParallel() - .rebalance(); - } - @Override - public void close() throws Exception { - super.close(); - if (tableScan != null) { - tableScan.close(); - } + return env.fromSource( + source, WatermarkStrategy.noWatermarks(), name, compactionTaskTypeInfo) + .forceNonParallel(); } } diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/operator/MonitorFunction.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/operator/MonitorSource.java similarity index 53% rename from paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/operator/MonitorFunction.java rename to paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/operator/MonitorSource.java index 3805f6f8c536..4ec0a4f99d9f 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/operator/MonitorFunction.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/operator/MonitorSource.java @@ -18,6 +18,10 @@ package org.apache.paimon.flink.source.operator; +import org.apache.paimon.flink.source.AbstractNonCoordinatedSource; +import org.apache.paimon.flink.source.AbstractNonCoordinatedSourceReader; +import org.apache.paimon.flink.source.SimpleSourceSplit; +import org.apache.paimon.flink.source.SplitListState; import org.apache.paimon.flink.utils.JavaTypeInfo; import org.apache.paimon.table.BucketMode; import org.apache.paimon.table.sink.ChannelComputer; @@ -27,22 +31,18 @@ import org.apache.paimon.table.source.Split; import org.apache.paimon.table.source.StreamTableScan; -import org.apache.flink.api.common.state.CheckpointListener; -import org.apache.flink.api.common.state.ListState; -import org.apache.flink.api.common.state.ListStateDescriptor; +import org.apache.flink.api.common.eventtime.Watermark; +import org.apache.flink.api.common.eventtime.WatermarkStrategy; import org.apache.flink.api.common.typeinfo.TypeInformation; -import org.apache.flink.api.common.typeutils.TypeSerializer; -import org.apache.flink.api.common.typeutils.base.LongSerializer; +import org.apache.flink.api.connector.source.Boundedness; +import org.apache.flink.api.connector.source.ReaderOutput; +import org.apache.flink.api.connector.source.SourceReader; +import org.apache.flink.api.connector.source.SourceReaderContext; import org.apache.flink.api.java.tuple.Tuple2; -import org.apache.flink.api.java.typeutils.runtime.TupleSerializer; -import org.apache.flink.runtime.state.FunctionInitializationContext; -import org.apache.flink.runtime.state.FunctionSnapshotContext; -import org.apache.flink.streaming.api.checkpoint.CheckpointedFunction; +import org.apache.flink.core.io.InputStatus; import org.apache.flink.streaming.api.datastream.DataStream; import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; -import org.apache.flink.streaming.api.functions.source.RichSourceFunction; -import org.apache.flink.streaming.api.watermark.Watermark; import org.apache.flink.table.data.RowData; import org.apache.flink.util.Preconditions; import org.slf4j.Logger; @@ -71,33 +71,23 @@ *

    Currently, there are two features that rely on this monitor: * *

      - *
    1. Consumer-id: rely on this function to do aligned snapshot consumption, and ensure that all + *
    2. Consumer-id: rely on this source to do aligned snapshot consumption, and ensure that all * data in a snapshot is consumed within each checkpoint. *
    3. Snapshot-watermark: when there is no watermark definition, the default Paimon table will * pass the watermark recorded in the snapshot. *
    */ -public class MonitorFunction extends RichSourceFunction - implements CheckpointedFunction, CheckpointListener { +public class MonitorSource extends AbstractNonCoordinatedSource { private static final long serialVersionUID = 1L; - private static final Logger LOG = LoggerFactory.getLogger(MonitorFunction.class); + private static final Logger LOG = LoggerFactory.getLogger(MonitorSource.class); private final ReadBuilder readBuilder; private final long monitorInterval; private final boolean emitSnapshotWatermark; - private volatile boolean isRunning = true; - - private transient StreamTableScan scan; - private transient SourceContext ctx; - - private transient ListState checkpointState; - private transient ListState> nextSnapshotState; - private transient TreeMap nextSnapshotPerCheckpoint; - - public MonitorFunction( + public MonitorSource( ReadBuilder readBuilder, long monitorInterval, boolean emitSnapshotWatermark) { this.readBuilder = readBuilder; this.monitorInterval = monitorInterval; @@ -105,40 +95,74 @@ public MonitorFunction( } @Override - public void initializeState(FunctionInitializationContext context) throws Exception { - this.scan = readBuilder.newStreamScan(); - - this.checkpointState = - context.getOperatorStateStore() - .getListState( - new ListStateDescriptor<>( - "next-snapshot", LongSerializer.INSTANCE)); - - @SuppressWarnings("unchecked") - final Class> typedTuple = - (Class>) (Class) Tuple2.class; - this.nextSnapshotState = - context.getOperatorStateStore() - .getListState( - new ListStateDescriptor<>( - "next-snapshot-per-checkpoint", - new TupleSerializer<>( - typedTuple, - new TypeSerializer[] { - LongSerializer.INSTANCE, LongSerializer.INSTANCE - }))); - - this.nextSnapshotPerCheckpoint = new TreeMap<>(); - - if (context.isRestored()) { - LOG.info("Restoring state for the {}.", getClass().getSimpleName()); + public Boundedness getBoundedness() { + return Boundedness.CONTINUOUS_UNBOUNDED; + } + + @Override + public SourceReader createReader( + SourceReaderContext sourceReaderContext) throws Exception { + return new Reader(); + } + + private class Reader extends AbstractNonCoordinatedSourceReader { + private static final String CHECKPOINT_STATE = "CS"; + private static final String NEXT_SNAPSHOT_STATE = "NSS"; + + private final StreamTableScan scan = readBuilder.newStreamScan(); + private final SplitListState checkpointState = + new SplitListState<>(CHECKPOINT_STATE, x -> Long.toString(x), Long::parseLong); + private final SplitListState> nextSnapshotState = + new SplitListState<>( + NEXT_SNAPSHOT_STATE, + x -> x.f0 + ":" + x.f1, + x -> + Tuple2.of( + Long.parseLong(x.split(":")[0]), + Long.parseLong(x.split(":")[1]))); + private final TreeMap nextSnapshotPerCheckpoint = new TreeMap<>(); + + @Override + public void notifyCheckpointComplete(long checkpointId) { + NavigableMap nextSnapshots = + nextSnapshotPerCheckpoint.headMap(checkpointId, true); + OptionalLong max = nextSnapshots.values().stream().mapToLong(Long::longValue).max(); + max.ifPresent(scan::notifyCheckpointComplete); + nextSnapshots.clear(); + } - List retrievedStates = new ArrayList<>(); - for (Long entry : this.checkpointState.get()) { - retrievedStates.add(entry); + @Override + public List snapshotState(long checkpointId) { + this.checkpointState.clear(); + Long nextSnapshot = this.scan.checkpoint(); + if (nextSnapshot != null) { + this.checkpointState.add(nextSnapshot); + this.nextSnapshotPerCheckpoint.put(checkpointId, nextSnapshot); } - // given that the parallelism of the function is 1, we can only have 1 retrieved items. + List> nextSnapshots = new ArrayList<>(); + this.nextSnapshotPerCheckpoint.forEach((k, v) -> nextSnapshots.add(new Tuple2<>(k, v))); + this.nextSnapshotState.update(nextSnapshots); + + if (LOG.isDebugEnabled()) { + LOG.debug("{} checkpoint {}.", getClass().getSimpleName(), nextSnapshot); + } + + List results = new ArrayList<>(); + results.addAll(checkpointState.snapshotState()); + results.addAll(nextSnapshotState.snapshotState()); + return results; + } + + @Override + public void addSplits(List list) { + LOG.info("Restoring state for the {}.", getClass().getSimpleName()); + checkpointState.restoreState(list); + nextSnapshotState.restoreState(list); + + List retrievedStates = checkpointState.get(); + + // given that the parallelism of the source is 1, we can only have 1 retrieved items. Preconditions.checkArgument( retrievedStates.size() <= 1, getClass().getSimpleName() + " retrieved invalid state."); @@ -150,80 +174,31 @@ public void initializeState(FunctionInitializationContext context) throws Except for (Tuple2 tuple2 : nextSnapshotState.get()) { nextSnapshotPerCheckpoint.put(tuple2.f0, tuple2.f1); } - } else { - LOG.info("No state to restore for the {}.", getClass().getSimpleName()); } - } - - @Override - public void snapshotState(FunctionSnapshotContext ctx) throws Exception { - this.checkpointState.clear(); - Long nextSnapshot = this.scan.checkpoint(); - if (nextSnapshot != null) { - this.checkpointState.add(nextSnapshot); - this.nextSnapshotPerCheckpoint.put(ctx.getCheckpointId(), nextSnapshot); - } - - List> nextSnapshots = new ArrayList<>(); - this.nextSnapshotPerCheckpoint.forEach((k, v) -> nextSnapshots.add(new Tuple2<>(k, v))); - this.nextSnapshotState.update(nextSnapshots); - if (LOG.isDebugEnabled()) { - LOG.debug("{} checkpoint {}.", getClass().getSimpleName(), nextSnapshot); - } - } - - @SuppressWarnings("BusyWait") - @Override - public void run(SourceContext ctx) throws Exception { - this.ctx = ctx; - while (isRunning) { + @Override + public InputStatus pollNext(ReaderOutput readerOutput) throws Exception { boolean isEmpty; - synchronized (ctx.getCheckpointLock()) { - if (!isRunning) { - return; - } - try { - List splits = scan.plan().splits(); - isEmpty = splits.isEmpty(); - splits.forEach(ctx::collect); - - if (emitSnapshotWatermark) { - Long watermark = scan.watermark(); - if (watermark != null) { - ctx.emitWatermark(new Watermark(watermark)); - } + try { + List splits = scan.plan().splits(); + isEmpty = splits.isEmpty(); + splits.forEach(readerOutput::collect); + + if (emitSnapshotWatermark) { + Long watermark = scan.watermark(); + if (watermark != null) { + readerOutput.emitWatermark(new Watermark(watermark)); } - } catch (EndOfScanException esf) { - LOG.info("Catching EndOfStreamException, the stream is finished."); - return; } + } catch (EndOfScanException esf) { + LOG.info("Catching EndOfStreamException, the stream is finished."); + return InputStatus.END_OF_INPUT; } if (isEmpty) { Thread.sleep(monitorInterval); } - } - } - - @Override - public void notifyCheckpointComplete(long checkpointId) { - NavigableMap nextSnapshots = - nextSnapshotPerCheckpoint.headMap(checkpointId, true); - OptionalLong max = nextSnapshots.values().stream().mapToLong(Long::longValue).max(); - max.ifPresent(scan::notifyCheckpointComplete); - nextSnapshots.clear(); - } - - @Override - public void cancel() { - // this is to cover the case where cancel() is called before the run() - if (ctx != null) { - synchronized (ctx.getCheckpointLock()) { - isRunning = false; - } - } else { - isRunning = false; + return InputStatus.MORE_AVAILABLE; } } @@ -237,9 +212,10 @@ public static DataStream buildSource( boolean shuffleBucketWithPartition, BucketMode bucketMode) { SingleOutputStreamOperator singleOutputStreamOperator = - env.addSource( - new MonitorFunction( + env.fromSource( + new MonitorSource( readBuilder, monitorInterval, emitSnapshotWatermark), + WatermarkStrategy.noWatermarks(), name + "-Monitor", new JavaTypeInfo<>(Split.class)) .forceNonParallel(); diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/operator/MultiTablesReadOperator.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/operator/MultiTablesReadOperator.java index 73d46ae1e3f1..fbc8bb9d756a 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/operator/MultiTablesReadOperator.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/operator/MultiTablesReadOperator.java @@ -52,9 +52,8 @@ /** * The operator that reads the Tuple2<{@link Split}, String> received from the preceding {@link - * CombinedAwareBatchSourceFunction} or {@link CombinedAwareStreamingSourceFunction}. Contrary to - * the {@link CombinedCompactorSourceFunction} which has a parallelism of 1, this operator can have - * DOP > 1. + * CombinedAwareBatchSource} or {@link CombinedAwareStreamingSource}. Contrary to the {@link + * CombinedCompactorSource} which has a parallelism of 1, this operator can have DOP > 1. */ public class MultiTablesReadOperator extends AbstractStreamOperator implements OneInputStreamOperator, RowData> { diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/operator/MultiUnawareTablesReadOperator.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/operator/MultiUnawareTablesReadOperator.java index c501c2519b41..0864741a178f 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/operator/MultiUnawareTablesReadOperator.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/operator/MultiUnawareTablesReadOperator.java @@ -44,7 +44,7 @@ /** * The operator is used for historical partition compaction. It reads {@link * MultiTableUnawareAppendCompactionTask} received from the preceding {@link - * CombinedUnawareBatchSourceFunction} and filter partitions which is not historical. + * CombinedUnawareBatchSource} and filter partitions which is not historical. */ public class MultiUnawareTablesReadOperator extends AbstractStreamOperator diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/operator/ReadOperator.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/operator/ReadOperator.java index 80c85f7cdb35..ccc66194560e 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/operator/ReadOperator.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/operator/ReadOperator.java @@ -38,8 +38,8 @@ /** * The operator that reads the {@link Split splits} received from the preceding {@link - * MonitorFunction}. Contrary to the {@link MonitorFunction} which has a parallelism of 1, this - * operator can have DOP > 1. + * MonitorSource}. Contrary to the {@link MonitorSource} which has a parallelism of 1, this operator + * can have DOP > 1. */ public class ReadOperator extends AbstractStreamOperator implements OneInputStreamOperator { @@ -54,9 +54,11 @@ public class ReadOperator extends AbstractStreamOperator private transient IOManager ioManager; private transient FileStoreSourceReaderMetrics sourceReaderMetrics; - // we create our own gauge for currentEmitEventTimeLag, because this operator is not a FLIP-27 + // we create our own gauge for currentEmitEventTimeLag and sourceIdleTime, because this operator + // is not a FLIP-27 // source and Flink can't automatically calculate this metric private transient long emitEventTimeLag = FileStoreSourceReaderMetrics.UNDEFINED; + private transient long idleStartTime = FileStoreSourceReaderMetrics.ACTIVE; private transient Counter numRecordsIn; public ReadOperator(ReadBuilder readBuilder) { @@ -69,6 +71,7 @@ public void open() throws Exception { this.sourceReaderMetrics = new FileStoreSourceReaderMetrics(getMetricGroup()); getMetricGroup().gauge(MetricNames.CURRENT_EMIT_EVENT_TIME_LAG, () -> emitEventTimeLag); + getMetricGroup().gauge(MetricNames.SOURCE_IDLE_TIME, this::getIdleTime); this.numRecordsIn = InternalSourceReaderMetricGroup.wrap(getMetricGroup()) .getIOMetricGroup() @@ -83,6 +86,7 @@ public void open() throws Exception { this.read = readBuilder.newRead().withIOManager(ioManager); this.reuseRow = new FlinkRowData(null); this.reuseRecord = new StreamRecord<>(reuseRow); + this.idlingStarted(); } @Override @@ -94,6 +98,8 @@ public void processElement(StreamRecord record) throws Exception { .earliestFileCreationEpochMillis() .orElse(FileStoreSourceReaderMetrics.UNDEFINED); sourceReaderMetrics.recordSnapshotUpdate(eventTime); + // update idleStartTime when reading a new split + idleStartTime = FileStoreSourceReaderMetrics.ACTIVE; boolean firstRecord = true; try (CloseableIterator iterator = @@ -113,6 +119,8 @@ public void processElement(StreamRecord record) throws Exception { output.collect(reuseRecord); } } + // start idle when data sending is completed + this.idlingStarted(); } @Override @@ -122,4 +130,18 @@ public void close() throws Exception { ioManager.close(); } } + + private void idlingStarted() { + if (!isIdling()) { + idleStartTime = System.currentTimeMillis(); + } + } + + private boolean isIdling() { + return idleStartTime != FileStoreSourceReaderMetrics.ACTIVE; + } + + private long getIdleTime() { + return isIdling() ? System.currentTimeMillis() - idleStartTime : 0; + } } diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/utils/FlinkCatalogPropertiesUtil.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/utils/FlinkCatalogPropertiesUtil.java index b0f99a6e89e4..fa84a1ca070d 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/utils/FlinkCatalogPropertiesUtil.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/utils/FlinkCatalogPropertiesUtil.java @@ -20,8 +20,7 @@ import org.apache.paimon.shade.guava30.com.google.common.collect.ImmutableSet; -import org.apache.flink.table.api.TableColumn; -import org.apache.flink.table.api.WatermarkSpec; +import org.apache.flink.table.api.Schema; import org.apache.flink.table.catalog.Column; import org.apache.flink.table.catalog.ResolvedSchema; import org.apache.flink.table.types.DataType; @@ -36,48 +35,23 @@ import java.util.stream.Collectors; import java.util.stream.Stream; -import static org.apache.flink.table.descriptors.DescriptorProperties.COMMENT; -import static org.apache.flink.table.descriptors.DescriptorProperties.DATA_TYPE; -import static org.apache.flink.table.descriptors.DescriptorProperties.EXPR; -import static org.apache.flink.table.descriptors.DescriptorProperties.METADATA; -import static org.apache.flink.table.descriptors.DescriptorProperties.NAME; -import static org.apache.flink.table.descriptors.DescriptorProperties.VIRTUAL; -import static org.apache.flink.table.descriptors.DescriptorProperties.WATERMARK; -import static org.apache.flink.table.descriptors.DescriptorProperties.WATERMARK_ROWTIME; -import static org.apache.flink.table.descriptors.DescriptorProperties.WATERMARK_STRATEGY_DATA_TYPE; -import static org.apache.flink.table.descriptors.DescriptorProperties.WATERMARK_STRATEGY_EXPR; -import static org.apache.flink.table.descriptors.Schema.SCHEMA; +import static org.apache.paimon.flink.utils.FlinkDescriptorProperties.COMMENT; +import static org.apache.paimon.flink.utils.FlinkDescriptorProperties.DATA_TYPE; +import static org.apache.paimon.flink.utils.FlinkDescriptorProperties.EXPR; +import static org.apache.paimon.flink.utils.FlinkDescriptorProperties.METADATA; +import static org.apache.paimon.flink.utils.FlinkDescriptorProperties.NAME; +import static org.apache.paimon.flink.utils.FlinkDescriptorProperties.VIRTUAL; +import static org.apache.paimon.flink.utils.FlinkDescriptorProperties.WATERMARK; +import static org.apache.paimon.flink.utils.FlinkDescriptorProperties.WATERMARK_ROWTIME; +import static org.apache.paimon.flink.utils.FlinkDescriptorProperties.WATERMARK_STRATEGY_DATA_TYPE; +import static org.apache.paimon.flink.utils.FlinkDescriptorProperties.WATERMARK_STRATEGY_EXPR; /** * Utilities for ser/deserializing non-physical columns and watermark into/from a map of string * properties. */ public class FlinkCatalogPropertiesUtil { - - public static Map serializeNonPhysicalColumns( - Map indexMap, List nonPhysicalColumns) { - Map serialized = new HashMap<>(); - for (TableColumn c : nonPhysicalColumns) { - int index = indexMap.get(c.getName()); - serialized.put(compoundKey(SCHEMA, index, NAME), c.getName()); - serialized.put( - compoundKey(SCHEMA, index, DATA_TYPE), - c.getType().getLogicalType().asSerializableString()); - if (c instanceof TableColumn.ComputedColumn) { - TableColumn.ComputedColumn computedColumn = (TableColumn.ComputedColumn) c; - serialized.put(compoundKey(SCHEMA, index, EXPR), computedColumn.getExpression()); - } else { - TableColumn.MetadataColumn metadataColumn = (TableColumn.MetadataColumn) c; - serialized.put( - compoundKey(SCHEMA, index, METADATA), - metadataColumn.getMetadataAlias().orElse(metadataColumn.getName())); - serialized.put( - compoundKey(SCHEMA, index, VIRTUAL), - Boolean.toString(metadataColumn.isVirtual())); - } - } - return serialized; - } + public static final String SCHEMA = "schema"; /** Serialize non-physical columns of new api. */ public static Map serializeNonPhysicalNewColumns(ResolvedSchema schema) { @@ -119,22 +93,6 @@ public static Map serializeNonPhysicalNewColumns(ResolvedSchema return serialized; } - public static Map serializeWatermarkSpec(WatermarkSpec watermarkSpec) { - Map serializedWatermarkSpec = new HashMap<>(); - String watermarkPrefix = compoundKey(SCHEMA, WATERMARK, 0); - serializedWatermarkSpec.put( - compoundKey(watermarkPrefix, WATERMARK_ROWTIME), - watermarkSpec.getRowtimeAttribute()); - serializedWatermarkSpec.put( - compoundKey(watermarkPrefix, WATERMARK_STRATEGY_EXPR), - watermarkSpec.getWatermarkExpr()); - serializedWatermarkSpec.put( - compoundKey(watermarkPrefix, WATERMARK_STRATEGY_DATA_TYPE), - watermarkSpec.getWatermarkExprOutputType().getLogicalType().asSerializableString()); - - return serializedWatermarkSpec; - } - public static Map serializeNewWatermarkSpec( org.apache.flink.table.catalog.WatermarkSpec watermarkSpec) { Map serializedWatermarkSpec = new HashMap<>(); @@ -219,7 +177,8 @@ private static boolean isColumnNameKey(String key) { && SCHEMA_COLUMN_NAME_SUFFIX.matcher(key.substring(SCHEMA.length() + 1)).matches(); } - public static TableColumn deserializeNonPhysicalColumn(Map options, int index) { + public static void deserializeNonPhysicalColumn( + Map options, int index, Schema.Builder builder) { String nameKey = compoundKey(SCHEMA, index, NAME); String dataTypeKey = compoundKey(SCHEMA, index, DATA_TYPE); String exprKey = compoundKey(SCHEMA, index, EXPR); @@ -227,45 +186,42 @@ public static TableColumn deserializeNonPhysicalColumn(Map optio String virtualKey = compoundKey(SCHEMA, index, VIRTUAL); String name = options.get(nameKey); - DataType dataType = - TypeConversions.fromLogicalToDataType( - LogicalTypeParser.parse(options.get(dataTypeKey))); - TableColumn column; if (options.containsKey(exprKey)) { - column = TableColumn.computed(name, dataType, options.get(exprKey)); + final String expr = options.get(exprKey); + builder.columnByExpression(name, expr); } else if (options.containsKey(metadataKey)) { String metadataAlias = options.get(metadataKey); boolean isVirtual = Boolean.parseBoolean(options.get(virtualKey)); - column = - metadataAlias.equals(name) - ? TableColumn.metadata(name, dataType, isVirtual) - : TableColumn.metadata(name, dataType, metadataAlias, isVirtual); + DataType dataType = + TypeConversions.fromLogicalToDataType( + LogicalTypeParser.parse( + options.get(dataTypeKey), + Thread.currentThread().getContextClassLoader())); + if (metadataAlias.equals(name)) { + builder.columnByMetadata(name, dataType, isVirtual); + } else { + builder.columnByMetadata(name, dataType, metadataAlias, isVirtual); + } } else { throw new RuntimeException( String.format( "Failed to build non-physical column. Current index is %s, options are %s", index, options)); } - - return column; } - public static WatermarkSpec deserializeWatermarkSpec(Map options) { + public static void deserializeWatermarkSpec( + Map options, Schema.Builder builder) { String watermarkPrefixKey = compoundKey(SCHEMA, WATERMARK); String rowtimeKey = compoundKey(watermarkPrefixKey, 0, WATERMARK_ROWTIME); String exprKey = compoundKey(watermarkPrefixKey, 0, WATERMARK_STRATEGY_EXPR); - String dataTypeKey = compoundKey(watermarkPrefixKey, 0, WATERMARK_STRATEGY_DATA_TYPE); String rowtimeAttribute = options.get(rowtimeKey); String watermarkExpressionString = options.get(exprKey); - DataType watermarkExprOutputType = - TypeConversions.fromLogicalToDataType( - LogicalTypeParser.parse(options.get(dataTypeKey))); - return new WatermarkSpec( - rowtimeAttribute, watermarkExpressionString, watermarkExprOutputType); + builder.watermark(rowtimeAttribute, watermarkExpressionString); } public static String compoundKey(Object... components) { diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/utils/FlinkDescriptorProperties.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/utils/FlinkDescriptorProperties.java new file mode 100644 index 000000000000..edc73ca7bf41 --- /dev/null +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/utils/FlinkDescriptorProperties.java @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.flink.utils; + +import org.apache.flink.table.api.Schema; + +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +import static org.apache.flink.util.Preconditions.checkNotNull; + +/** + * Utility class for having a unified string-based representation of Table API related classes such + * as Schema, TypeInformation, etc. + * + *

    Note to implementers: Please try to reuse key names as much as possible. Key-names should be + * hierarchical and lower case. Use "-" instead of dots or camel case. E.g., + * connector.schema.start-from = from-earliest. Try not to use the higher level in a key-name. E.g., + * instead of connector.kafka.kafka-version use connector.kafka.version. + * + *

    Properties with key normalization enabled contain only lower-case keys. + */ +public class FlinkDescriptorProperties { + + public static final String NAME = "name"; + + public static final String DATA_TYPE = "data-type"; + + public static final String EXPR = "expr"; + + public static final String METADATA = "metadata"; + + public static final String VIRTUAL = "virtual"; + + public static final String WATERMARK = "watermark"; + + public static final String WATERMARK_ROWTIME = "rowtime"; + + public static final String WATERMARK_STRATEGY = "strategy"; + + public static final String WATERMARK_STRATEGY_EXPR = WATERMARK_STRATEGY + '.' + EXPR; + + public static final String WATERMARK_STRATEGY_DATA_TYPE = WATERMARK_STRATEGY + '.' + DATA_TYPE; + + public static final String PRIMARY_KEY_NAME = "primary-key.name"; + + public static final String PRIMARY_KEY_COLUMNS = "primary-key.columns"; + + public static final String COMMENT = "comment"; + + public static void removeSchemaKeys(String key, Schema schema, Map options) { + checkNotNull(key); + checkNotNull(schema); + + List subKeys = Arrays.asList(NAME, DATA_TYPE, EXPR, METADATA, VIRTUAL); + for (int idx = 0; idx < schema.getColumns().size(); idx++) { + for (String subKey : subKeys) { + options.remove(key + '.' + idx + '.' + subKey); + } + } + + if (!schema.getWatermarkSpecs().isEmpty()) { + subKeys = + Arrays.asList( + WATERMARK_ROWTIME, + WATERMARK_STRATEGY_EXPR, + WATERMARK_STRATEGY_DATA_TYPE); + for (int idx = 0; idx < schema.getWatermarkSpecs().size(); idx++) { + for (String subKey : subKeys) { + options.remove(key + '.' + WATERMARK + '.' + idx + '.' + subKey); + } + } + } + + schema.getPrimaryKey() + .ifPresent( + pk -> { + options.remove(key + '.' + PRIMARY_KEY_NAME); + options.remove(key + '.' + PRIMARY_KEY_COLUMNS); + }); + } +} diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/utils/InternalTypeInfo.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/utils/InternalTypeInfo.java index 4ea5db9f34d4..60898421ddea 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/utils/InternalTypeInfo.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/utils/InternalTypeInfo.java @@ -22,6 +22,7 @@ import org.apache.paimon.types.RowType; import org.apache.flink.api.common.ExecutionConfig; +import org.apache.flink.api.common.serialization.SerializerConfig; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.common.typeutils.TypeSerializer; @@ -73,8 +74,17 @@ public boolean isKeyType() { return false; } - @Override - public TypeSerializer createSerializer(ExecutionConfig config) { + /** + * Do not annotate with @override here to maintain compatibility with Flink 1.18-. + */ + public TypeSerializer createSerializer(SerializerConfig config) { + return this.createSerializer((ExecutionConfig) null); + } + + /** + * Do not annotate with @override here to maintain compatibility with Flink 2.0+. + */ + public TypeSerializer createSerializer(ExecutionConfig executionConfig) { return serializer.duplicate(); } diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/utils/JavaTypeInfo.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/utils/JavaTypeInfo.java index a36243c5bdac..4aea809b51bc 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/utils/JavaTypeInfo.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/utils/JavaTypeInfo.java @@ -20,6 +20,7 @@ import org.apache.flink.annotation.PublicEvolving; import org.apache.flink.api.common.ExecutionConfig; +import org.apache.flink.api.common.serialization.SerializerConfig; import org.apache.flink.api.common.typeinfo.AtomicType; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.common.typeutils.TypeComparator; @@ -78,7 +79,16 @@ public boolean isKeyType() { return Comparable.class.isAssignableFrom(typeClass); } - @Override + /** + * Do not annotate with @override here to maintain compatibility with Flink 1.18-. + */ + public TypeSerializer createSerializer(SerializerConfig config) { + return this.createSerializer((ExecutionConfig) null); + } + + /** + * Do not annotate with @override here to maintain compatibility with Flink 2.0+. + */ public TypeSerializer createSerializer(ExecutionConfig config) { return new JavaSerializer<>(this.typeClass); } @@ -91,7 +101,9 @@ public TypeComparator createComparator( @SuppressWarnings("rawtypes") GenericTypeComparator comparator = new GenericTypeComparator( - sortOrderAscending, createSerializer(executionConfig), this.typeClass); + sortOrderAscending, + new JavaSerializer<>(this.typeClass), + this.typeClass); return (TypeComparator) comparator; } diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/utils/RuntimeContextUtils.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/utils/RuntimeContextUtils.java new file mode 100644 index 000000000000..34e0d041b6a0 --- /dev/null +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/utils/RuntimeContextUtils.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.flink.utils; + +import org.apache.flink.api.common.functions.RuntimeContext; + +/** Utility methods about Flink runtime context to resolve compatibility issues. */ +public class RuntimeContextUtils { + public static int getNumberOfParallelSubtasks(RuntimeContext context) { + return context.getTaskInfo().getNumberOfParallelSubtasks(); + } + + public static int getIndexOfThisSubtask(RuntimeContext context) { + return context.getTaskInfo().getIndexOfThisSubtask(); + } +} diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/utils/TableStatsUtil.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/utils/TableStatsUtil.java new file mode 100644 index 000000000000..fe10fa63c049 --- /dev/null +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/utils/TableStatsUtil.java @@ -0,0 +1,281 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.flink.utils; + +import org.apache.paimon.Snapshot; +import org.apache.paimon.data.Decimal; +import org.apache.paimon.stats.ColStats; +import org.apache.paimon.stats.Statistics; +import org.apache.paimon.table.FileStoreTable; +import org.apache.paimon.types.DataField; +import org.apache.paimon.types.DataTypeRoot; + +import org.apache.flink.table.catalog.exceptions.CatalogException; +import org.apache.flink.table.catalog.stats.CatalogColumnStatistics; +import org.apache.flink.table.catalog.stats.CatalogColumnStatisticsDataBase; +import org.apache.flink.table.catalog.stats.CatalogColumnStatisticsDataBinary; +import org.apache.flink.table.catalog.stats.CatalogColumnStatisticsDataBoolean; +import org.apache.flink.table.catalog.stats.CatalogColumnStatisticsDataDate; +import org.apache.flink.table.catalog.stats.CatalogColumnStatisticsDataDouble; +import org.apache.flink.table.catalog.stats.CatalogColumnStatisticsDataLong; +import org.apache.flink.table.catalog.stats.CatalogColumnStatisticsDataString; +import org.apache.flink.table.catalog.stats.CatalogTableStatistics; + +import javax.annotation.Nullable; + +import java.math.BigDecimal; +import java.sql.Timestamp; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** Utility methods for analysis table. */ +public class TableStatsUtil { + + /** create Paimon statistics. */ + @Nullable + public static Statistics createTableStats( + FileStoreTable table, CatalogTableStatistics catalogTableStatistics) { + Snapshot snapshot = table.snapshotManager().latestSnapshot(); + if (snapshot == null) { + return null; + } + return new Statistics( + snapshot.id(), + snapshot.schemaId(), + catalogTableStatistics.getRowCount(), + catalogTableStatistics.getTotalSize()); + } + + /** Create Paimon statistics from given Flink columnStatistics. */ + @Nullable + public static Statistics createTableColumnStats( + FileStoreTable table, CatalogColumnStatistics columnStatistics) { + if (!table.statistics().isPresent()) { + return null; + } + Statistics statistics = table.statistics().get(); + List fields = table.schema().fields(); + Map> tableColumnStatsMap = new HashMap<>(fields.size()); + for (DataField field : fields) { + CatalogColumnStatisticsDataBase catalogColumnStatisticsDataBase = + columnStatistics.getColumnStatisticsData().get(field.name()); + if (catalogColumnStatisticsDataBase == null) { + continue; + } + tableColumnStatsMap.put( + field.name(), getPaimonColStats(field, catalogColumnStatisticsDataBase)); + } + statistics.colStats().putAll(tableColumnStatsMap); + return statistics; + } + + /** Convert Flink ColumnStats to Paimon ColStats according to Paimon column type. */ + private static ColStats getPaimonColStats( + DataField field, CatalogColumnStatisticsDataBase colStat) { + DataTypeRoot typeRoot = field.type().getTypeRoot(); + if (colStat instanceof CatalogColumnStatisticsDataString) { + CatalogColumnStatisticsDataString stringColStat = + (CatalogColumnStatisticsDataString) colStat; + if (typeRoot.equals(DataTypeRoot.CHAR) || typeRoot.equals(DataTypeRoot.VARCHAR)) { + return ColStats.newColStats( + field.id(), + null != stringColStat.getNdv() ? stringColStat.getNdv() : null, + null, + null, + null != stringColStat.getNullCount() ? stringColStat.getNullCount() : null, + null != stringColStat.getAvgLength() + ? stringColStat.getAvgLength().longValue() + : null, + null != stringColStat.getMaxLength() ? stringColStat.getMaxLength() : null); + } + } else if (colStat instanceof CatalogColumnStatisticsDataBoolean) { + CatalogColumnStatisticsDataBoolean booleanColStat = + (CatalogColumnStatisticsDataBoolean) colStat; + if (typeRoot.equals(DataTypeRoot.BOOLEAN)) { + return ColStats.newColStats( + field.id(), + (booleanColStat.getFalseCount() > 0 ? 1L : 0) + + (booleanColStat.getTrueCount() > 0 ? 1L : 0), + null, + null, + booleanColStat.getNullCount(), + null, + null); + } + } else if (colStat instanceof CatalogColumnStatisticsDataLong) { + CatalogColumnStatisticsDataLong longColStat = (CatalogColumnStatisticsDataLong) colStat; + if (typeRoot.equals(DataTypeRoot.INTEGER)) { + return ColStats.newColStats( + field.id(), + null != longColStat.getNdv() ? longColStat.getNdv() : null, + null != longColStat.getMin() ? longColStat.getMin().intValue() : null, + null != longColStat.getMax() ? longColStat.getMax().intValue() : null, + null != longColStat.getNullCount() ? longColStat.getNullCount() : null, + null, + null); + } else if (typeRoot.equals(DataTypeRoot.TINYINT)) { + return ColStats.newColStats( + field.id(), + null != longColStat.getNdv() ? longColStat.getNdv() : null, + null != longColStat.getMin() ? longColStat.getMin().byteValue() : null, + null != longColStat.getMax() ? longColStat.getMax().byteValue() : null, + null != longColStat.getNullCount() ? longColStat.getNullCount() : null, + null, + null); + + } else if (typeRoot.equals(DataTypeRoot.SMALLINT)) { + return ColStats.newColStats( + field.id(), + null != longColStat.getNdv() ? longColStat.getNdv() : null, + null != longColStat.getMin() ? longColStat.getMin().shortValue() : null, + null != longColStat.getMax() ? longColStat.getMax().shortValue() : null, + null != longColStat.getNullCount() ? longColStat.getNullCount() : null, + null, + null); + } else if (typeRoot.equals(DataTypeRoot.BIGINT)) { + return ColStats.newColStats( + field.id(), + null != longColStat.getNdv() ? longColStat.getNdv() : null, + null != longColStat.getMin() ? longColStat.getMin() : null, + null != longColStat.getMax() ? longColStat.getMax() : null, + null != longColStat.getNullCount() ? longColStat.getNullCount() : null, + null, + null); + } else if (typeRoot.equals(DataTypeRoot.TIMESTAMP_WITH_LOCAL_TIME_ZONE)) { + return ColStats.newColStats( + field.id(), + null != longColStat.getNdv() ? longColStat.getNdv() : null, + null != longColStat.getMin() + ? org.apache.paimon.data.Timestamp.fromSQLTimestamp( + new Timestamp(longColStat.getMin())) + : null, + null != longColStat.getMax() + ? org.apache.paimon.data.Timestamp.fromSQLTimestamp( + new Timestamp(longColStat.getMax())) + : null, + null != longColStat.getNullCount() ? longColStat.getNullCount() : null, + null, + null); + } + } else if (colStat instanceof CatalogColumnStatisticsDataDouble) { + CatalogColumnStatisticsDataDouble doubleColumnStatsData = + (CatalogColumnStatisticsDataDouble) colStat; + if (typeRoot.equals(DataTypeRoot.FLOAT)) { + return ColStats.newColStats( + field.id(), + null != doubleColumnStatsData.getNdv() + ? doubleColumnStatsData.getNdv() + : null, + null != doubleColumnStatsData.getMin() + ? doubleColumnStatsData.getMin().floatValue() + : null, + null != doubleColumnStatsData.getMax() + ? doubleColumnStatsData.getMax().floatValue() + : null, + null != doubleColumnStatsData.getNullCount() + ? doubleColumnStatsData.getNullCount() + : null, + null, + null); + } else if (typeRoot.equals(DataTypeRoot.DOUBLE)) { + return ColStats.newColStats( + field.id(), + null != doubleColumnStatsData.getNdv() + ? doubleColumnStatsData.getNdv() + : null, + null != doubleColumnStatsData.getMin() + ? doubleColumnStatsData.getMin() + : null, + null != doubleColumnStatsData.getMax() + ? doubleColumnStatsData.getMax() + : null, + null != doubleColumnStatsData.getNullCount() + ? doubleColumnStatsData.getNullCount() + : null, + null, + null); + } else if (typeRoot.equals(DataTypeRoot.DECIMAL)) { + BigDecimal max = BigDecimal.valueOf(doubleColumnStatsData.getMax()); + BigDecimal min = BigDecimal.valueOf(doubleColumnStatsData.getMin()); + return ColStats.newColStats( + field.id(), + null != doubleColumnStatsData.getNdv() + ? doubleColumnStatsData.getNdv() + : null, + null != doubleColumnStatsData.getMin() + ? Decimal.fromBigDecimal(min, min.precision(), min.scale()) + : null, + null != doubleColumnStatsData.getMax() + ? Decimal.fromBigDecimal(max, max.precision(), max.scale()) + : null, + null != doubleColumnStatsData.getNullCount() + ? doubleColumnStatsData.getNullCount() + : null, + null, + null); + } + } else if (colStat instanceof CatalogColumnStatisticsDataDate) { + CatalogColumnStatisticsDataDate dateColumnStatsData = + (CatalogColumnStatisticsDataDate) colStat; + if (typeRoot.equals(DataTypeRoot.DATE)) { + return ColStats.newColStats( + field.id(), + null != dateColumnStatsData.getNdv() ? dateColumnStatsData.getNdv() : null, + null != dateColumnStatsData.getMin() + ? new Long(dateColumnStatsData.getMin().getDaysSinceEpoch()) + .intValue() + : null, + null != dateColumnStatsData.getMax() + ? new Long(dateColumnStatsData.getMax().getDaysSinceEpoch()) + .intValue() + : null, + null != dateColumnStatsData.getNullCount() + ? dateColumnStatsData.getNullCount() + : null, + null, + null); + } + } else if (colStat instanceof CatalogColumnStatisticsDataBinary) { + CatalogColumnStatisticsDataBinary binaryColumnStatsData = + (CatalogColumnStatisticsDataBinary) colStat; + if (typeRoot.equals(DataTypeRoot.VARBINARY) || typeRoot.equals(DataTypeRoot.BINARY)) { + return ColStats.newColStats( + field.id(), + null, + null, + null, + null != binaryColumnStatsData.getNullCount() + ? binaryColumnStatsData.getNullCount() + : null, + null != binaryColumnStatsData.getAvgLength() + ? binaryColumnStatsData.getAvgLength().longValue() + : null, + null != binaryColumnStatsData.getMaxLength() + ? binaryColumnStatsData.getMaxLength() + : null); + } + } + throw new CatalogException( + String.format( + "Flink does not support convert ColumnStats '%s' for Paimon column " + + "type '%s' yet", + colStat, field.type())); + } +} diff --git a/paimon-flink/paimon-flink-common/src/main/resources/META-INF/services/org.apache.paimon.factories.Factory b/paimon-flink/paimon-flink-common/src/main/resources/META-INF/services/org.apache.paimon.factories.Factory index 3ae35ade54bb..6251189560f6 100644 --- a/paimon-flink/paimon-flink-common/src/main/resources/META-INF/services/org.apache.paimon.factories.Factory +++ b/paimon-flink/paimon-flink-common/src/main/resources/META-INF/services/org.apache.paimon.factories.Factory @@ -21,10 +21,13 @@ org.apache.paimon.flink.action.DropPartitionActionFactory org.apache.paimon.flink.action.DeleteActionFactory org.apache.paimon.flink.action.MergeIntoActionFactory org.apache.paimon.flink.action.RollbackToActionFactory +org.apache.paimon.flink.action.RollbackToTimestampActionFactory org.apache.paimon.flink.action.CreateTagActionFactory org.apache.paimon.flink.action.CreateTagFromTimestampActionFactory org.apache.paimon.flink.action.CreateTagFromWatermarkActionFactory org.apache.paimon.flink.action.DeleteTagActionFactory +org.apache.paimon.flink.action.ExpireTagsActionFactory +org.apache.paimon.flink.action.ReplaceTagActionFactory org.apache.paimon.flink.action.ResetConsumerActionFactory org.apache.paimon.flink.action.MigrateTableActionFactory org.apache.paimon.flink.action.MigrateFileActionFactory @@ -49,12 +52,16 @@ org.apache.paimon.flink.procedure.CreateTagProcedure org.apache.paimon.flink.procedure.CreateTagFromTimestampProcedure org.apache.paimon.flink.procedure.CreateTagFromWatermarkProcedure org.apache.paimon.flink.procedure.DeleteTagProcedure +org.apache.paimon.flink.procedure.ExpireTagsProcedure +org.apache.paimon.flink.procedure.ReplaceTagProcedure org.apache.paimon.flink.procedure.CreateBranchProcedure org.apache.paimon.flink.procedure.DeleteBranchProcedure org.apache.paimon.flink.procedure.DropPartitionProcedure org.apache.paimon.flink.procedure.MergeIntoProcedure org.apache.paimon.flink.procedure.ResetConsumerProcedure org.apache.paimon.flink.procedure.RollbackToProcedure +org.apache.paimon.flink.procedure.RollbackToTimestampProcedure +org.apache.paimon.flink.procedure.RollbackToWatermarkProcedure org.apache.paimon.flink.procedure.MigrateTableProcedure org.apache.paimon.flink.procedure.MigrateDatabaseProcedure org.apache.paimon.flink.procedure.MigrateFileProcedure @@ -62,6 +69,7 @@ org.apache.paimon.flink.procedure.RemoveOrphanFilesProcedure org.apache.paimon.flink.procedure.QueryServiceProcedure org.apache.paimon.flink.procedure.ExpireSnapshotsProcedure org.apache.paimon.flink.procedure.ExpirePartitionsProcedure +org.apache.paimon.flink.procedure.PurgeFilesProcedure org.apache.paimon.flink.procedure.privilege.InitFileBasedPrivilegeProcedure org.apache.paimon.flink.procedure.privilege.CreatePrivilegedUserProcedure org.apache.paimon.flink.procedure.privilege.DropPrivilegedUserProcedure @@ -72,3 +80,5 @@ org.apache.paimon.flink.procedure.RenameTagProcedure org.apache.paimon.flink.procedure.FastForwardProcedure org.apache.paimon.flink.procedure.MarkPartitionDoneProcedure org.apache.paimon.flink.procedure.CloneProcedure +org.apache.paimon.flink.procedure.CompactManifestProcedure +org.apache.paimon.flink.procedure.RefreshObjectTableProcedure diff --git a/paimon-flink/paimon-flink-common/src/main/resources/META-INF/services/org.apache.paimon.format.FileFormatFactory b/paimon-flink/paimon-flink-common/src/main/resources/META-INF/services/org.apache.paimon.format.FileFormatFactory new file mode 100644 index 000000000000..6e7553d5c668 --- /dev/null +++ b/paimon-flink/paimon-flink-common/src/main/resources/META-INF/services/org.apache.paimon.format.FileFormatFactory @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +org.apache.paimon.flink.compact.changelog.format.CompactedChangelogReadOnlyFormat$OrcFactory +org.apache.paimon.flink.compact.changelog.format.CompactedChangelogReadOnlyFormat$ParquetFactory +org.apache.paimon.flink.compact.changelog.format.CompactedChangelogReadOnlyFormat$AvroFactory diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/AbstractFlinkTableFactoryTest.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/AbstractFlinkTableFactoryTest.java index aeb46da8d1b7..38d48fa21d2d 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/AbstractFlinkTableFactoryTest.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/AbstractFlinkTableFactoryTest.java @@ -63,8 +63,7 @@ public void testSchemaEquals() { @Test public void testGetDynamicOptions() { Configuration configuration = new Configuration(); - configuration.setString("paimon.catalog1.db.T.k1", "v1"); - configuration.setString("paimon.*.db.*.k2", "v2"); + configuration.setString("k1", "v2"); ObjectIdentifier identifier = ObjectIdentifier.of("catalog1", "db", "T"); DynamicTableFactory.Context context = new FactoryUtil.DefaultDynamicTableContext( @@ -74,9 +73,25 @@ public void testGetDynamicOptions() { configuration, AbstractFlinkTableFactoryTest.class.getClassLoader(), false); - Map options = - AbstractFlinkTableFactory.getDynamicTableConfigOptions(context); - assertThat(options).isEqualTo(ImmutableMap.of("k1", "v1", "k2", "v2")); + Map options = AbstractFlinkTableFactory.getDynamicConfigOptions(context); + assertThat(options).isEqualTo(ImmutableMap.of("k1", "v2")); + + configuration = new Configuration(); + configuration.setString("k1", "v2"); + configuration.setString("k3", "v3"); + configuration.setString("paimon.catalog1.db.T.k1", "v1"); + configuration.setString("paimon.*.db.*.k2", "v2"); + identifier = ObjectIdentifier.of("catalog1", "db", "T"); + context = + new FactoryUtil.DefaultDynamicTableContext( + identifier, + null, + new HashMap<>(), + configuration, + AbstractFlinkTableFactoryTest.class.getClassLoader(), + false); + options = AbstractFlinkTableFactory.getDynamicConfigOptions(context); + assertThat(options).isEqualTo(ImmutableMap.of("k1", "v1", "k2", "v2", "k3", "v3")); } private void innerTest(RowType r1, RowType r2, boolean expectEquals) { diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/AppendOnlyTableITCase.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/AppendOnlyTableITCase.java index 0c9f4ec6c5e9..e29f8ab56ad7 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/AppendOnlyTableITCase.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/AppendOnlyTableITCase.java @@ -20,6 +20,7 @@ import org.apache.paimon.CoreOptions; import org.apache.paimon.Snapshot; +import org.apache.paimon.catalog.Catalog; import org.apache.paimon.utils.BlockingIterator; import org.apache.flink.table.planner.factories.TestValuesTableFactory; @@ -305,6 +306,22 @@ public void testReadWriteBranch() throws Exception { assertThat(rows).containsExactlyInAnyOrder(Row.of(2), Row.of(1)); } + @Test + public void testBranchNotExist() throws Exception { + // create table + sql("CREATE TABLE T (id INT)"); + // insert data + batchSql("INSERT INTO T VALUES (1)"); + // create tag + paimonTable("T").createTag("tag1", 1); + // create branch + paimonTable("T").createBranch("branch1", "tag1"); + // call the FileSystemCatalog.getDataTableSchema() function + assertThatThrownBy(() -> paimonTable("T$branch_branch2")) + .isInstanceOf(Catalog.TableNotExistException.class) + .hasMessage("Table %s does not exist.", "default.T$branch_branch2"); + } + @Override protected List ddl() { return Arrays.asList( diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/BatchFileStoreITCase.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/BatchFileStoreITCase.java index f03a1636850f..d48b6e771236 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/BatchFileStoreITCase.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/BatchFileStoreITCase.java @@ -23,6 +23,7 @@ import org.apache.paimon.table.FileStoreTable; import org.apache.paimon.utils.BlockingIterator; import org.apache.paimon.utils.DateTimeUtils; +import org.apache.paimon.utils.SnapshotNotExistException; import org.apache.flink.api.dag.Transformation; import org.apache.flink.types.Row; @@ -33,6 +34,7 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; +import java.math.BigDecimal; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -111,8 +113,8 @@ public void testTimeTravelRead() throws Exception { assertThatThrownBy(() -> batchSql("SELECT * FROM T /*+ OPTIONS('scan.snapshot-id'='0') */")) .satisfies( anyCauseMatches( - IllegalArgumentException.class, - "The specified scan snapshotId 0 is out of available snapshotId range [1, 4].")); + SnapshotNotExistException.class, + "Specified parameter scan.snapshot-id = 0 is not exist, you can set it in range from 1 to 4.")); assertThatThrownBy( () -> @@ -120,8 +122,8 @@ public void testTimeTravelRead() throws Exception { "SELECT * FROM T /*+ OPTIONS('scan.mode'='from-snapshot-full','scan.snapshot-id'='0') */")) .satisfies( anyCauseMatches( - IllegalArgumentException.class, - "The specified scan snapshotId 0 is out of available snapshotId range [1, 4].")); + SnapshotNotExistException.class, + "Specified parameter scan.snapshot-id = 0 is not exist, you can set it in range from 1 to 4.")); assertThat( batchSql( @@ -559,19 +561,53 @@ public void testCountStarAppendWithDv() { String sql = "SELECT COUNT(*) FROM count_append_dv"; assertThat(sql(sql)).containsOnly(Row.of(2L)); - validateCount1NotPushDown(sql); + validateCount1PushDown(sql); } @Test public void testCountStarPK() { - sql("CREATE TABLE count_pk (f0 INT PRIMARY KEY NOT ENFORCED, f1 STRING)"); - sql("INSERT INTO count_pk VALUES (1, 'a'), (2, 'b')"); + sql( + "CREATE TABLE count_pk (f0 INT PRIMARY KEY NOT ENFORCED, f1 STRING) WITH ('file.format' = 'avro')"); + sql("INSERT INTO count_pk VALUES (1, 'a'), (2, 'b'), (3, 'c'), (4, 'd')"); + sql("INSERT INTO count_pk VALUES (1, 'e')"); String sql = "SELECT COUNT(*) FROM count_pk"; - assertThat(sql(sql)).containsOnly(Row.of(2L)); + assertThat(sql(sql)).containsOnly(Row.of(4L)); validateCount1NotPushDown(sql); } + @Test + public void testCountStarPKDv() { + sql( + "CREATE TABLE count_pk_dv (f0 INT PRIMARY KEY NOT ENFORCED, f1 STRING) WITH (" + + "'file.format' = 'avro', " + + "'deletion-vectors.enabled' = 'true')"); + sql("INSERT INTO count_pk_dv VALUES (1, 'a'), (2, 'b'), (3, 'c'), (4, 'd')"); + sql("INSERT INTO count_pk_dv VALUES (1, 'e')"); + + String sql = "SELECT COUNT(*) FROM count_pk_dv"; + assertThat(sql(sql)).containsOnly(Row.of(4L)); + validateCount1PushDown(sql); + } + + @Test + public void testParquetRowDecimalAndTimestamp() { + sql( + "CREATE TABLE parquet_row_decimal(`row` ROW) WITH ('file.format' = 'parquet')"); + sql("INSERT INTO parquet_row_decimal VALUES ( (ROW(1.2)) )"); + + assertThat(sql("SELECT * FROM parquet_row_decimal")) + .containsExactly(Row.of(Row.of(new BigDecimal("1.2")))); + + sql( + "CREATE TABLE parquet_row_timestamp(`row` ROW) WITH ('file.format' = 'parquet')"); + sql("INSERT INTO parquet_row_timestamp VALUES ( (ROW(TIMESTAMP'2024-11-13 18:00:00')) )"); + + assertThat(sql("SELECT * FROM parquet_row_timestamp")) + .containsExactly( + Row.of(Row.of(DateTimeUtils.toLocalDateTime("2024-11-13 18:00:00", 0)))); + } + private void validateCount1PushDown(String sql) { Transformation transformation = AbstractTestBase.translate(tEnv, sql); while (!transformation.getInputs().isEmpty()) { diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/BranchSqlITCase.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/BranchSqlITCase.java index 6970eb043b25..2566fbe92e4c 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/BranchSqlITCase.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/BranchSqlITCase.java @@ -18,6 +18,7 @@ package org.apache.paimon.flink; +import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.table.FileStoreTable; import org.apache.paimon.utils.BlockingIterator; import org.apache.paimon.utils.SnapshotManager; @@ -26,6 +27,7 @@ import org.apache.flink.types.Row; import org.apache.flink.util.CloseableIterator; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; import java.io.IOException; import java.util.ArrayList; @@ -194,44 +196,13 @@ public void testDeleteBranchTable() throws Exception { sql("CALL sys.create_branch('default.T', 'test', 'tag1')"); sql("CALL sys.create_branch('default.T', 'test2', 'tag2')"); - assertThat(collectResult("SELECT branch_name, created_from_snapshot FROM `T$branches`")) - .containsExactlyInAnyOrder("+I[test, 1]", "+I[test2, 2]"); + assertThat(collectResult("SELECT branch_name FROM `T$branches`")) + .containsExactlyInAnyOrder("+I[test]", "+I[test2]"); sql("CALL sys.delete_branch('default.T', 'test')"); - assertThat(collectResult("SELECT branch_name, created_from_snapshot FROM `T$branches`")) - .containsExactlyInAnyOrder("+I[test2, 2]"); - } - - @Test - public void testBranchManagerGetBranchSnapshotsList() throws Exception { - sql( - "CREATE TABLE T (" - + " pt INT" - + ", k INT" - + ", v STRING" - + ", PRIMARY KEY (pt, k) NOT ENFORCED" - + " ) PARTITIONED BY (pt) WITH (" - + " 'bucket' = '2'" - + " )"); - - sql("INSERT INTO T VALUES (1, 10, 'hxh')"); - sql("INSERT INTO T VALUES (1, 20, 'hxh')"); - sql("INSERT INTO T VALUES (1, 30, 'hxh')"); - - FileStoreTable table = paimonTable("T"); - checkSnapshots(table.snapshotManager(), 1, 3); - - sql("CALL sys.create_tag('default.T', 'tag1', 1)"); - sql("CALL sys.create_tag('default.T', 'tag2', 2)"); - sql("CALL sys.create_tag('default.T', 'tag3', 3)"); - - sql("CALL sys.create_branch('default.T', 'test1', 'tag1')"); - sql("CALL sys.create_branch('default.T', 'test2', 'tag2')"); - sql("CALL sys.create_branch('default.T', 'test3', 'tag3')"); - - assertThat(collectResult("SELECT created_from_snapshot FROM `T$branches`")) - .containsExactlyInAnyOrder("+I[1]", "+I[2]", "+I[3]"); + assertThat(collectResult("SELECT branch_name FROM `T$branches`")) + .containsExactlyInAnyOrder("+I[test2]"); } @Test @@ -377,6 +348,11 @@ public void testBranchOptionsTable() throws Exception { "+I[bucket, 2]", "+I[snapshot.time-retained, 1 h]", "+I[scan.infer-parallelism, false]"); + assertThat(collectResult("SELECT * FROM t$options /*+ OPTIONS('branch'='test') */")) + .containsExactlyInAnyOrder( + "+I[bucket, 2]", + "+I[snapshot.time-retained, 1 h]", + "+I[scan.infer-parallelism, false]"); } @Test @@ -390,6 +366,10 @@ public void testBranchSchemasTable() throws Exception { sql("ALTER TABLE t$branch_b1 SET ('snapshot.time-retained' = '5 h')"); assertThat(collectResult("SELECT schema_id FROM t$branch_b1$schemas order by schema_id")) .containsExactlyInAnyOrder("+I[0]", "+I[1]"); + assertThat( + collectResult( + "SELECT schema_id FROM t$schemas /*+ OPTIONS('branch'='b1') */ order by schema_id")) + .containsExactlyInAnyOrder("+I[0]", "+I[1]"); } @Test @@ -403,6 +383,8 @@ public void testBranchAuditLogTable() throws Exception { sql("INSERT INTO t$branch_b1 VALUES (3, 4)"); assertThat(collectResult("SELECT * FROM t$branch_b1$audit_log")) .containsExactlyInAnyOrder("+I[+I, 3, 4]"); + assertThat(collectResult("SELECT * FROM t$audit_log /*+ OPTIONS('branch'='b1') */")) + .containsExactlyInAnyOrder("+I[+I, 3, 4]"); } @Test @@ -415,6 +397,8 @@ public void testBranchReadOptimizedTable() throws Exception { sql("INSERT INTO t$branch_b1 VALUES (3, 4)"); assertThat(collectResult("SELECT * FROM t$branch_b1$ro")) .containsExactlyInAnyOrder("+I[3, 4]"); + assertThat(collectResult("SELECT * FROM t$ro /*+ OPTIONS('branch'='b1') */")) + .containsExactlyInAnyOrder("+I[3, 4]"); } @Test @@ -430,6 +414,10 @@ public void testBranchFilesTable() throws Exception { .containsExactlyInAnyOrder("+I[{a=1, b=2}]"); assertThat(collectResult("SELECT min_value_stats FROM t$branch_b1$files")) .containsExactlyInAnyOrder("+I[{a=3, b=4}]", "+I[{a=5, b=6}]"); + assertThat( + collectResult( + "SELECT min_value_stats FROM t$files /*+ OPTIONS('branch'='b1') */")) + .containsExactlyInAnyOrder("+I[{a=3, b=4}]", "+I[{a=5, b=6}]"); } @Test @@ -446,9 +434,14 @@ public void testBranchTagsTable() throws Exception { .containsExactlyInAnyOrder("+I[tag1, 1, 1]"); assertThat(collectResult("SELECT tag_name,snapshot_id,record_count FROM t$branch_b1$tags")) .containsExactlyInAnyOrder("+I[tag1, 1, 1]", "+I[tag2, 2, 2]"); + assertThat( + collectResult( + "SELECT tag_name,snapshot_id,record_count FROM t$tags /*+ OPTIONS('branch'='b1') */")) + .containsExactlyInAnyOrder("+I[tag1, 1, 1]", "+I[tag2, 2, 2]"); } @Test + @Timeout(60) public void testBranchConsumersTable() throws Exception { sql("CREATE TABLE t (a INT, b INT)"); sql("INSERT INTO t VALUES (1, 2), (3,4)"); @@ -460,10 +453,19 @@ public void testBranchConsumersTable() throws Exception { "SELECT * FROM t$branch_b1 /*+ OPTIONS('consumer-id'='id1','consumer.expiration-time'='3h') */")); sql("INSERT INTO t$branch_b1 VALUES (5, 6), (7, 8)"); assertThat(iterator.collect(2)).containsExactlyInAnyOrder(Row.of(5, 6), Row.of(7, 8)); + List branchResult; + do { + branchResult = collectResult("SELECT * FROM t$branch_b1$consumers"); + if (!branchResult.isEmpty()) { + break; + } + Thread.sleep(1000); + } while (true); iterator.close(); assertThat(collectResult("SELECT * FROM t$consumers")).isEmpty(); - assertThat(collectResult("SELECT * FROM t$branch_b1$consumers")) + assertThat(branchResult).containsExactlyInAnyOrder("+I[id1, 2]"); + assertThat(collectResult("SELECT * FROM t$consumers /*+ OPTIONS('branch'='b1') */")) .containsExactlyInAnyOrder("+I[id1, 2]"); } @@ -488,6 +490,31 @@ public void testBranchManifestsTable() { .isTrue(); assertThat((long) row.getField(2)).isGreaterThan(0L); }); + List dynamicOptionRes = + sql( + "SELECT schema_id, file_name, file_size FROM t$manifests /*+ OPTIONS('branch'='b1') */"); + assertThat(dynamicOptionRes).containsExactlyInAnyOrderElementsOf(res); + } + + @Test + public void testBranchSnapshotsTable() throws Exception { + sql("CREATE TABLE t (a INT, b INT)"); + sql("INSERT INTO t VALUES (1, 2)"); + + sql("CALL sys.create_branch('default.t', 'b1')"); + sql("INSERT INTO t$branch_b1 VALUES (3, 4)"); + sql("INSERT INTO t$branch_b1 VALUES (5, 6)"); + + assertThat(collectResult("SELECT snapshot_id, schema_id, commit_kind FROM t$snapshots")) + .containsExactlyInAnyOrder("+I[1, 0, APPEND]"); + assertThat( + collectResult( + "SELECT snapshot_id, schema_id, commit_kind FROM t$branch_b1$snapshots")) + .containsExactlyInAnyOrder("+I[1, 0, APPEND]", "+I[2, 0, APPEND]"); + assertThat( + collectResult( + "SELECT snapshot_id, schema_id, commit_kind FROM t$snapshots /*+ OPTIONS('branch'='b1') */")) + .containsExactlyInAnyOrder("+I[1, 0, APPEND]", "+I[2, 0, APPEND]"); } @Test @@ -509,6 +536,10 @@ public void testBranchPartitionsTable() throws Exception { collectResult( "SELECT `partition`, record_count, file_count FROM t$branch_b1$partitions")) .containsExactlyInAnyOrder("+I[[1], 2, 2]", "+I[[2], 3, 2]"); + assertThat( + collectResult( + "SELECT `partition`, record_count, file_count FROM t$partitions /*+ OPTIONS('branch'='b1') */")) + .containsExactlyInAnyOrder("+I[[1], 2, 2]", "+I[[2], 3, 2]"); } @Test @@ -529,6 +560,43 @@ public void testCannotSetEmptyFallbackBranch() { .satisfies(anyCauseMatches(IllegalArgumentException.class, errMsg)); } + @Test + public void testReadBranchTableWithMultiSchemaIds() throws Exception { + sql( + "CREATE TABLE T (" + + " pt INT" + + ", k INT" + + ", v STRING" + + ", PRIMARY KEY (pt, k) NOT ENFORCED" + + " ) PARTITIONED BY (pt) WITH (" + + " 'bucket' = '2'" + + " )"); + + sql("INSERT INTO T VALUES" + " (1, 10, 'apple')," + " (1, 20, 'banana')"); + + sql("ALTER TABLE `T` ADD (v2 INT)"); + + sql("INSERT INTO T VALUES" + " (2, 10, 'cat', 2)," + " (2, 20, 'dog', 2)"); + + sql("ALTER TABLE `T` ADD (v3 INT)"); + + sql("CALL sys.create_tag('default.T', 'tag1', 2)"); + + sql("CALL sys.create_branch('default.T', 'test', 'tag1')"); + + FileStoreTable table = paimonTable("T"); + SchemaManager schemaManager = new SchemaManager(table.fileIO(), table.location(), "test"); + List schemaIds = schemaManager.listAllIds(); + assertThat(schemaIds.size()).isEqualTo(2); + + assertThat(collectResult("SELECT * FROM T$branch_test")) + .containsExactlyInAnyOrder( + "+I[1, 10, apple, null]", + "+I[1, 20, banana, null]", + "+I[2, 10, cat, 2]", + "+I[2, 20, dog, 2]"); + } + private List collectResult(String sql) throws Exception { List result = new ArrayList<>(); try (CloseableIterator it = tEnv.executeSql(sql).collect()) { diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/CatalogTableITCase.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/CatalogTableITCase.java index 975c6a49007f..10b03b7139ae 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/CatalogTableITCase.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/CatalogTableITCase.java @@ -21,8 +21,6 @@ import org.apache.paimon.catalog.Catalog; import org.apache.paimon.table.system.AllTableOptionsTable; import org.apache.paimon.table.system.CatalogOptionsTable; -import org.apache.paimon.table.system.SinkTableLineageTable; -import org.apache.paimon.table.system.SourceTableLineageTable; import org.apache.paimon.utils.BlockingIterator; import org.apache.commons.lang3.StringUtils; @@ -33,6 +31,7 @@ import org.apache.flink.table.catalog.exceptions.TableNotPartitionedException; import org.apache.flink.types.Row; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; import javax.annotation.Nonnull; @@ -42,10 +41,10 @@ import java.util.stream.Collectors; import static org.apache.flink.table.api.config.TableConfigOptions.TABLE_DML_SYNC; -import static org.apache.paimon.flink.FlinkCatalog.LAST_UPDATE_TIME_KEY; -import static org.apache.paimon.flink.FlinkCatalog.NUM_FILES_KEY; -import static org.apache.paimon.flink.FlinkCatalog.NUM_ROWS_KEY; -import static org.apache.paimon.flink.FlinkCatalog.TOTAL_SIZE_KEY; +import static org.apache.paimon.catalog.Catalog.LAST_UPDATE_TIME_PROP; +import static org.apache.paimon.catalog.Catalog.NUM_FILES_PROP; +import static org.apache.paimon.catalog.Catalog.NUM_ROWS_PROP; +import static org.apache.paimon.catalog.Catalog.TOTAL_SIZE_PROP; import static org.apache.paimon.testutils.assertj.PaimonAssertions.anyCauseMatches; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatCode; @@ -70,14 +69,23 @@ public void testSnapshotsTable() throws Exception { sql("CREATE TABLE T (a INT, b INT)"); sql("INSERT INTO T VALUES (1, 2)"); sql("INSERT INTO T VALUES (3, 4)"); + sql("INSERT INTO T VALUES (5, 6)"); List result = sql("SELECT snapshot_id, schema_id, commit_kind FROM T$snapshots"); - assertThat(result).containsExactly(Row.of(1L, 0L, "APPEND"), Row.of(2L, 0L, "APPEND")); + assertThat(result) + .containsExactly( + Row.of(1L, 0L, "APPEND"), + Row.of(2L, 0L, "APPEND"), + Row.of(3L, 0L, "APPEND")); result = sql( "SELECT snapshot_id, schema_id, commit_kind FROM T$snapshots WHERE schema_id = 0"); - assertThat(result).containsExactly(Row.of(1L, 0L, "APPEND"), Row.of(2L, 0L, "APPEND")); + assertThat(result) + .containsExactly( + Row.of(1L, 0L, "APPEND"), + Row.of(2L, 0L, "APPEND"), + Row.of(3L, 0L, "APPEND")); result = sql( @@ -87,7 +95,7 @@ public void testSnapshotsTable() throws Exception { result = sql( "SELECT snapshot_id, schema_id, commit_kind FROM T$snapshots WHERE snapshot_id > 1"); - assertThat(result).containsExactly(Row.of(2L, 0L, "APPEND")); + assertThat(result).containsExactly(Row.of(2L, 0L, "APPEND"), Row.of(3L, 0L, "APPEND")); result = sql( @@ -97,12 +105,30 @@ public void testSnapshotsTable() throws Exception { result = sql( "SELECT snapshot_id, schema_id, commit_kind FROM T$snapshots WHERE snapshot_id >= 1"); - assertThat(result).contains(Row.of(1L, 0L, "APPEND"), Row.of(2L, 0L, "APPEND")); + assertThat(result) + .contains( + Row.of(1L, 0L, "APPEND"), + Row.of(2L, 0L, "APPEND"), + Row.of(3L, 0L, "APPEND")); result = sql( "SELECT snapshot_id, schema_id, commit_kind FROM T$snapshots WHERE snapshot_id <= 2"); assertThat(result).contains(Row.of(1L, 0L, "APPEND"), Row.of(2L, 0L, "APPEND")); + + result = + sql( + "SELECT snapshot_id, schema_id, commit_kind FROM T$snapshots WHERE snapshot_id in (1, 2)"); + assertThat(result).contains(Row.of(1L, 0L, "APPEND"), Row.of(2L, 0L, "APPEND")); + + result = + sql( + "SELECT snapshot_id, schema_id, commit_kind FROM T$snapshots WHERE snapshot_id in (1, 2) or schema_id=0"); + assertThat(result) + .contains( + Row.of(1L, 0L, "APPEND"), + Row.of(2L, 0L, "APPEND"), + Row.of(3L, 0L, "APPEND")); } @Test @@ -162,7 +188,8 @@ public void testCreateSystemDatabase() { public void testChangeTableInSystemDatabase() { sql("USE sys"); assertThatCode(() -> sql("ALTER TABLE all_table_options SET ('bucket-num' = '5')")) - .hasRootCauseMessage("Can't alter system table."); + .rootCause() + .hasMessageContaining("Only support alter data table, but is: "); } @Test @@ -171,9 +198,7 @@ public void testSystemDatabase() { assertThat(sql("SHOW TABLES")) .containsExactlyInAnyOrder( Row.of(AllTableOptionsTable.ALL_TABLE_OPTIONS), - Row.of(CatalogOptionsTable.CATALOG_OPTIONS), - Row.of(SourceTableLineageTable.SOURCE_TABLE_LINEAGE), - Row.of(SinkTableLineageTable.SINK_TABLE_LINEAGE)); + Row.of(CatalogOptionsTable.CATALOG_OPTIONS)); } @Test @@ -222,17 +247,20 @@ public void testSchemasTable() { sql("ALTER TABLE T SET ('snapshot.num-retained.min' = '18')"); sql("ALTER TABLE T SET ('manifest.format' = 'avro')"); - assertThat(sql("SHOW CREATE TABLE T$schemas").toString()) - .isEqualTo( - "[+I[CREATE TABLE `PAIMON`.`default`.`T$schemas` (\n" - + " `schema_id` BIGINT NOT NULL,\n" - + " `fields` VARCHAR(2147483647) NOT NULL,\n" - + " `partition_keys` VARCHAR(2147483647) NOT NULL,\n" - + " `primary_keys` VARCHAR(2147483647) NOT NULL,\n" - + " `options` VARCHAR(2147483647) NOT NULL,\n" - + " `comment` VARCHAR(2147483647),\n" - + " `update_time` TIMESTAMP(3) NOT NULL\n" - + ") ]]"); + String actualResult = sql("SHOW CREATE TABLE T$schemas").toString(); + String expectedResult = + "[+I[CREATE TABLE `PAIMON`.`default`.`T$schemas` (\n" + + " `schema_id` BIGINT NOT NULL,\n" + + " `fields` VARCHAR(2147483647) NOT NULL,\n" + + " `partition_keys` VARCHAR(2147483647) NOT NULL,\n" + + " `primary_keys` VARCHAR(2147483647) NOT NULL,\n" + + " `options` VARCHAR(2147483647) NOT NULL,\n" + + " `comment` VARCHAR(2147483647),\n" + + " `update_time` TIMESTAMP(3) NOT NULL\n" + + ") ]]"; + actualResult = actualResult.replace(" ", "").replace("\n", ""); + expectedResult = expectedResult.replace(" ", "").replace("\n", ""); + assertThat(actualResult).isEqualTo(expectedResult); List result = sql( @@ -271,7 +299,7 @@ public void testSchemasTable() { result = sql( "SELECT schema_id, fields, partition_keys, " - + "primary_keys, options, `comment` FROM T$schemas where schema_id>0 and schema_id<3"); + + "primary_keys, options, `comment` FROM T$schemas where schema_id>0 and schema_id<3 order by schema_id"); assertThat(result.toString()) .isEqualTo( "[+I[1, [{\"id\":0,\"name\":\"a\",\"type\":\"INT NOT NULL\"}," @@ -281,6 +309,42 @@ public void testSchemasTable() { + "{\"id\":2,\"name\":\"c\",\"type\":\"STRING\"}], [], [\"a\"], {\"a.aa.aaa\":\"val1\",\"snapshot.time-retained\":\"5 h\"," + "\"b.bb.bbb\":\"val2\",\"snapshot.num-retained.max\":\"20\"}, ]]"); + // test for IN filter + result = + sql( + "SELECT schema_id, fields, partition_keys, " + + "primary_keys, options, `comment` FROM T$schemas where schema_id in (1, 3) order by schema_id"); + assertThat(result.toString()) + .isEqualTo( + "[+I[1, [{\"id\":0,\"name\":\"a\",\"type\":\"INT NOT NULL\"}," + + "{\"id\":1,\"name\":\"b\",\"type\":\"INT\"},{\"id\":2,\"name\":\"c\",\"type\":\"STRING\"}], [], [\"a\"], " + + "{\"a.aa.aaa\":\"val1\",\"snapshot.time-retained\":\"5 h\",\"b.bb.bbb\":\"val2\"}, ], " + + "+I[3, [{\"id\":0,\"name\":\"a\",\"type\":\"INT NOT NULL\"},{\"id\":1,\"name\":\"b\",\"type\":\"INT\"}," + + "{\"id\":2,\"name\":\"c\",\"type\":\"STRING\"}], [], [\"a\"], " + + "{\"a.aa.aaa\":\"val1\",\"snapshot.time-retained\":\"5 h\",\"b.bb.bbb\":\"val2\",\"snapshot.num-retained.max\":\"20\",\"snapshot.num-retained.min\":\"18\"}, ]]"); + + result = + sql( + "SELECT schema_id, fields, partition_keys, " + + "primary_keys, options, `comment` FROM T$schemas where schema_id in (1, 3) or fields='[{\"id\":0,\"name\":\"a\",\"type\":\"INT NOT NULL\"},{\"id\":1,\"name\":\"b\",\"type\":\"INT\"},{\"id\":2,\"name\":\"c\",\"type\":\"STRING\"}]' order by schema_id"); + assertThat(result.toString()) + .isEqualTo( + "[+I[0, [{\"id\":0,\"name\":\"a\",\"type\":\"INT NOT NULL\"}," + + "{\"id\":1,\"name\":\"b\",\"type\":\"INT\"},{\"id\":2,\"name\":\"c\",\"type\":\"STRING\"}], [], [\"a\"], " + + "{\"a.aa.aaa\":\"val1\",\"b.bb.bbb\":\"val2\"}, ], " + + "+I[1, [{\"id\":0,\"name\":\"a\",\"type\":\"INT NOT NULL\"}," + + "{\"id\":1,\"name\":\"b\",\"type\":\"INT\"},{\"id\":2,\"name\":\"c\",\"type\":\"STRING\"}], [], [\"a\"], " + + "{\"a.aa.aaa\":\"val1\",\"snapshot.time-retained\":\"5 h\",\"b.bb.bbb\":\"val2\"}, ], " + + "+I[2, [{\"id\":0,\"name\":\"a\",\"type\":\"INT NOT NULL\"}," + + "{\"id\":1,\"name\":\"b\",\"type\":\"INT\"},{\"id\":2,\"name\":\"c\",\"type\":\"STRING\"}], [], [\"a\"], " + + "{\"a.aa.aaa\":\"val1\",\"snapshot.time-retained\":\"5 h\",\"b.bb.bbb\":\"val2\",\"snapshot.num-retained.max\":\"20\"}, ], " + + "+I[3, [{\"id\":0,\"name\":\"a\",\"type\":\"INT NOT NULL\"},{\"id\":1,\"name\":\"b\",\"type\":\"INT\"}," + + "{\"id\":2,\"name\":\"c\",\"type\":\"STRING\"}], [], [\"a\"], " + + "{\"a.aa.aaa\":\"val1\",\"snapshot.time-retained\":\"5 h\",\"b.bb.bbb\":\"val2\",\"snapshot.num-retained.max\":\"20\",\"snapshot.num-retained.min\":\"18\"}, ], " + + "+I[4, [{\"id\":0,\"name\":\"a\",\"type\":\"INT NOT NULL\"},{\"id\":1,\"name\":\"b\",\"type\":\"INT\"},{\"id\":2,\"name\":\"c\",\"type\":\"STRING\"}], [], [\"a\"], " + + "{\"a.aa.aaa\":\"val1\",\"snapshot.time-retained\":\"5 h\",\"b.bb.bbb\":\"val2\",\"snapshot.num-retained.max\":\"20\",\"manifest.format\":\"avro\"," + + "\"snapshot.num-retained.min\":\"18\"}, ]]"); + // check with not exist schema id assertThatThrownBy( () -> @@ -554,10 +618,10 @@ void testPKTableGetPartition() throws Exception { assertThat(partitionPropertiesMap1) .allSatisfy( (par, properties) -> { - assertThat(properties.get(NUM_ROWS_KEY)).isEqualTo("2"); - assertThat(properties.get(LAST_UPDATE_TIME_KEY)).isNotBlank(); - assertThat(properties.get(NUM_FILES_KEY)).isEqualTo("1"); - assertThat(properties.get(TOTAL_SIZE_KEY)).isNotBlank(); + assertThat(properties.get(NUM_ROWS_PROP)).isEqualTo("2"); + assertThat(properties.get(LAST_UPDATE_TIME_PROP)).isNotBlank(); + assertThat(properties.get(NUM_FILES_PROP)).isEqualTo("1"); + assertThat(properties.get(TOTAL_SIZE_PROP)).isNotBlank(); }); // update p1 data sql("UPDATE PK_T SET word = 'c' WHERE id = 2"); @@ -589,8 +653,8 @@ void testNonPKTableGetPartition() throws Exception { assertThat(partitionPropertiesMap1) .allSatisfy( (par, properties) -> { - assertThat(properties.get(NUM_ROWS_KEY)).isEqualTo("1"); - assertThat(properties.get(LAST_UPDATE_TIME_KEY)).isNotBlank(); + assertThat(properties.get(NUM_ROWS_PROP)).isEqualTo("1"); + assertThat(properties.get(LAST_UPDATE_TIME_PROP)).isNotBlank(); }); // append data to p1 @@ -844,23 +908,39 @@ public void testTagsTable() throws Exception { sql("CREATE TABLE T (a INT, b INT)"); sql("INSERT INTO T VALUES (1, 2)"); sql("INSERT INTO T VALUES (3, 4)"); + sql("INSERT INTO T VALUES (5, 6)"); paimonTable("T").createTag("tag1", 1); paimonTable("T").createTag("tag2", 2); + paimonTable("T").createTag("tag3", 3); List result = sql( "SELECT tag_name, snapshot_id, schema_id, record_count FROM T$tags ORDER BY tag_name"); - - assertThat(result).containsExactly(Row.of("tag1", 1L, 0L, 1L), Row.of("tag2", 2L, 0L, 2L)); + assertThat(result) + .containsExactly( + Row.of("tag1", 1L, 0L, 1L), + Row.of("tag2", 2L, 0L, 2L), + Row.of("tag3", 3L, 0L, 3L)); result = sql( "SELECT tag_name, snapshot_id, schema_id, record_count FROM T$tags where tag_name = 'tag1' "); assertThat(result).containsExactly(Row.of("tag1", 1L, 0L, 1L)); + + result = + sql( + "SELECT tag_name, snapshot_id, schema_id, record_count FROM T$tags where tag_name in ('tag1', 'tag3')"); + assertThat(result).containsExactly(Row.of("tag1", 1L, 0L, 1L), Row.of("tag3", 3L, 0L, 3L)); + + result = + sql( + "SELECT tag_name, snapshot_id, schema_id, record_count FROM T$tags where tag_name in ('tag1') or snapshot_id=2"); + assertThat(result).containsExactly(Row.of("tag1", 1L, 0L, 1L), Row.of("tag2", 2L, 0L, 2L)); } @Test + @Timeout(60) public void testConsumersTable() throws Exception { batchSql("CREATE TABLE T (a INT, b INT)"); batchSql("INSERT INTO T VALUES (1, 2)"); @@ -873,9 +953,17 @@ public void testConsumersTable() throws Exception { batchSql("INSERT INTO T VALUES (5, 6), (7, 8)"); assertThat(iterator.collect(2)).containsExactlyInAnyOrder(Row.of(1, 2), Row.of(3, 4)); + + List result; + do { + result = sql("SELECT * FROM T$consumers"); + if (!result.isEmpty()) { + break; + } + Thread.sleep(1000); + } while (true); iterator.close(); - List result = sql("SELECT * FROM T$consumers"); assertThat(result).hasSize(1); assertThat(result.get(0).getField(0)).isEqualTo("my1"); assertThat((Long) result.get(0).getField(1)).isGreaterThanOrEqualTo(3); @@ -893,7 +981,8 @@ public void testConsumerIdExpInBatchMode() { "SELECT * FROM T /*+ OPTIONS('consumer-id' = 'test-id') */ WHERE a = 1")) .rootCause() .isInstanceOf(IllegalArgumentException.class) - .hasMessage("consumer.expiration-time should be specified when using consumer-id."); + .hasMessageContaining( + "You need to configure 'consumer.expiration-time' (ALTER TABLE) and restart your write job for it"); } @Test @@ -906,7 +995,8 @@ public void testConsumerIdExpInStreamingMode() { streamSqlIter( "SELECT * FROM T /*+ OPTIONS('consumer-id'='test-id') */")) .isInstanceOf(IllegalArgumentException.class) - .hasMessage("consumer.expiration-time should be specified when using consumer-id."); + .hasMessageContaining( + "You need to configure 'consumer.expiration-time' (ALTER TABLE) and restart your write job for it"); } @Test @@ -1063,13 +1153,13 @@ private static void assertPartitionUpdateTo( Long expectedNumFiles) { Map newPartitionProperties = newProperties.get(partition); Map oldPartitionProperties = oldProperties.get(partition); - assertThat(newPartitionProperties.get(NUM_ROWS_KEY)) + assertThat(newPartitionProperties.get(NUM_ROWS_PROP)) .isEqualTo(String.valueOf(expectedNumRows)); - assertThat(Long.valueOf(newPartitionProperties.get(LAST_UPDATE_TIME_KEY))) - .isGreaterThan(Long.valueOf(oldPartitionProperties.get(LAST_UPDATE_TIME_KEY))); - assertThat(newPartitionProperties.get(NUM_FILES_KEY)) + assertThat(Long.valueOf(newPartitionProperties.get(LAST_UPDATE_TIME_PROP))) + .isGreaterThan(Long.valueOf(oldPartitionProperties.get(LAST_UPDATE_TIME_PROP))); + assertThat(newPartitionProperties.get(NUM_FILES_PROP)) .isEqualTo(String.valueOf(expectedNumFiles)); - assertThat(Long.valueOf(newPartitionProperties.get(TOTAL_SIZE_KEY))) - .isGreaterThan(Long.valueOf(oldPartitionProperties.get(TOTAL_SIZE_KEY))); + assertThat(Long.valueOf(newPartitionProperties.get(TOTAL_SIZE_PROP))) + .isGreaterThan(Long.valueOf(oldPartitionProperties.get(TOTAL_SIZE_PROP))); } } diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/ContinuousFileStoreITCase.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/ContinuousFileStoreITCase.java index cf97f7b67d4d..b44885832804 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/ContinuousFileStoreITCase.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/ContinuousFileStoreITCase.java @@ -28,6 +28,7 @@ import org.apache.flink.table.api.StatementSet; import org.apache.flink.table.api.ValidationException; +import org.apache.flink.table.planner.factories.TestValuesTableFactory; import org.apache.flink.types.Row; import org.apache.flink.types.RowKind; import org.apache.flink.util.CloseableIterator; @@ -43,6 +44,7 @@ import java.util.Set; import java.util.stream.Collectors; +import static org.apache.paimon.testutils.assertj.PaimonAssertions.anyCauseMatches; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -118,7 +120,14 @@ public void testConsumerId() throws Exception { assertThat(iterator.collect(2)) .containsExactlyInAnyOrder(Row.of("1", "2", "3"), Row.of("4", "5", "6")); - Thread.sleep(1000); + List result; + do { + result = sql("SELECT * FROM %s$consumers", table); + if (!result.isEmpty()) { + break; + } + Thread.sleep(1000); + } while (true); iterator.close(); iterator = @@ -629,4 +638,29 @@ public void testScanFromChangelog(String changelogProducer) throws Exception { assertThat(iterator.collect(1)).containsExactlyInAnyOrder(Row.of("10", "11", "12")); iterator.close(); } + + @Test + public void testAvroRetractNotNullField() { + List input = + Arrays.asList( + Row.ofKind(RowKind.INSERT, 1, "A"), Row.ofKind(RowKind.DELETE, 1, "A")); + String id = TestValuesTableFactory.registerData(input); + sEnv.executeSql( + String.format( + "CREATE TEMPORARY TABLE source (pk INT PRIMARY KEY NOT ENFORCED, a STRING) " + + "WITH ('connector'='values', 'bounded'='true', 'data-id'='%s', " + + "'changelog-mode' = 'I,D,UA,UB')", + id)); + + sql( + "CREATE TABLE avro_sink (pk INT PRIMARY KEY NOT ENFORCED, a STRING NOT NULL) " + + " WITH ('file.format' = 'avro', 'merge-engine' = 'aggregation')"); + + assertThatThrownBy( + () -> sEnv.executeSql("INSERT INTO avro_sink select * from source").await()) + .satisfies( + anyCauseMatches( + RuntimeException.class, + "Caught NullPointerException, the possible reason is you have set following options together")); + } } diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/FileStoreITCase.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/FileStoreITCase.java index 6a2c7b071d2d..5245114e80ee 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/FileStoreITCase.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/FileStoreITCase.java @@ -36,6 +36,7 @@ import org.apache.paimon.utils.BranchManager; import org.apache.paimon.utils.FailingFileIO; +import org.apache.flink.api.common.eventtime.WatermarkStrategy; import org.apache.flink.api.common.functions.MapFunction; import org.apache.flink.api.connector.source.Boundedness; import org.apache.flink.api.dag.Transformation; @@ -450,7 +451,12 @@ private void sinkAndValidate( throw new UnsupportedOperationException(); } DataStreamSource source = - env.addSource(new FiniteTestSource<>(src, true), InternalTypeInfo.of(TABLE_TYPE)); + env.fromSource( + new FiniteTestSource<>(src, true), + WatermarkStrategy.noWatermarks(), + "FiniteTestSource", + InternalTypeInfo.of(TABLE_TYPE)); + source.forceNonParallel(); new FlinkSinkBuilder(table).forRowData(source).build(); env.execute(); assertThat(iterator.collect(expected.length)).containsExactlyInAnyOrder(expected); @@ -521,9 +527,13 @@ public static DataStreamSource buildTestSource( StreamExecutionEnvironment env, boolean isBatch) { return isBatch ? env.fromCollection(SOURCE_DATA, InternalTypeInfo.of(TABLE_TYPE)) - : env.addSource( - new FiniteTestSource<>(SOURCE_DATA, false), - InternalTypeInfo.of(TABLE_TYPE)); + : (DataStreamSource) + env.fromSource( + new FiniteTestSource<>(SOURCE_DATA, false), + WatermarkStrategy.noWatermarks(), + "FiniteTestSource", + InternalTypeInfo.of(TABLE_TYPE)) + .forceNonParallel(); } public static List executeAndCollect(DataStream source) throws Exception { diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/FileSystemCatalogITCase.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/FileSystemCatalogITCase.java index 239043ff79e1..915c93680a0d 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/FileSystemCatalogITCase.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/FileSystemCatalogITCase.java @@ -27,7 +27,6 @@ import org.apache.paimon.fs.Path; import org.apache.paimon.utils.BlockingIterator; -import org.apache.flink.streaming.api.environment.ExecutionCheckpointingOptions; import org.apache.flink.table.api.TableEnvironment; import org.apache.flink.types.Row; import org.apache.flink.util.CloseableIterator; @@ -60,7 +59,7 @@ public void setup() { tableEnvironmentBuilder() .streamingMode() .parallelism(1) - .setConf(ExecutionCheckpointingOptions.ENABLE_UNALIGNED, false) + .setString("execution.checkpointing.unaligned.enabled", "false") .build(); path = getTempDirPath(); tEnv.executeSql( diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/FiniteTestSource.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/FiniteTestSource.java index 9c5254d6283b..6691b9c09514 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/FiniteTestSource.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/FiniteTestSource.java @@ -18,16 +18,18 @@ package org.apache.paimon.flink; +import org.apache.paimon.flink.source.AbstractNonCoordinatedSource; +import org.apache.paimon.flink.source.AbstractNonCoordinatedSourceReader; +import org.apache.paimon.flink.source.SimpleSourceSplit; +import org.apache.paimon.flink.source.SplitListState; import org.apache.paimon.utils.Preconditions; -import org.apache.flink.api.common.state.CheckpointListener; -import org.apache.flink.api.common.state.ListState; -import org.apache.flink.api.common.state.ListStateDescriptor; -import org.apache.flink.api.common.typeutils.base.IntSerializer; -import org.apache.flink.runtime.state.FunctionInitializationContext; -import org.apache.flink.runtime.state.FunctionSnapshotContext; +import org.apache.flink.api.connector.source.Boundedness; +import org.apache.flink.api.connector.source.ReaderOutput; +import org.apache.flink.api.connector.source.SourceReader; +import org.apache.flink.api.connector.source.SourceReaderContext; +import org.apache.flink.core.io.InputStatus; import org.apache.flink.streaming.api.checkpoint.CheckpointedFunction; -import org.apache.flink.streaming.api.functions.source.SourceFunction; import java.util.ArrayList; import java.util.List; @@ -39,8 +41,7 @@ * *

    The reason this class is rewritten is to support {@link CheckpointedFunction}. */ -public class FiniteTestSource - implements SourceFunction, CheckpointedFunction, CheckpointListener { +public class FiniteTestSource extends AbstractNonCoordinatedSource { private static final long serialVersionUID = 1L; @@ -48,27 +49,78 @@ public class FiniteTestSource private final boolean emitOnce; - private volatile boolean running = true; - - private transient int numCheckpointsComplete; - - private transient ListState checkpointedState; - - private volatile int numTimesEmitted; - public FiniteTestSource(List elements, boolean emitOnce) { this.elements = elements; this.emitOnce = emitOnce; } @Override - public void initializeState(FunctionInitializationContext context) throws Exception { - this.checkpointedState = - context.getOperatorStateStore() - .getListState( - new ListStateDescriptor<>("emit-times", IntSerializer.INSTANCE)); + public Boundedness getBoundedness() { + return Boundedness.BOUNDED; + } + + @Override + public SourceReader createReader(SourceReaderContext sourceReaderContext) + throws Exception { + return new Reader<>(elements, emitOnce); + } + + private static class Reader extends AbstractNonCoordinatedSourceReader { + + private final List elements; + + private final boolean emitOnce; + + private final SplitListState checkpointedState = + new SplitListState<>("emit-times", x -> Integer.toString(x), Integer::parseInt); + + private int numTimesEmitted = 0; + + private int numCheckpointsComplete; + + private Integer checkpointToAwait; + + private Reader(List elements, boolean emitOnce) { + this.elements = elements; + this.emitOnce = emitOnce; + this.numCheckpointsComplete = 0; + } + + @Override + public synchronized InputStatus pollNext(ReaderOutput readerOutput) { + if (checkpointToAwait == null) { + checkpointToAwait = numCheckpointsComplete + 2; + } + switch (numTimesEmitted) { + case 0: + emitElements(readerOutput, false); + if (numCheckpointsComplete < checkpointToAwait) { + return InputStatus.MORE_AVAILABLE; + } + emitElements(readerOutput, true); + if (numCheckpointsComplete < checkpointToAwait + 2) { + return InputStatus.MORE_AVAILABLE; + } + break; + case 1: + emitElements(readerOutput, true); + if (numCheckpointsComplete < checkpointToAwait) { + return InputStatus.MORE_AVAILABLE; + } + break; + case 2: + // Maybe missed notifyCheckpointComplete, wait next notifyCheckpointComplete + if (numCheckpointsComplete < checkpointToAwait) { + return InputStatus.MORE_AVAILABLE; + } + break; + } + return InputStatus.END_OF_INPUT; + } - if (context.isRestored()) { + @Override + public void addSplits(List list) { + checkpointedState.restoreState(list); List retrievedStates = new ArrayList<>(); for (Integer entry : this.checkpointedState.get()) { retrievedStates.add(entry); @@ -85,76 +137,27 @@ public void initializeState(FunctionInitializationContext context) throws Except getClass().getSimpleName() + " retrieved invalid numTimesEmitted: " + numTimesEmitted); - } else { - this.numTimesEmitted = 0; } - } - @Override - public void run(SourceContext ctx) throws Exception { - switch (numTimesEmitted) { - case 0: - emitElementsAndWaitForCheckpoints(ctx, false); - emitElementsAndWaitForCheckpoints(ctx, true); - break; - case 1: - emitElementsAndWaitForCheckpoints(ctx, true); - break; - case 2: - // Maybe missed notifyCheckpointComplete, wait next notifyCheckpointComplete - final Object lock = ctx.getCheckpointLock(); - synchronized (lock) { - int checkpointToAwait = numCheckpointsComplete + 2; - while (running && numCheckpointsComplete < checkpointToAwait) { - lock.wait(1); - } - } - break; + @Override + public List snapshotState(long l) { + this.checkpointedState.clear(); + this.checkpointedState.add(this.numTimesEmitted); + return this.checkpointedState.snapshotState(); } - } - private void emitElementsAndWaitForCheckpoints(SourceContext ctx, boolean isSecond) - throws InterruptedException { - final Object lock = ctx.getCheckpointLock(); + @Override + public void notifyCheckpointComplete(long checkpointId) { + numCheckpointsComplete++; + } - final int checkpointToAwait; - synchronized (lock) { - checkpointToAwait = numCheckpointsComplete + 2; + private void emitElements(ReaderOutput readerOutput, boolean isSecond) { if (!isSecond || !emitOnce) { for (T t : elements) { - ctx.collect(t); + readerOutput.collect(t); } } numTimesEmitted++; } - - synchronized (lock) { - while (running && numCheckpointsComplete < checkpointToAwait) { - lock.wait(1); - } - } - } - - @Override - public void cancel() { - running = false; - } - - @Override - public void notifyCheckpointComplete(long checkpointId) { - numCheckpointsComplete++; - } - - @Override - public void notifyCheckpointAborted(long checkpointId) {} - - @Override - public void snapshotState(FunctionSnapshotContext context) throws Exception { - Preconditions.checkState( - this.checkpointedState != null, - "The " + getClass().getSimpleName() + " has not been properly initialized."); - - this.checkpointedState.clear(); - this.checkpointedState.add(this.numTimesEmitted); } } diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/FirstRowITCase.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/FirstRowITCase.java index e44aa4c63d89..dc7bf397e4a9 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/FirstRowITCase.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/FirstRowITCase.java @@ -95,7 +95,7 @@ public void testLocalMerge() { sql( "CREATE TABLE IF NOT EXISTS T1 (" + "a INT, b INT, c STRING, PRIMARY KEY (a, b) NOT ENFORCED)" - + " PARTITIONED BY (b) WITH ('merge-engine'='first-row', 'local-merge-buffer-size' = '1m'," + + " PARTITIONED BY (b) WITH ('merge-engine'='first-row', 'local-merge-buffer-size' = '5m'," + " 'file.format'='avro', 'changelog-producer' = 'lookup');"); batchSql("INSERT INTO T1 VALUES (1, 1, '1'), (1, 1, '2'), (2, 3, '3')"); List result = batchSql("SELECT * FROM T1"); diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/FlinkAnalyzeTableITCase.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/FlinkAnalyzeTableITCase.java new file mode 100644 index 000000000000..e186080d9f45 --- /dev/null +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/FlinkAnalyzeTableITCase.java @@ -0,0 +1,199 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.flink; + +import org.apache.paimon.catalog.Catalog; +import org.apache.paimon.data.Decimal; +import org.apache.paimon.stats.ColStats; +import org.apache.paimon.stats.Statistics; +import org.apache.paimon.table.FileStoreTable; +import org.apache.paimon.utils.DateTimeUtils; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.Collections; +import java.util.Map; +import java.util.Optional; + +import static org.assertj.core.api.Assertions.assertThat; + +/** IT cases for analyze table. */ +public class FlinkAnalyzeTableITCase extends CatalogITCaseBase { + + @Test + public void testAnalyzeTable() throws Catalog.TableNotExistException { + sql( + "CREATE TABLE T (" + + " id STRING" + + ", name STRING" + + ", i INT" + + ", l bigint" + + ", PRIMARY KEY (id) NOT ENFORCED" + + " ) WITH (" + + " 'bucket' = '2'" + + " )"); + sql("INSERT INTO T VALUES ('1', 'a', 1, 1)"); + sql("INSERT INTO T VALUES ('2', 'aaa', 1, 2)"); + sql("ANALYZE TABLE T COMPUTE STATISTICS"); + + FileStoreTable table = paimonTable("T"); + Optional statisticsOpt = table.statistics(); + assertThat(statisticsOpt.isPresent()).isTrue(); + Statistics stats = statisticsOpt.get(); + + assertThat(stats.mergedRecordCount().isPresent()).isTrue(); + Assertions.assertEquals(2L, stats.mergedRecordCount().getAsLong()); + + Assertions.assertTrue(stats.mergedRecordSize().isPresent()); + Assertions.assertTrue(stats.colStats().isEmpty()); + + // by default, caching catalog should cache it + Optional newStats = table.statistics(); + assertThat(newStats.isPresent()).isTrue(); + assertThat(newStats.get()).isSameAs(stats); + + // copy the table + newStats = table.copy(Collections.singletonMap("a", "b")).statistics(); + assertThat(newStats.isPresent()).isTrue(); + assertThat(newStats.get()).isSameAs(stats); + } + + @Test + public void testAnalyzeTableColumn() throws Catalog.TableNotExistException { + sql( + "CREATE TABLE T (" + + "id STRING, name STRING, bytes_col BYTES, int_col INT, long_col bigint,\n" + + "float_col FLOAT, double_col DOUBLE, decimal_col DECIMAL(10, 5), boolean_col BOOLEAN, date_col DATE,\n" + + "timestamp_col TIMESTAMP_LTZ, binary_col BINARY, varbinary_col VARBINARY, char_col CHAR(20), varchar_col VARCHAR(20),\n" + + "tinyint_col TINYINT, smallint_col SMALLINT" + + ", PRIMARY KEY (id) NOT ENFORCED" + + " ) WITH (" + + " 'bucket' = '2'" + + " )"); + sql( + "INSERT INTO T VALUES ('1', 'a', CAST('your_binary_data' AS BYTES), 1, 1, 1.0, 1.0, 13.12345, true, cast('2020-01-01' as date), cast('2024-01-01 00:00:00' as TIMESTAMP_LTZ), CAST('example binary1' AS BINARY), CAST('example binary1' AS VARBINARY), 'a', 'a',CAST(1 AS TINYINT), CAST(2 AS SMALLINT))"); + sql( + "INSERT INTO T VALUES ('2', 'aaa', CAST('your_binary_data' AS BYTES), 1, 1, 1.0, 5.0, 12.12345, true, cast('2021-01-02' as date), cast('2024-01-02 00:00:00' as TIMESTAMP_LTZ), CAST('example binary1' AS BINARY), CAST('example binary1' AS VARBINARY), 'aaa', 'aaa', CAST(2 AS TINYINT), CAST(4 AS SMALLINT))"); + + sql( + "INSERT INTO T VALUES ('3', 'bbbb', CAST('data' AS BYTES), 4, 19, 7.0, 1.0, 14.12345, true, cast(NULL as date), cast('2024-01-02 05:00:00' as TIMESTAMP_LTZ), CAST(NULL AS BINARY), CAST('example binary1' AS VARBINARY), 'aaa', 'aaa', CAST(NULL AS TINYINT), CAST(4 AS SMALLINT))"); + + sql( + "INSERT INTO T VALUES ('4', 'aa', CAST(NULL AS BYTES), 1, 1, 1.0, 1.0, 14.12345, false, cast(NULL as date), cast(NULL as TIMESTAMP_LTZ), CAST(NULL AS BINARY), CAST('example' AS VARBINARY), 'aba', 'aaab', CAST(NULL AS TINYINT), CAST(4 AS SMALLINT))"); + + sql("ANALYZE TABLE T COMPUTE STATISTICS FOR ALL COLUMNS"); + + Optional statisticsOpt = paimonTable("T").statistics(); + assertThat(statisticsOpt.isPresent()).isTrue(); + Statistics stats = statisticsOpt.get(); + + assertThat(stats.mergedRecordCount().isPresent()).isTrue(); + Assertions.assertEquals(4L, stats.mergedRecordCount().getAsLong()); + + Map> colStats = stats.colStats(); + Assertions.assertEquals( + ColStats.newColStats(0, 4L, null, null, 0L, 1L, 1L), colStats.get("id")); + Assertions.assertEquals( + ColStats.newColStats(1, 4L, null, null, 0L, 2L, 4L), colStats.get("name")); + + Assertions.assertEquals( + ColStats.newColStats(2, null, null, null, 1L, null, null), + colStats.get("bytes_col")); + + Assertions.assertEquals( + ColStats.newColStats(3, 2L, 1, 4, 0L, null, null), colStats.get("int_col")); + + Assertions.assertEquals( + ColStats.newColStats(4, 2L, 1L, 19L, 0L, null, null), colStats.get("long_col")); + + Assertions.assertEquals( + ColStats.newColStats(5, 2L, 1.0f, 7.0f, 0L, null, null), colStats.get("float_col")); + + Assertions.assertEquals( + ColStats.newColStats(6, 2L, 1.0d, 5.0d, 0L, null, null), + colStats.get("double_col")); + + Assertions.assertEquals( + ColStats.newColStats( + 7, + 3L, + Decimal.fromBigDecimal(new java.math.BigDecimal("12.12345"), 10, 5), + Decimal.fromBigDecimal(new java.math.BigDecimal("14.12345"), 10, 5), + 0L, + null, + null), + colStats.get("decimal_col")); + + Assertions.assertEquals( + ColStats.newColStats(8, 2L, null, null, 0L, null, null), + colStats.get("boolean_col")); + + Assertions.assertEquals( + ColStats.newColStats(9, 2L, 18262, 18629, 2L, null, null), + colStats.get("date_col")); + + Assertions.assertEquals( + ColStats.newColStats( + 10, + 3L, + DateTimeUtils.parseTimestampData("2024-01-01 00:00:00", 0), + DateTimeUtils.parseTimestampData("2024-01-02 05:00:00", 0), + 1L, + null, + null), + colStats.get("timestamp_col")); + + Assertions.assertEquals( + ColStats.newColStats(11, null, null, null, 2L, null, null), + colStats.get("binary_col")); + + Assertions.assertEquals( + ColStats.newColStats(12, null, null, null, 0L, null, null), + colStats.get("varbinary_col")); + + Assertions.assertEquals( + ColStats.newColStats(13, 3L, null, null, 0L, 20L, 20L), colStats.get("char_col")); + + Assertions.assertEquals( + ColStats.newColStats(14, 3L, null, null, 0L, 2L, 4L), colStats.get("varchar_col")); + + Assertions.assertEquals( + ColStats.newColStats( + 15, + 2L, + new Integer(1).byteValue(), + new Integer(2).byteValue(), + 2L, + null, + null), + colStats.get("tinyint_col")); + + Assertions.assertEquals( + ColStats.newColStats( + 16, + 2L, + new Integer(2).shortValue(), + new Integer(4).shortValue(), + 0L, + null, + null), + colStats.get("smallint_col")); + } +} diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/FlinkBatchJobPartitionMarkdoneITCase.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/FlinkBatchJobPartitionMarkdoneITCase.java index 9c97151ecbe7..340213aeae1b 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/FlinkBatchJobPartitionMarkdoneITCase.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/FlinkBatchJobPartitionMarkdoneITCase.java @@ -161,7 +161,7 @@ private FileStoreTable buildFileStoreTable(int[] partitions, int[] primaryKey) options.set(BUCKET, 3); options.set(PATH, getTempDirPath()); options.set(FILE_FORMAT, CoreOptions.FILE_FORMAT_AVRO); - options.set(FlinkConnectorOptions.PARTITION_MARK_DONE_WHEN_END_INPUT.key(), "true"); + options.set(CoreOptions.PARTITION_MARK_DONE_WHEN_END_INPUT.key(), "true"); Path tablePath = new CoreOptions(options.toMap()).path(); if (primaryKey.length == 0) { diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/FlinkCatalogPropertiesUtilTest.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/FlinkCatalogPropertiesUtilTest.java index 9268a236b6cb..e32150b1fe82 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/FlinkCatalogPropertiesUtilTest.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/FlinkCatalogPropertiesUtilTest.java @@ -21,27 +21,35 @@ import org.apache.paimon.flink.utils.FlinkCatalogPropertiesUtil; import org.apache.flink.table.api.DataTypes; -import org.apache.flink.table.api.TableColumn; -import org.apache.flink.table.api.WatermarkSpec; +import org.apache.flink.table.api.Schema; +import org.apache.flink.table.catalog.Column; +import org.apache.flink.table.catalog.ResolvedSchema; +import org.apache.flink.table.catalog.WatermarkSpec; +import org.apache.flink.table.expressions.Expression; +import org.apache.flink.table.expressions.ExpressionVisitor; +import org.apache.flink.table.expressions.ResolvedExpression; +import org.apache.flink.table.expressions.SqlCallExpression; +import org.apache.flink.table.types.DataType; import org.junit.jupiter.api.Test; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; -import static org.apache.flink.table.descriptors.DescriptorProperties.DATA_TYPE; -import static org.apache.flink.table.descriptors.DescriptorProperties.EXPR; -import static org.apache.flink.table.descriptors.DescriptorProperties.METADATA; -import static org.apache.flink.table.descriptors.DescriptorProperties.NAME; -import static org.apache.flink.table.descriptors.DescriptorProperties.VIRTUAL; -import static org.apache.flink.table.descriptors.DescriptorProperties.WATERMARK; -import static org.apache.flink.table.descriptors.DescriptorProperties.WATERMARK_ROWTIME; -import static org.apache.flink.table.descriptors.DescriptorProperties.WATERMARK_STRATEGY_DATA_TYPE; -import static org.apache.flink.table.descriptors.DescriptorProperties.WATERMARK_STRATEGY_EXPR; -import static org.apache.flink.table.descriptors.Schema.SCHEMA; +import static org.apache.paimon.flink.utils.FlinkCatalogPropertiesUtil.SCHEMA; import static org.apache.paimon.flink.utils.FlinkCatalogPropertiesUtil.compoundKey; +import static org.apache.paimon.flink.utils.FlinkDescriptorProperties.DATA_TYPE; +import static org.apache.paimon.flink.utils.FlinkDescriptorProperties.EXPR; +import static org.apache.paimon.flink.utils.FlinkDescriptorProperties.METADATA; +import static org.apache.paimon.flink.utils.FlinkDescriptorProperties.NAME; +import static org.apache.paimon.flink.utils.FlinkDescriptorProperties.VIRTUAL; +import static org.apache.paimon.flink.utils.FlinkDescriptorProperties.WATERMARK; +import static org.apache.paimon.flink.utils.FlinkDescriptorProperties.WATERMARK_ROWTIME; +import static org.apache.paimon.flink.utils.FlinkDescriptorProperties.WATERMARK_STRATEGY_DATA_TYPE; +import static org.apache.paimon.flink.utils.FlinkDescriptorProperties.WATERMARK_STRATEGY_EXPR; import static org.assertj.core.api.Assertions.assertThat; /** Test for {@link FlinkCatalogPropertiesUtil}. */ @@ -49,18 +57,27 @@ public class FlinkCatalogPropertiesUtilTest { @Test public void testSerDeNonPhysicalColumns() { - Map indexMap = new HashMap<>(); - indexMap.put("comp", 2); - indexMap.put("meta1", 3); - indexMap.put("meta2", 5); - List columns = new ArrayList<>(); - columns.add(TableColumn.computed("comp", DataTypes.INT(), "`k` * 2")); - columns.add(TableColumn.metadata("meta1", DataTypes.VARCHAR(10))); - columns.add(TableColumn.metadata("meta2", DataTypes.BIGINT().notNull(), "price", true)); + List columns = new ArrayList<>(); + columns.add(new Schema.UnresolvedComputedColumn("comp", new SqlCallExpression("`k` * 2"))); + columns.add( + new Schema.UnresolvedMetadataColumn("meta1", DataTypes.VARCHAR(10), null, false)); + columns.add( + new Schema.UnresolvedMetadataColumn( + "meta2", DataTypes.BIGINT().notNull(), "price", true, null)); + + List resolvedColumns = new ArrayList<>(); + resolvedColumns.add(Column.physical("phy1", DataTypes.INT())); + resolvedColumns.add(Column.physical("phy2", DataTypes.INT())); + resolvedColumns.add( + Column.computed("comp", new TestResolvedExpression("`k` * 2", DataTypes.INT()))); + resolvedColumns.add(Column.metadata("meta1", DataTypes.VARCHAR(10), null, false)); + resolvedColumns.add(Column.physical("phy3", DataTypes.INT())); + resolvedColumns.add(Column.metadata("meta2", DataTypes.BIGINT().notNull(), "price", true)); // validate serialization Map serialized = - FlinkCatalogPropertiesUtil.serializeNonPhysicalColumns(indexMap, columns); + FlinkCatalogPropertiesUtil.serializeNonPhysicalNewColumns( + new ResolvedSchema(resolvedColumns, Collections.emptyList(), null)); Map expected = new HashMap<>(); expected.put(compoundKey(SCHEMA, 2, NAME), "comp"); @@ -80,27 +97,26 @@ public void testSerDeNonPhysicalColumns() { assertThat(serialized).containsExactlyInAnyOrderEntriesOf(expected); // validate deserialization - List deserialized = new ArrayList<>(); - deserialized.add(FlinkCatalogPropertiesUtil.deserializeNonPhysicalColumn(serialized, 2)); - deserialized.add(FlinkCatalogPropertiesUtil.deserializeNonPhysicalColumn(serialized, 3)); - deserialized.add(FlinkCatalogPropertiesUtil.deserializeNonPhysicalColumn(serialized, 5)); + Schema.Builder builder = Schema.newBuilder(); + FlinkCatalogPropertiesUtil.deserializeNonPhysicalColumn(serialized, 2, builder); + FlinkCatalogPropertiesUtil.deserializeNonPhysicalColumn(serialized, 3, builder); + FlinkCatalogPropertiesUtil.deserializeNonPhysicalColumn(serialized, 5, builder); - assertThat(deserialized).isEqualTo(columns); - - // validate that + assertThat(builder.build().getColumns()) + .containsExactly(columns.toArray(new Schema.UnresolvedColumn[0])); } @Test public void testSerDeWatermarkSpec() { WatermarkSpec watermarkSpec = - new WatermarkSpec( + WatermarkSpec.of( "test_time", - "`test_time` - INTERVAL '0.001' SECOND", - DataTypes.TIMESTAMP(3)); + new TestResolvedExpression( + "`test_time` - INTERVAL '0.001' SECOND", DataTypes.TIMESTAMP(3))); // validate serialization Map serialized = - FlinkCatalogPropertiesUtil.serializeWatermarkSpec(watermarkSpec); + FlinkCatalogPropertiesUtil.serializeNewWatermarkSpec(watermarkSpec); Map expected = new HashMap<>(); String watermarkPrefix = compoundKey(SCHEMA, WATERMARK, 0); @@ -113,9 +129,13 @@ public void testSerDeWatermarkSpec() { assertThat(serialized).containsExactlyInAnyOrderEntriesOf(expected); // validate serialization - WatermarkSpec deserialized = - FlinkCatalogPropertiesUtil.deserializeWatermarkSpec(serialized); - assertThat(deserialized).isEqualTo(watermarkSpec); + Schema.Builder builder = Schema.newBuilder(); + FlinkCatalogPropertiesUtil.deserializeWatermarkSpec(serialized, builder); + assertThat(builder.build().getWatermarkSpecs()).hasSize(1); + Schema.UnresolvedWatermarkSpec actual = builder.build().getWatermarkSpecs().get(0); + assertThat(actual.getColumnName()).isEqualTo(watermarkSpec.getRowtimeAttribute()); + assertThat(actual.getWatermarkExpression().asSummaryString()) + .isEqualTo(watermarkSpec.getWatermarkExpression().asSummaryString()); } @Test @@ -150,4 +170,44 @@ public void testNonPhysicalColumnsCount() { oldStyleOptions, Arrays.asList("phy1", "phy2"))) .isEqualTo(3); } + + private static class TestResolvedExpression implements ResolvedExpression { + private final String name; + private final DataType outputDataType; + + private TestResolvedExpression(String name, DataType outputDataType) { + this.name = name; + this.outputDataType = outputDataType; + } + + @Override + public DataType getOutputDataType() { + return outputDataType; + } + + @Override + public List getResolvedChildren() { + return Collections.emptyList(); + } + + @Override + public String asSummaryString() { + return new SqlCallExpression(name).asSummaryString(); + } + + @Override + public String asSerializableString() { + return name; + } + + @Override + public List getChildren() { + return Collections.emptyList(); + } + + @Override + public R accept(ExpressionVisitor expressionVisitor) { + return null; + } + } } diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/FlinkCatalogTest.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/FlinkCatalogTest.java index 4f2db7aabeed..e4286eb18172 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/FlinkCatalogTest.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/FlinkCatalogTest.java @@ -28,9 +28,12 @@ import org.apache.paimon.flink.log.LogStoreTableFactory; import org.apache.paimon.fs.Path; import org.apache.paimon.options.Options; +import org.apache.paimon.table.FileStoreTable; +import org.apache.paimon.table.Table; import org.apache.paimon.shade.guava30.com.google.common.collect.ImmutableList; +import org.apache.flink.configuration.Configuration; import org.apache.flink.table.api.DataTypes; import org.apache.flink.table.api.Schema; import org.apache.flink.table.api.TableDescriptor; @@ -42,6 +45,7 @@ import org.apache.flink.table.catalog.CatalogTable; import org.apache.flink.table.catalog.Column; import org.apache.flink.table.catalog.IntervalFreshness; +import org.apache.flink.table.catalog.ObjectIdentifier; import org.apache.flink.table.catalog.ObjectPath; import org.apache.flink.table.catalog.ResolvedCatalogMaterializedTable; import org.apache.flink.table.catalog.ResolvedCatalogTable; @@ -60,6 +64,7 @@ import org.apache.flink.table.expressions.ResolvedExpression; import org.apache.flink.table.expressions.utils.ResolvedExpressionMock; import org.apache.flink.table.factories.DynamicTableFactory; +import org.apache.flink.table.factories.FactoryUtil; import org.apache.flink.table.refresh.RefreshHandler; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -88,6 +93,7 @@ import static org.apache.paimon.flink.FlinkCatalogOptions.DISABLE_CREATE_TABLE_IN_DEFAULT_DB; import static org.apache.paimon.flink.FlinkCatalogOptions.LOG_SYSTEM_AUTO_REGISTER; import static org.apache.paimon.flink.FlinkConnectorOptions.LOG_SYSTEM; +import static org.apache.paimon.flink.FlinkTestBase.createResolvedTable; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatCode; import static org.assertj.core.api.Assertions.assertThatCollection; @@ -746,6 +752,52 @@ void testCreateTableFromTableDescriptor() throws Exception { checkCreateTable(path1, catalogTable, (CatalogTable) catalog.getTable(path1)); } + @Test + void testBuildPaimonTableWithCustomScheme() throws Exception { + catalog.createDatabase(path1.getDatabaseName(), null, false); + CatalogTable table = createTable(optionProvider(false).iterator().next()); + catalog.createTable(path1, table, false); + checkCreateTable(path1, table, catalog.getTable(path1)); + + List columns = + Arrays.asList( + Column.physical("first", DataTypes.STRING()), + Column.physical("second", DataTypes.INT()), + Column.physical("third", DataTypes.STRING()), + Column.physical( + "four", + DataTypes.ROW( + DataTypes.FIELD("f1", DataTypes.STRING()), + DataTypes.FIELD("f2", DataTypes.INT()), + DataTypes.FIELD( + "f3", + DataTypes.MAP( + DataTypes.STRING(), DataTypes.INT()))))); + DynamicTableFactory.Context context = + new FactoryUtil.DefaultDynamicTableContext( + ObjectIdentifier.of( + "default", path1.getDatabaseName(), path1.getObjectName()), + createResolvedTable( + new HashMap() { + { + put("path", "unsupported-scheme://foobar"); + } + }, + columns, + Collections.emptyList(), + Collections.emptyList()), + Collections.emptyMap(), + new Configuration(), + Thread.currentThread().getContextClassLoader(), + false); + + FlinkTableFactory factory = (FlinkTableFactory) catalog.getFactory().get(); + Table builtTable = factory.buildPaimonTable(context); + assertThat(builtTable).isInstanceOf(FileStoreTable.class); + assertThat(((FileStoreTable) builtTable).schema().fieldNames()) + .containsExactly("first", "second", "third", "four"); + } + private void checkCreateTable( ObjectPath path, CatalogBaseTable expected, CatalogBaseTable actual) { checkEquals( @@ -798,7 +850,7 @@ private static void checkEquals(CatalogBaseTable t1, CatalogBaseTable t2) { assertThat(t2.getComment()).isEqualTo(t1.getComment()); assertThat(t2.getOptions()).isEqualTo(t1.getOptions()); if (t1.getTableKind() == CatalogBaseTable.TableKind.TABLE) { - assertThat(t2.getSchema()).isEqualTo(t1.getSchema()); + assertThat(t2.getUnresolvedSchema()).isEqualTo(t1.getUnresolvedSchema()); assertThat(((CatalogTable) (t2)).getPartitionKeys()) .isEqualTo(((CatalogTable) (t1)).getPartitionKeys()); assertThat(((CatalogTable) (t2)).isPartitioned()) @@ -812,7 +864,12 @@ private static void checkEquals(CatalogBaseTable t1, CatalogBaseTable t2) { t2.getUnresolvedSchema() .resolve(new TestSchemaResolver())) .build()) - .isEqualTo(t1.getSchema().toSchema()); + .isEqualTo( + Schema.newBuilder() + .fromResolvedSchema( + t1.getUnresolvedSchema() + .resolve(new TestSchemaResolver())) + .build()); assertThat(mt2.getPartitionKeys()).isEqualTo(mt1.getPartitionKeys()); assertThat(mt2.isPartitioned()).isEqualTo(mt1.isPartitioned()); // validate definition query diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/FlinkJobRecoveryITCase.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/FlinkJobRecoveryITCase.java new file mode 100644 index 000000000000..8df379a71b78 --- /dev/null +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/FlinkJobRecoveryITCase.java @@ -0,0 +1,333 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.flink; + +import org.apache.paimon.Snapshot; +import org.apache.paimon.table.BucketMode; +import org.apache.paimon.utils.Pair; + +import org.apache.flink.api.common.JobID; +import org.apache.flink.api.common.JobStatus; +import org.apache.flink.configuration.CheckpointingOptions; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.configuration.ExternalizedCheckpointRetention; +import org.apache.flink.configuration.StateRecoveryOptions; +import org.apache.flink.core.execution.JobClient; +import org.apache.flink.runtime.execution.ExecutionState; +import org.apache.flink.runtime.minicluster.MiniCluster; +import org.apache.flink.types.Row; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; + +import java.io.IOException; +import java.lang.reflect.Field; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; + +import static org.assertj.core.api.Assertions.assertThat; + +/** Test case for flink source / sink restore from savepoint. */ +@SuppressWarnings("BusyWait") +public class FlinkJobRecoveryITCase extends CatalogITCaseBase { + + private static final String MINI_CLUSTER_FIELD = "miniCluster"; + + @BeforeEach + @Override + public void before() throws IOException { + super.before(); + + // disable checkpoint + sEnv.getConfig() + .getConfiguration() + .set( + CheckpointingOptions.EXTERNALIZED_CHECKPOINT_RETENTION, + ExternalizedCheckpointRetention.RETAIN_ON_CANCELLATION) + .removeKey("execution.checkpointing.interval"); + + // insert source data + batchSql("INSERT INTO source_table1 VALUES (1, 'test-1', '20241030')"); + batchSql("INSERT INTO source_table1 VALUES (2, 'test-2', '20241030')"); + batchSql("INSERT INTO source_table1 VALUES (3, 'test-3', '20241030')"); + batchSql( + "INSERT INTO source_table2 VALUES (4, 'test-4', '20241031'), (5, 'test-5', '20241031'), (6, 'test-6', '20241031')"); + } + + @Override + protected List ddl() { + return Arrays.asList( + String.format( + "CREATE CATALOG `fs_catalog` WITH ('type'='paimon', 'warehouse'='%s')", + path), + "CREATE TABLE IF NOT EXISTS `source_table1` (k INT, f1 STRING, dt STRING) WITH ('bucket'='1', 'bucket-key'='k')", + "CREATE TABLE IF NOT EXISTS `source_table2` (k INT, f1 STRING, dt STRING) WITH ('bucket'='1', 'bucket-key'='k')"); + } + + @ParameterizedTest + @EnumSource(BucketMode.class) + @Timeout(300) + public void testRestoreFromSavepointWithJobGraphChange(BucketMode bucketMode) throws Exception { + createTargetTable("target_table", bucketMode); + String beforeRecoverSql = + "INSERT INTO `target_table` /*+ OPTIONS('sink.operator-uid.suffix'='test-uid') */ SELECT * FROM source_table1 /*+ OPTIONS('source.operator-uid.suffix'='test-uid') */"; + String beforeRecoverCheckSql = "SELECT * FROM target_table"; + List beforeRecoverExpectedRows = + Arrays.asList( + Row.of(1, "test-1", "20241030"), + Row.of(2, "test-2", "20241030"), + Row.of(3, "test-3", "20241030")); + String afterRecoverSql = + "INSERT INTO `target_table` /*+ OPTIONS('sink.operator-uid.suffix'='test-uid') */ (SELECT * FROM source_table1 /*+ OPTIONS('source.operator-uid.suffix'='test-uid') */ UNION ALL SELECT * FROM source_table2)"; + String afterRecoverCheckSql = "SELECT * FROM target_table"; + List afterRecoverExpectedRows = + Arrays.asList( + Row.of(1, "test-1", "20241030"), + Row.of(2, "test-2", "20241030"), + Row.of(3, "test-3", "20241030"), + Row.of(4, "test-4", "20241031"), + Row.of(5, "test-5", "20241031"), + Row.of(6, "test-6", "20241031")); + testRecoverFromSavepoint( + beforeRecoverSql, + beforeRecoverCheckSql, + beforeRecoverExpectedRows, + afterRecoverSql, + afterRecoverCheckSql, + afterRecoverExpectedRows, + Collections.emptyList(), + Pair.of("target_table", "target_table"), + Collections.emptyMap()); + } + + @Test + @Timeout(300) + public void testRestoreFromSavepointWithIgnoreSourceState() throws Exception { + createTargetTable("target_table", BucketMode.HASH_FIXED); + String beforeRecoverSql = "INSERT INTO `target_table` SELECT * FROM source_table1"; + String beforeRecoverCheckSql = "SELECT * FROM target_table"; + List beforeRecoverExpectedRows = + Arrays.asList( + Row.of(1, "test-1", "20241030"), + Row.of(2, "test-2", "20241030"), + Row.of(3, "test-3", "20241030")); + String afterRecoverSql = + "INSERT INTO `target_table` SELECT * FROM source_table2 /*+ OPTIONS('source.operator-uid.suffix'='test-uid') */"; + String afterRecoverCheckSql = "SELECT * FROM target_table"; + List afterRecoverExpectedRows = + Arrays.asList( + Row.of(1, "test-1", "20241030"), + Row.of(2, "test-2", "20241030"), + Row.of(3, "test-3", "20241030"), + Row.of(4, "test-4", "20241031"), + Row.of(5, "test-5", "20241031"), + Row.of(6, "test-6", "20241031")); + testRecoverFromSavepoint( + beforeRecoverSql, + beforeRecoverCheckSql, + beforeRecoverExpectedRows, + afterRecoverSql, + afterRecoverCheckSql, + afterRecoverExpectedRows, + Collections.emptyList(), + Pair.of("target_table", "target_table"), + Collections.singletonMap( + StateRecoveryOptions.SAVEPOINT_IGNORE_UNCLAIMED_STATE.key(), "true")); + } + + @Test + @Timeout(300) + public void testRestoreFromSavepointWithIgnoreSinkState() throws Exception { + createTargetTable("target_table", BucketMode.HASH_FIXED); + createTargetTable("target_table2", BucketMode.HASH_FIXED); + + String beforeRecoverSql = "INSERT INTO `target_table` SELECT * FROM source_table1"; + String beforeRecoverCheckSql = "SELECT * FROM target_table"; + List beforeRecoverExpectedRows = + Arrays.asList( + Row.of(1, "test-1", "20241030"), + Row.of(2, "test-2", "20241030"), + Row.of(3, "test-3", "20241030")); + String afterRecoverSql = + "INSERT INTO `target_table2` /*+ OPTIONS('sink.operator-uid.suffix'='test-uid') */ SELECT * FROM source_table1"; + String afterRecoverCheckSql = "SELECT * FROM target_table2"; + List afterRecoverExpectedRows = + Arrays.asList( + Row.of(7, "test-7", "20241030"), + Row.of(8, "test-8", "20241030"), + Row.of(9, "test-9", "20241030")); + String updateSql = + "INSERT INTO source_table1 VALUES (7, 'test-7', '20241030'), (8, 'test-8', '20241030'), (9, 'test-9', '20241030')"; + testRecoverFromSavepoint( + beforeRecoverSql, + beforeRecoverCheckSql, + beforeRecoverExpectedRows, + afterRecoverSql, + afterRecoverCheckSql, + afterRecoverExpectedRows, + Collections.singletonList(updateSql), + Pair.of("target_table", "target_table2"), + Collections.singletonMap( + StateRecoveryOptions.SAVEPOINT_IGNORE_UNCLAIMED_STATE.key(), "true")); + } + + private void testRecoverFromSavepoint( + String beforeRecoverSql, + String beforeRecoverCheckSql, + List beforeRecoverExpectedRows, + String afterRecoverSql, + String afterRecoverCheckSql, + List afterRecoverExpectedRows, + List updateSql, + Pair targetTables, + Map recoverOptions) + throws Exception { + + //noinspection OptionalGetWithoutIsPresent + JobClient jobClient = sEnv.executeSql(beforeRecoverSql).getJobClient().get(); + String checkpointPath = + triggerCheckpointAndWaitForWrites( + jobClient, targetTables.getLeft(), beforeRecoverExpectedRows.size()); + jobClient.cancel().get(); + + List rows = batchSql(beforeRecoverCheckSql); + assertThat(rows.size()).isEqualTo(beforeRecoverExpectedRows.size()); + assertThat(rows).containsExactlyInAnyOrder(beforeRecoverExpectedRows.toArray(new Row[0])); + + for (String sql : updateSql) { + batchSql(sql); + } + + Configuration config = sEnv.getConfig().getConfiguration(); + // use config string to stay compatible with flink 1.19- + config.setString("execution.state-recovery.path", checkpointPath); + for (Map.Entry entry : recoverOptions.entrySet()) { + config.setString(entry.getKey(), entry.getValue()); + } + + //noinspection OptionalGetWithoutIsPresent + jobClient = sEnv.executeSql(afterRecoverSql).getJobClient().get(); + triggerCheckpointAndWaitForWrites( + jobClient, targetTables.getRight(), afterRecoverExpectedRows.size()); + jobClient.cancel().get(); + + rows = batchSql(afterRecoverCheckSql); + assertThat(rows.size()).isEqualTo(afterRecoverExpectedRows.size()); + assertThat(rows).containsExactlyInAnyOrder(afterRecoverExpectedRows.toArray(new Row[0])); + } + + private void createTargetTable(String tableName, BucketMode bucketMode) { + switch (bucketMode) { + case HASH_FIXED: + batchSql( + String.format( + "CREATE TABLE IF NOT EXISTS `%s` (k INT, f1 STRING, pt STRING, PRIMARY KEY(k, pt) NOT ENFORCED) WITH ('bucket'='2', 'commit.force-create-snapshot'='true')", + tableName)); + return; + case HASH_DYNAMIC: + batchSql( + String.format( + "CREATE TABLE IF NOT EXISTS `%s` (k INT, f1 STRING, pt STRING, PRIMARY KEY(k, pt) NOT ENFORCED) WITH ('bucket'='-1', 'commit.force-create-snapshot'='true')", + tableName)); + return; + case CROSS_PARTITION: + batchSql( + String.format( + "CREATE TABLE IF NOT EXISTS `%s` (k INT, f1 STRING, pt STRING, PRIMARY KEY(k) NOT ENFORCED) WITH ('bucket'='-1', 'commit.force-create-snapshot'='true')", + tableName)); + return; + case BUCKET_UNAWARE: + batchSql( + String.format( + "CREATE TABLE IF NOT EXISTS `%s` (k INT, f1 STRING, pt STRING) WITH ('bucket'='-1', 'commit.force-create-snapshot'='true')", + tableName)); + return; + default: + throw new IllegalArgumentException("Unsupported bucket mode: " + bucketMode); + } + } + + private Snapshot waitForNewSnapshot(String tableName, long initialSnapshot) + throws InterruptedException { + Snapshot snapshot = findLatestSnapshot(tableName); + while (snapshot == null || snapshot.id() == initialSnapshot) { + Thread.sleep(2000L); + snapshot = findLatestSnapshot(tableName); + } + return snapshot; + } + + @SuppressWarnings("unchecked") + private T reflectGetMiniCluster(Object instance) + throws NoSuchFieldException, IllegalAccessException { + Field field = instance.getClass().getDeclaredField(MINI_CLUSTER_FIELD); + field.setAccessible(true); + return (T) field.get(instance); + } + + private String triggerCheckpointAndWaitForWrites( + JobClient jobClient, String targetTable, long totalRecords) throws Exception { + //noinspection resource + MiniCluster miniCluster = reflectGetMiniCluster(jobClient); + JobID jobID = jobClient.getJobID(); + JobStatus jobStatus = jobClient.getJobStatus().get(); + while (jobStatus == JobStatus.INITIALIZING || jobStatus == JobStatus.CREATED) { + Thread.sleep(2000L); + jobStatus = jobClient.getJobStatus().get(); + } + + if (jobStatus != JobStatus.RUNNING) { + throw new IllegalStateException("Job status is not RUNNING"); + } + + AtomicBoolean allTaskRunning = new AtomicBoolean(false); + while (!allTaskRunning.get()) { + allTaskRunning.set(true); + Thread.sleep(2000L); + miniCluster + .getExecutionGraph(jobID) + .thenAccept( + eg -> + eg.getAllExecutionVertices() + .forEach( + ev -> { + if (ev.getExecutionState() + != ExecutionState.RUNNING) { + allTaskRunning.set(false); + } + })) + .get(); + } + + String checkpointPath = miniCluster.triggerCheckpoint(jobID).get(); + Snapshot snapshot = waitForNewSnapshot(targetTable, -1L); + //noinspection DataFlowIssue + while (snapshot.totalRecordCount() < totalRecords) { + checkpointPath = miniCluster.triggerCheckpoint(jobID).get(); + snapshot = waitForNewSnapshot(targetTable, snapshot.id()); + } + + return checkpointPath; + } +} diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/FlinkLineageITCase.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/FlinkLineageITCase.java deleted file mode 100644 index 5b61d5272f80..000000000000 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/FlinkLineageITCase.java +++ /dev/null @@ -1,206 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.paimon.flink; - -import org.apache.paimon.lineage.DataLineageEntity; -import org.apache.paimon.lineage.LineageMeta; -import org.apache.paimon.lineage.LineageMetaFactory; -import org.apache.paimon.lineage.TableLineageEntity; -import org.apache.paimon.predicate.Predicate; - -import org.apache.flink.configuration.PipelineOptions; -import org.apache.flink.table.api.ValidationException; -import org.apache.flink.types.Row; -import org.apache.flink.util.CloseableIterator; -import org.junit.jupiter.api.Test; - -import javax.annotation.Nullable; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; - -import static org.apache.paimon.options.CatalogOptions.LINEAGE_META; -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; - -/** ITCase for flink table and data lineage. */ -public class FlinkLineageITCase extends CatalogITCaseBase { - private static final String THROWING_META = "throwing-meta"; - private static final Map> jobSourceTableLineages = - new HashMap<>(); - private static final Map> jobSinkTableLineages = - new HashMap<>(); - - @Override - protected List ddl() { - return Collections.singletonList("CREATE TABLE IF NOT EXISTS T (a INT, b INT, c INT)"); - } - - @Override - protected Map catalogOptions() { - return Collections.singletonMap(LINEAGE_META.key(), THROWING_META); - } - - @Test - public void testTableLineage() throws Exception { - // Validate for source and sink lineage when pipeline name is null - assertThatThrownBy( - () -> tEnv.executeSql("INSERT INTO T VALUES (1, 2, 3),(4, 5, 6);").await()) - .hasCauseExactlyInstanceOf(ValidationException.class) - .hasRootCauseMessage("Cannot get pipeline name for lineage meta."); - assertThatThrownBy(() -> tEnv.executeSql("SELECT * FROM T").collect().close()) - .hasCauseExactlyInstanceOf(ValidationException.class) - .hasRootCauseMessage("Cannot get pipeline name for lineage meta."); - - // Call storeSinkTableLineage and storeSourceTableLineage methods - tEnv.getConfig().getConfiguration().set(PipelineOptions.NAME, "insert_t_job"); - tEnv.executeSql("INSERT INTO T VALUES (1, 2, 3),(4, 5, 6);").await(); - assertThat(jobSinkTableLineages).isNotEmpty(); - TableLineageEntity sinkTableLineage = - jobSinkTableLineages.get("insert_t_job").get("default.T.insert_t_job"); - assertThat(sinkTableLineage.getTable()).isEqualTo("T"); - - List sinkTableRows = new ArrayList<>(); - try (CloseableIterator iterator = - tEnv.executeSql("SELECT * FROM sys.sink_table_lineage").collect()) { - while (iterator.hasNext()) { - sinkTableRows.add(iterator.next()); - } - } - assertThat(sinkTableRows.size()).isEqualTo(1); - Row sinkTableRow = sinkTableRows.get(0); - assertThat(sinkTableRow.getField("database_name")).isEqualTo("default"); - assertThat(sinkTableRow.getField("table_name")).isEqualTo("T"); - assertThat(sinkTableRow.getField("job_name")).isEqualTo("insert_t_job"); - - tEnv.getConfig().getConfiguration().set(PipelineOptions.NAME, "select_t_job"); - tEnv.executeSql("SELECT * FROM T").collect().close(); - assertThat(jobSourceTableLineages).isNotEmpty(); - TableLineageEntity sourceTableLineage = - jobSourceTableLineages.get("select_t_job").get("default.T.select_t_job"); - assertThat(sourceTableLineage.getTable()).isEqualTo("T"); - - List sourceTableRows = new ArrayList<>(); - try (CloseableIterator iterator = - tEnv.executeSql("SELECT * FROM sys.source_table_lineage").collect()) { - while (iterator.hasNext()) { - sourceTableRows.add(iterator.next()); - } - } - assertThat(sourceTableRows.size()).isEqualTo(1); - Row sourceTableRow = sourceTableRows.get(0); - assertThat(sourceTableRow.getField("database_name")).isEqualTo("default"); - assertThat(sourceTableRow.getField("table_name")).isEqualTo("T"); - assertThat(sourceTableRow.getField("job_name")).isEqualTo("select_t_job"); - } - - private static String getTableLineageKey(TableLineageEntity entity) { - return String.format("%s.%s.%s", entity.getDatabase(), entity.getTable(), entity.getJob()); - } - - /** Factory to create throwing lineage meta. */ - public static class TestingMemoryLineageMetaFactory implements LineageMetaFactory { - private static final long serialVersionUID = 1L; - - @Override - public String identifier() { - return THROWING_META; - } - - @Override - public LineageMeta create(LineageMetaContext context) { - return new TestingMemoryLineageMeta(); - } - } - - /** Throwing specific exception in each method. */ - private static class TestingMemoryLineageMeta implements LineageMeta { - - @Override - public void saveSourceTableLineage(TableLineageEntity entity) { - jobSourceTableLineages - .computeIfAbsent(entity.getJob(), key -> new HashMap<>()) - .put(getTableLineageKey(entity), entity); - } - - @Override - public void deleteSourceTableLineage(String job) { - jobSourceTableLineages.remove(job); - } - - @Override - public Iterator sourceTableLineages(@Nullable Predicate predicate) { - return jobSourceTableLineages.values().stream() - .flatMap(v -> v.values().stream()) - .iterator(); - } - - @Override - public void saveSinkTableLineage(TableLineageEntity entity) { - assertThat(entity.getJob()).isEqualTo("insert_t_job"); - assertThat(entity.getTable()).isEqualTo("T"); - assertThat(entity.getDatabase()).isEqualTo("default"); - jobSinkTableLineages - .computeIfAbsent(entity.getJob(), key -> new HashMap<>()) - .put(getTableLineageKey(entity), entity); - } - - @Override - public Iterator sinkTableLineages(@Nullable Predicate predicate) { - return jobSinkTableLineages.values().stream() - .flatMap(v -> v.values().stream()) - .iterator(); - } - - @Override - public void deleteSinkTableLineage(String job) { - jobSinkTableLineages.remove(job); - } - - @Override - public void saveSourceDataLineage(DataLineageEntity entity) { - assertThat(entity.getJob()).isEqualTo("select_t_job"); - assertThat(entity.getTable()).isEqualTo("T"); - assertThat(entity.getDatabase()).isEqualTo("default"); - throw new UnsupportedOperationException("Method saveSinkTableLineage is not supported"); - } - - @Override - public Iterator sourceDataLineages(@Nullable Predicate predicate) { - throw new UnsupportedOperationException(); - } - - @Override - public void saveSinkDataLineage(DataLineageEntity entity) { - throw new UnsupportedOperationException(); - } - - @Override - public Iterator sinkDataLineages(@Nullable Predicate predicate) { - throw new UnsupportedOperationException(); - } - - @Override - public void close() throws Exception {} - } -} diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/LookupJoinITCase.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/LookupJoinITCase.java index 46399f85632a..a6abde57b80c 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/LookupJoinITCase.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/LookupJoinITCase.java @@ -25,6 +25,7 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.EnumSource; +import org.junit.jupiter.params.provider.ValueSource; import java.util.Collections; import java.util.List; @@ -134,45 +135,40 @@ public void testLookup(LookupCacheMode cacheMode) throws Exception { iterator.close(); } - @ParameterizedTest - @EnumSource(LookupCacheMode.class) - public void testLookupIgnoreScanOptions(LookupCacheMode cacheMode) throws Exception { - initTable(cacheMode); - sql("INSERT INTO DIM VALUES (1, 11, 111, 1111), (2, 22, 222, 2222)"); + @Test + public void testLookupIgnoreScanOptions() throws Exception { + sql( + "CREATE TABLE d (\n" + + " pt INT,\n" + + " id INT,\n" + + " data STRING,\n" + + " PRIMARY KEY (pt, id) NOT ENFORCED\n" + + ") PARTITIONED BY (pt) WITH ( 'bucket' = '1', 'continuous.discovery-interval'='1 ms' )"); + sql( + "CREATE TABLE t1 (\n" + + " pt INT,\n" + + " id INT,\n" + + " data STRING,\n" + + " `proctime` AS PROCTIME(),\n" + + " PRIMARY KEY (pt, id) NOT ENFORCED\n" + + ") PARTITIONED BY (pt) with ( 'continuous.discovery-interval'='1 ms' )"); - String scanOption; - if (ThreadLocalRandom.current().nextBoolean()) { - scanOption = "'scan.mode'='latest'"; - } else { - scanOption = "'scan.snapshot-id'='2'"; - } - String query = - "SELECT T.i, D.j, D.k1, D.k2 FROM T LEFT JOIN DIM /*+ OPTIONS(" - + scanOption - + ") */" - + " for system_time as of T.proctime AS D ON T.i = D.i"; - BlockingIterator iterator = BlockingIterator.of(sEnv.executeSql(query).collect()); + sql("INSERT INTO d VALUES (1, 1, 'one'), (2, 2, 'two'), (3, 3, 'three')"); + sql("INSERT INTO t1 VALUES (1, 1, 'one'), (2, 2, 'two'), (3, 3, 'three')"); - sql("INSERT INTO T VALUES (1), (2), (3)"); - List result = iterator.collect(3); - assertThat(result) - .containsExactlyInAnyOrder( - Row.of(1, 11, 111, 1111), - Row.of(2, 22, 222, 2222), - Row.of(3, null, null, null)); + BlockingIterator streamIter = + streamSqlBlockIter( + "SELECT T.pt, T.id, T.data, D.pt, D.id, D.data " + + "FROM t1 AS T LEFT JOIN d /*+ OPTIONS('lookup.dynamic-partition'='max_pt()', 'scan.snapshot-id'='2') */ " + + "FOR SYSTEM_TIME AS OF T.proctime AS D ON T.id = D.id"); - sql("INSERT INTO DIM VALUES (2, 44, 444, 4444), (3, 33, 333, 3333)"); - Thread.sleep(2000); // wait refresh - sql("INSERT INTO T VALUES (1), (2), (3), (4)"); - result = iterator.collect(4); - assertThat(result) + assertThat(streamIter.collect(3)) .containsExactlyInAnyOrder( - Row.of(1, 11, 111, 1111), - Row.of(2, 44, 444, 4444), - Row.of(3, 33, 333, 3333), - Row.of(4, null, null, null)); + Row.of(1, 1, "one", null, null, null), + Row.of(2, 2, "two", null, null, null), + Row.of(3, 3, "three", 3, 3, "three")); - iterator.close(); + streamIter.close(); } @ParameterizedTest @@ -982,4 +978,32 @@ public void testPartialCacheBucketKeyOrder(LookupCacheMode mode) throws Exceptio iterator.close(); } + + @ParameterizedTest + @ValueSource(booleans = {true, false}) + public void testOverwriteDimTable(boolean isPkTable) throws Exception { + sql( + "CREATE TABLE DIM (i INT %s, v int, pt STRING) " + + "PARTITIONED BY (pt) WITH ('continuous.discovery-interval'='1 ms')", + isPkTable ? "PRIMARY KEY NOT ENFORCED" : ""); + + BlockingIterator iterator = + streamSqlBlockIter( + "SELECT T.i, D.v, D.pt FROM T LEFT JOIN DIM FOR SYSTEM_TIME AS OF T.proctime AS D ON T.i = D.i"); + + sql("INSERT INTO DIM VALUES (1, 11, 'A'), (2, 22, 'B')"); + sql("INSERT INTO T VALUES (1), (2)"); + + List result = iterator.collect(2); + assertThat(result).containsExactlyInAnyOrder(Row.of(1, 11, "A"), Row.of(2, 22, "B")); + + sql("INSERT OVERWRITE DIM PARTITION (pt='B') VALUES (3, 33)"); + Thread.sleep(2000); // wait refresh + sql("INSERT INTO T VALUES (3)"); + + result = iterator.collect(1); + assertThat(result).containsExactlyInAnyOrder(Row.of(3, 33, "B")); + + iterator.close(); + } } diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/ObjectTableITCase.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/ObjectTableITCase.java new file mode 100644 index 000000000000..d3ad1d4a52f4 --- /dev/null +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/ObjectTableITCase.java @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.flink; + +import org.apache.paimon.fs.FileIO; +import org.apache.paimon.fs.Path; +import org.apache.paimon.fs.local.LocalFileIO; + +import org.apache.flink.types.Row; +import org.junit.jupiter.api.Test; + +import java.io.IOException; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** ITCase for object table. */ +public class ObjectTableITCase extends CatalogITCaseBase { + + @Test + public void testIllegalObjectTable() { + assertThatThrownBy( + () -> + sql( + "CREATE TABLE T (a INT, b INT, c INT) WITH ('type' = 'object-table')")) + .rootCause() + .hasMessageContaining("Schema of Object Table can be empty or"); + assertThatThrownBy(() -> sql("CREATE TABLE T WITH ('type' = 'object-table')")) + .rootCause() + .hasMessageContaining("Object table should have object-location option."); + } + + @Test + public void testObjectTableRefresh() throws IOException { + Path objectLocation = new Path(path + "/object-location"); + FileIO fileIO = LocalFileIO.create(); + sql( + "CREATE TABLE T WITH ('type' = 'object-table', 'object-location' = '%s')", + objectLocation); + + // add new file + fileIO.overwriteFileUtf8(new Path(objectLocation, "f0"), "1,2,3"); + sql("CALL sys.refresh_object_table('default.T')"); + assertThat(sql("SELECT name, length FROM T")).containsExactlyInAnyOrder(Row.of("f0", 5L)); + + // add new file + fileIO.overwriteFileUtf8(new Path(objectLocation, "f1"), "4,5,6"); + sql("CALL sys.refresh_object_table('default.T')"); + assertThat(sql("SELECT name, length FROM T")) + .containsExactlyInAnyOrder(Row.of("f0", 5L), Row.of("f1", 5L)); + + // delete file + fileIO.deleteQuietly(new Path(objectLocation, "f0")); + sql("CALL sys.refresh_object_table('default.T')"); + assertThat(sql("SELECT name, length FROM T")).containsExactlyInAnyOrder(Row.of("f1", 5L)); + + // time travel + assertThat(sql("SELECT name, length FROM T /*+ OPTIONS('scan.snapshot-id' = '1') */")) + .containsExactlyInAnyOrder(Row.of("f0", 5L)); + + // insert into + assertThatThrownBy(() -> sql("INSERT INTO T SELECT * FROM T")) + .rootCause() + .hasMessageContaining("Object table does not support Write."); + assertThat(sql("SELECT name, length FROM T")).containsExactlyInAnyOrder(Row.of("f1", 5L)); + } + + @Test + public void testObjectTableRefreshInPrivileged() throws IOException { + sql("CALL sys.init_file_based_privilege('root-passwd')"); + + tEnv.executeSql( + String.format( + "CREATE CATALOG rootcat WITH (\n" + + " 'type' = 'paimon',\n" + + " 'warehouse' = '%s',\n" + + " 'user' = 'root',\n" + + " 'password' = 'root-passwd'\n" + + ")", + path)); + tEnv.useCatalog("rootcat"); + + Path objectLocation = new Path(path + "/object-location"); + FileIO fileIO = LocalFileIO.create(); + sql( + "CREATE TABLE T WITH ('type' = 'object-table', 'object-location' = '%s')", + objectLocation); + + // add new file + fileIO.overwriteFileUtf8(new Path(objectLocation, "f0"), "1,2,3"); + sql("CALL sys.refresh_object_table('default.T')"); + assertThat(sql("SELECT name, length FROM T")).containsExactlyInAnyOrder(Row.of("f0", 5L)); + } +} diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/PartialUpdateITCase.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/PartialUpdateITCase.java index 68f109f0427a..76ee8309e8b5 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/PartialUpdateITCase.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/PartialUpdateITCase.java @@ -351,7 +351,7 @@ public void testLocalMerge() { + "d INT," + "PRIMARY KEY (k, d) NOT ENFORCED) PARTITIONED BY (d) " + " WITH ('merge-engine'='partial-update', " - + "'local-merge-buffer-size'='1m'" + + "'local-merge-buffer-size'='5m'" + ");"); sql("INSERT INTO T1 VALUES (1, CAST(NULL AS INT), 1), (2, 1, 1), (1, 2, 1)"); @@ -588,7 +588,7 @@ public void testIgnoreDelete(boolean localMerge) throws Exception { + " 'changelog-producer' = 'lookup'" + ")"); if (localMerge) { - sql("ALTER TABLE ignore_delete SET ('local-merge-buffer-size' = '256 kb')"); + sql("ALTER TABLE ignore_delete SET ('local-merge-buffer-size' = '5m')"); } sql("INSERT INTO ignore_delete VALUES (1, CAST (NULL AS STRING), 'apple')"); @@ -646,4 +646,43 @@ public void testRemoveRecordOnDelete() { assertThat(sql("SELECT * FROM remove_record_on_delete")) .containsExactlyInAnyOrder(Row.of(1, "A", "apache")); } + + @Test + public void testRemoveRecordOnDeleteLookup() throws Exception { + sql( + "CREATE TABLE remove_record_on_delete (pk INT PRIMARY KEY NOT ENFORCED, a STRING, b STRING) WITH (" + + " 'merge-engine' = 'partial-update'," + + " 'partial-update.remove-record-on-delete' = 'true'," + + " 'changelog-producer' = 'lookup'" + + ")"); + + sql("INSERT INTO remove_record_on_delete VALUES (1, CAST (NULL AS STRING), 'apple')"); + + // delete record + sql("DELETE FROM remove_record_on_delete WHERE pk = 1"); + + // batch read + assertThat(sql("SELECT * FROM remove_record_on_delete")).isEmpty(); + + // insert records + sql("INSERT INTO remove_record_on_delete VALUES (1, CAST (NULL AS STRING), 'apache')"); + sql("INSERT INTO remove_record_on_delete VALUES (1, 'A', CAST (NULL AS STRING))"); + + // batch read + assertThat(sql("SELECT * FROM remove_record_on_delete")) + .containsExactlyInAnyOrder(Row.of(1, "A", "apache")); + + // streaming read results has -U + BlockingIterator iterator = + streamSqlBlockIter( + "SELECT * FROM remove_record_on_delete /*+ OPTIONS('scan.timestamp-millis' = '0') */"); + assertThat(iterator.collect(5)) + .containsExactly( + Row.ofKind(RowKind.INSERT, 1, null, "apple"), + Row.ofKind(RowKind.DELETE, 1, null, "apple"), + Row.ofKind(RowKind.INSERT, 1, null, "apache"), + Row.ofKind(RowKind.UPDATE_BEFORE, 1, null, "apache"), + Row.ofKind(RowKind.UPDATE_AFTER, 1, "A", "apache")); + iterator.close(); + } } diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/PreAggregationITCase.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/PreAggregationITCase.java index e4c90695b1b2..b8dfd8f6a86e 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/PreAggregationITCase.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/PreAggregationITCase.java @@ -1172,7 +1172,7 @@ protected List ddl() { + "PRIMARY KEY (k, d) NOT ENFORCED) PARTITIONED BY (d) " + " WITH ('merge-engine'='aggregation', " + "'fields.v.aggregate-function'='sum'," - + "'local-merge-buffer-size'='1m'" + + "'local-merge-buffer-size'='5m'" + ");"); } diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/PredicateITCase.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/PredicateITCase.java index f50f7db60f02..0668c17eea7c 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/PredicateITCase.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/PredicateITCase.java @@ -21,6 +21,8 @@ import org.apache.flink.types.Row; import org.junit.jupiter.api.Test; +import java.util.concurrent.ThreadLocalRandom; + import static org.assertj.core.api.Assertions.assertThat; /** Predicate ITCase. */ @@ -50,6 +52,42 @@ public void testAppendFilterBucket() throws Exception { innerTestAllFields(); } + @Test + public void testIntegerFilter() { + int rand = ThreadLocalRandom.current().nextInt(3); + String fileFormat; + if (rand == 0) { + fileFormat = "avro"; + } else if (rand == 1) { + fileFormat = "parquet"; + } else { + fileFormat = "orc"; + } + + sql( + "CREATE TABLE T (" + + "a TINYINT," + + "b SMALLINT," + + "c INT," + + "d BIGINT" + + ") WITH (" + + "'file.format' = '%s'" + + ")", + fileFormat); + sql( + "INSERT INTO T VALUES (CAST (1 AS TINYINT), CAST (1 AS SMALLINT), 1, 1), " + + "(CAST (2 AS TINYINT), CAST (2 AS SMALLINT), 2, 2)"); + + Row expectedResult = Row.of((byte) 1, (short) 1, 1, 1L); + assertThat(sql("SELECT * FROM T WHERE a = CAST (1 AS TINYINT)")) + .containsExactly(expectedResult); + assertThat(sql("SELECT * FROM T WHERE b = CAST (1 AS SMALLINT)")) + .containsExactly(expectedResult); + assertThat(sql("SELECT * FROM T WHERE c = 1")).containsExactly(expectedResult); + assertThat(sql("SELECT * FROM T WHERE d = CAST (1 AS BIGINT)")) + .containsExactly(expectedResult); + } + private void writeRecords() throws Exception { sql("INSERT INTO T VALUES (1, 2), (3, 4), (5, 6), (7, 8), (9, 10)"); } diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/PrimaryKeyFileStoreTableITCase.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/PrimaryKeyFileStoreTableITCase.java index 2bdd90a963d9..4ee539c4fd27 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/PrimaryKeyFileStoreTableITCase.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/PrimaryKeyFileStoreTableITCase.java @@ -37,8 +37,11 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Timeout; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; import java.io.IOException; +import java.nio.file.Files; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -48,12 +51,16 @@ import java.util.UUID; import java.util.concurrent.ThreadLocalRandom; import java.util.stream.Collectors; +import java.util.stream.Stream; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatCode; /** Tests for changelog table with primary keys. */ public class PrimaryKeyFileStoreTableITCase extends AbstractTestBase { + private static final int TIMEOUT = 180; + // ------------------------------------------------------------------------ // Test Utilities // ------------------------------------------------------------------------ @@ -67,7 +74,7 @@ public void before() throws IOException { ThreadLocalRandom random = ThreadLocalRandom.current(); tableDefaultProperties = new HashMap<>(); if (random.nextBoolean()) { - tableDefaultProperties.put(CoreOptions.LOCAL_MERGE_BUFFER_SIZE.key(), "256 kb"); + tableDefaultProperties.put(CoreOptions.LOCAL_MERGE_BUFFER_SIZE.key(), "5m"); } } @@ -90,12 +97,38 @@ private String createCatalogSql(String catalogName, String warehouse) { catalogName, warehouse, defaultPropertyString); } + private CloseableIterator collect(TableResult result) { + return collect(result, TIMEOUT); + } + + private CloseableIterator collect(TableResult result, int timeout) { + JobClient client = result.getJobClient().get(); + Thread timeoutThread = + new Thread( + () -> { + for (int i = 0; i < timeout; i++) { + try { + Thread.sleep(1000); + if (client.getJobStatus().get().isGloballyTerminalState()) { + return; + } + } catch (Exception e) { + client.cancel(); + throw new RuntimeException(e); + } + } + client.cancel(); + }); + timeoutThread.start(); + return result.collect(); + } + // ------------------------------------------------------------------------ // Constructed Tests // ------------------------------------------------------------------------ @Test - @Timeout(180) + @Timeout(TIMEOUT) public void testFullCompactionTriggerInterval() throws Exception { innerTestChangelogProducing( Arrays.asList( @@ -104,7 +137,7 @@ public void testFullCompactionTriggerInterval() throws Exception { } @Test - @Timeout(180) + @Timeout(TIMEOUT) public void testFullCompactionWithLongCheckpointInterval() throws Exception { // create table TableEnvironment bEnv = tableEnvironmentBuilder().batchMode().parallelism(1).build(); @@ -130,7 +163,7 @@ public void testFullCompactionWithLongCheckpointInterval() throws Exception { .build(); sEnv.executeSql(createCatalogSql("testCatalog", path)); sEnv.executeSql("USE CATALOG testCatalog"); - CloseableIterator it = sEnv.executeSql("SELECT * FROM T").collect(); + CloseableIterator it = collect(sEnv.executeSql("SELECT * FROM T")); // run compact job StreamExecutionEnvironment env = @@ -163,7 +196,7 @@ public void testFullCompactionWithLongCheckpointInterval() throws Exception { } @Test - @Timeout(180) + @Timeout(TIMEOUT) public void testLookupChangelog() throws Exception { innerTestChangelogProducing(Collections.singletonList("'changelog-producer' = 'lookup'")); } @@ -185,7 +218,7 @@ public void testTableReadWriteBranch() throws Exception { + "'bucket' = '2'" + ")"); - CloseableIterator it = sEnv.executeSql("SELECT * FROM T2").collect(); + CloseableIterator it = collect(sEnv.executeSql("SELECT * FROM T2")); // insert data sEnv.executeSql("INSERT INTO T2 VALUES (1, 'A')").await(); @@ -208,7 +241,7 @@ public void testTableReadWriteBranch() throws Exception { sEnv.executeSql("ALTER TABLE T2 SET ('changelog-producer'='full-compaction')"); CloseableIterator branchIt = - sEnv.executeSql("select * from T2 /*+ OPTIONS('branch' = 'branch1') */").collect(); + collect(sEnv.executeSql("select * from T2 /*+ OPTIONS('branch' = 'branch1') */")); // insert data to branch sEnv.executeSql( "INSERT INTO T2/*+ OPTIONS('branch' = 'branch1') */ VALUES (10, 'v10'),(11, 'v11'),(12, 'v12')") @@ -256,7 +289,7 @@ private void innerTestChangelogProducing(List options) throws Exception sEnv.executeSql( "INSERT INTO T SELECT SUM(i) AS k, g AS v FROM `default_catalog`.`default_database`.`S` GROUP BY g"); - CloseableIterator it = sEnv.executeSql("SELECT * FROM T").collect(); + CloseableIterator it = collect(sEnv.executeSql("SELECT * FROM T")); // write initial data sEnv.executeSql( @@ -303,9 +336,7 @@ private void innerTestChangelogProducing(List options) throws Exception public void testBatchJobWithConflictAndRestart() throws Exception { TableEnvironment tEnv = tableEnvironmentBuilder().batchMode().allowRestart(10).build(); tEnv.executeSql( - "CREATE CATALOG mycat WITH ( 'type' = 'paimon', 'warehouse' = '" - + getTempDirPath() - + "' )"); + "CREATE CATALOG mycat WITH ( 'type' = 'paimon', 'warehouse' = '" + path + "' )"); tEnv.executeSql("USE CATALOG mycat"); tEnv.executeSql( "CREATE TABLE t ( k INT, v INT, PRIMARY KEY (k) NOT ENFORCED ) " @@ -326,7 +357,7 @@ public void testBatchJobWithConflictAndRestart() throws Exception { result1.await(); result2.await(); - try (CloseableIterator it = tEnv.executeSql("SELECT * FROM t").collect()) { + try (CloseableIterator it = collect(tEnv.executeSql("SELECT * FROM t"))) { for (int i = 0; i < 3; i++) { assertThat(it).hasNext(); Row row = it.next(); @@ -335,19 +366,287 @@ public void testBatchJobWithConflictAndRestart() throws Exception { } } + @Timeout(TIMEOUT) + @ParameterizedTest() + @ValueSource(booleans = {false, true}) + public void testRecreateTableWithException(boolean isReloadData) throws Exception { + TableEnvironment bEnv = tableEnvironmentBuilder().batchMode().build(); + bEnv.executeSql(createCatalogSql("testCatalog", path + "/warehouse")); + bEnv.executeSql("USE CATALOG testCatalog"); + bEnv.executeSql( + "CREATE TABLE t ( pt INT, k INT, v INT, PRIMARY KEY (pt, k) NOT ENFORCED ) " + + "PARTITIONED BY (pt) " + + "WITH (" + + " 'bucket' = '2'\n" + + " ,'continuous.discovery-interval' = '1s'\n" + + ")"); + + TableEnvironment sEnv = + tableEnvironmentBuilder() + .streamingMode() + .parallelism(4) + .checkpointIntervalMs(1000) + .build(); + sEnv.executeSql(createCatalogSql("testCatalog", path + "/warehouse")); + sEnv.executeSql("USE CATALOG testCatalog"); + CloseableIterator it = collect(sEnv.executeSql("SELECT * FROM t")); + + // first write + List values = new ArrayList<>(); + for (int i = 0; i < 10; i++) { + values.add(String.format("(0, %d, %d)", i, i)); + values.add(String.format("(1, %d, %d)", i, i)); + } + bEnv.executeSql("INSERT INTO t VALUES " + String.join(", ", values)).await(); + List expected = new ArrayList<>(); + for (int i = 0; i < 10; i++) { + expected.add(Row.ofKind(RowKind.INSERT, 0, i, i)); + expected.add(Row.ofKind(RowKind.INSERT, 1, i, i)); + } + assertStreamingResult(it, expected); + + // second write + values.clear(); + for (int i = 0; i < 10; i++) { + values.add(String.format("(0, %d, %d)", i, i + 1)); + values.add(String.format("(1, %d, %d)", i, i + 1)); + } + bEnv.executeSql("INSERT INTO t VALUES " + String.join(", ", values)).await(); + + // start a read job + for (int i = 0; i < 10; i++) { + expected.add(Row.ofKind(RowKind.UPDATE_BEFORE, 0, i, i)); + expected.add(Row.ofKind(RowKind.UPDATE_BEFORE, 1, i, i)); + expected.add(Row.ofKind(RowKind.UPDATE_AFTER, 0, i, i + 1)); + expected.add(Row.ofKind(RowKind.UPDATE_AFTER, 1, i, i + 1)); + } + assertStreamingResult(it, expected.subList(20, 60)); + + // delete table and recreate a same table + bEnv.executeSql("DROP TABLE t"); + bEnv.executeSql( + "CREATE TABLE t ( pt INT, k INT, v INT, PRIMARY KEY (pt, k) NOT ENFORCED ) " + + "PARTITIONED BY (pt) " + + "WITH (" + + " 'bucket' = '2'\n" + + ")"); + + // if reload data, it will generate a new snapshot for recreated table + if (isReloadData) { + bEnv.executeSql("INSERT INTO t VALUES " + String.join(", ", values)).await(); + } + assertThatCode(it::next) + .rootCause() + .hasMessageContaining( + "The next expected snapshot is too big! Most possible cause might be the table had been recreated."); + } + + @Test + @Timeout(TIMEOUT) + public void testChangelogCompactInBatchWrite() throws Exception { + TableEnvironment bEnv = tableEnvironmentBuilder().batchMode().build(); + String catalogDdl = + "CREATE CATALOG mycat WITH ( 'type' = 'paimon', 'warehouse' = '" + path + "' )"; + bEnv.executeSql(catalogDdl); + bEnv.executeSql("USE CATALOG mycat"); + bEnv.executeSql( + "CREATE TABLE t ( pt INT, k INT, v INT, PRIMARY KEY (pt, k) NOT ENFORCED ) " + + "PARTITIONED BY (pt) " + + "WITH (" + + " 'bucket' = '10',\n" + + " 'changelog-producer' = 'lookup',\n" + + " 'changelog.precommit-compact' = 'true',\n" + + " 'snapshot.num-retained.min' = '3',\n" + + " 'snapshot.num-retained.max' = '3'\n" + + ")"); + + TableEnvironment sEnv = + tableEnvironmentBuilder().streamingMode().checkpointIntervalMs(1000).build(); + sEnv.executeSql(catalogDdl); + sEnv.executeSql("USE CATALOG mycat"); + + List values = new ArrayList<>(); + for (int i = 0; i < 1000; i++) { + values.add(String.format("(0, %d, %d)", i, i)); + values.add(String.format("(1, %d, %d)", i, i)); + } + bEnv.executeSql("INSERT INTO t VALUES " + String.join(", ", values)).await(); + + List compactedChangelogs2 = listAllFilesWithPrefix("compacted-changelog-"); + assertThat(compactedChangelogs2).hasSize(2); + assertThat(listAllFilesWithPrefix("changelog-")).isEmpty(); + + List expected = new ArrayList<>(); + for (int i = 0; i < 1000; i++) { + expected.add(Row.ofKind(RowKind.INSERT, 0, i, i)); + expected.add(Row.ofKind(RowKind.INSERT, 1, i, i)); + } + assertStreamingResult( + sEnv.executeSql("SELECT * FROM t /*+ OPTIONS('scan.snapshot-id' = '1') */"), + expected); + + values.clear(); + for (int i = 0; i < 1000; i++) { + values.add(String.format("(0, %d, %d)", i, i + 1)); + values.add(String.format("(1, %d, %d)", i, i + 1)); + } + bEnv.executeSql("INSERT INTO t VALUES " + String.join(", ", values)).await(); + + assertThat(listAllFilesWithPrefix("compacted-changelog-")).hasSize(4); + assertThat(listAllFilesWithPrefix("changelog-")).isEmpty(); + + for (int i = 0; i < 1000; i++) { + expected.add(Row.ofKind(RowKind.UPDATE_BEFORE, 0, i, i)); + expected.add(Row.ofKind(RowKind.UPDATE_BEFORE, 1, i, i)); + expected.add(Row.ofKind(RowKind.UPDATE_AFTER, 0, i, i + 1)); + expected.add(Row.ofKind(RowKind.UPDATE_AFTER, 1, i, i + 1)); + } + assertStreamingResult( + sEnv.executeSql("SELECT * FROM t /*+ OPTIONS('scan.snapshot-id' = '1') */"), + expected); + + values.clear(); + for (int i = 0; i < 1000; i++) { + values.add(String.format("(0, %d, %d)", i, i + 2)); + values.add(String.format("(1, %d, %d)", i, i + 2)); + } + bEnv.executeSql("INSERT INTO t VALUES " + String.join(", ", values)).await(); + + assertThat(listAllFilesWithPrefix("compacted-changelog-")).hasSize(4); + assertThat(listAllFilesWithPrefix("changelog-")).isEmpty(); + LocalFileIO fileIO = LocalFileIO.create(); + for (String p : compactedChangelogs2) { + assertThat(fileIO.exists(new Path(p))).isFalse(); + } + + expected = expected.subList(2000, 6000); + for (int i = 0; i < 1000; i++) { + expected.add(Row.ofKind(RowKind.UPDATE_BEFORE, 0, i, i + 1)); + expected.add(Row.ofKind(RowKind.UPDATE_BEFORE, 1, i, i + 1)); + expected.add(Row.ofKind(RowKind.UPDATE_AFTER, 0, i, i + 2)); + expected.add(Row.ofKind(RowKind.UPDATE_AFTER, 1, i, i + 2)); + } + assertStreamingResult( + sEnv.executeSql("SELECT * FROM t /*+ OPTIONS('scan.snapshot-id' = '1') */"), + expected); + } + + @Test + @Timeout(TIMEOUT) + public void testChangelogCompactInStreamWrite() throws Exception { + TableEnvironment sEnv = + tableEnvironmentBuilder() + .streamingMode() + .checkpointIntervalMs(2000) + .parallelism(4) + .build(); + + sEnv.executeSql(createCatalogSql("testCatalog", path + "/warehouse")); + sEnv.executeSql("USE CATALOG testCatalog"); + sEnv.executeSql( + "CREATE TABLE t ( pt INT, k INT, v INT, PRIMARY KEY (pt, k) NOT ENFORCED ) " + + "PARTITIONED BY (pt) " + + "WITH (" + + " 'bucket' = '10',\n" + + " 'changelog-producer' = 'lookup',\n" + + " 'changelog.precommit-compact' = 'true'\n" + + ")"); + + Path inputPath = new Path(path, "input"); + LocalFileIO.create().mkdirs(inputPath); + sEnv.executeSql( + "CREATE TABLE `default_catalog`.`default_database`.`s` ( pt INT, k INT, v INT, PRIMARY KEY (pt, k) NOT ENFORCED) " + + "WITH ( 'connector' = 'filesystem', 'format' = 'testcsv', 'path' = '" + + inputPath + + "', 'source.monitor-interval' = '500ms' )"); + + sEnv.executeSql("INSERT INTO t SELECT * FROM `default_catalog`.`default_database`.`s`"); + CloseableIterator it = collect(sEnv.executeSql("SELECT * FROM t")); + + // write initial data + List values = new ArrayList<>(); + for (int i = 0; i < 100; i++) { + values.add(String.format("(0, %d, %d)", i, i)); + values.add(String.format("(1, %d, %d)", i, i)); + } + sEnv.executeSql( + "INSERT INTO `default_catalog`.`default_database`.`s` VALUES " + + String.join(", ", values)) + .await(); + + List expected = new ArrayList<>(); + for (int i = 0; i < 100; i++) { + expected.add(Row.ofKind(RowKind.INSERT, 0, i, i)); + expected.add(Row.ofKind(RowKind.INSERT, 1, i, i)); + } + assertStreamingResult(it, expected); + + List compactedChangelogs2 = listAllFilesWithPrefix("compacted-changelog-"); + assertThat(compactedChangelogs2).hasSize(2); + assertThat(listAllFilesWithPrefix("changelog-")).isEmpty(); + + // write update data + values.clear(); + for (int i = 0; i < 100; i++) { + values.add(String.format("(0, %d, %d)", i, i + 1)); + values.add(String.format("(1, %d, %d)", i, i + 1)); + } + sEnv.executeSql( + "INSERT INTO `default_catalog`.`default_database`.`s` VALUES " + + String.join(", ", values)) + .await(); + for (int i = 0; i < 100; i++) { + expected.add(Row.ofKind(RowKind.UPDATE_BEFORE, 0, i, i)); + expected.add(Row.ofKind(RowKind.UPDATE_BEFORE, 1, i, i)); + expected.add(Row.ofKind(RowKind.UPDATE_AFTER, 0, i, i + 1)); + expected.add(Row.ofKind(RowKind.UPDATE_AFTER, 1, i, i + 1)); + } + assertStreamingResult(it, expected.subList(200, 600)); + assertThat(listAllFilesWithPrefix("compacted-changelog-")).hasSize(4); + assertThat(listAllFilesWithPrefix("changelog-")).isEmpty(); + } + + private List listAllFilesWithPrefix(String prefix) throws Exception { + try (Stream stream = Files.walk(java.nio.file.Paths.get(path))) { + return stream.filter(Files::isRegularFile) + .filter(p -> p.getFileName().toString().startsWith(prefix)) + .map(java.nio.file.Path::toString) + .collect(Collectors.toList()); + } + } + + private void assertStreamingResult(TableResult result, List expected) throws Exception { + List actual = new ArrayList<>(); + try (CloseableIterator it = collect(result)) { + while (actual.size() < expected.size() && it.hasNext()) { + actual.add(it.next()); + } + } + assertThat(actual).hasSameElementsAs(expected); + } + + private void assertStreamingResult(CloseableIterator it, List expected) { + List actual = new ArrayList<>(); + while (actual.size() < expected.size() && it.hasNext()) { + actual.add(it.next()); + } + + assertThat(actual).hasSameElementsAs(expected); + } + // ------------------------------------------------------------------------ // Random Tests // ------------------------------------------------------------------------ @Test - @Timeout(180) + @Timeout(TIMEOUT) public void testNoChangelogProducerBatchRandom() throws Exception { TableEnvironment bEnv = tableEnvironmentBuilder().batchMode().build(); testNoChangelogProducerRandom(bEnv, 1, false); } @Test - @Timeout(180) + @Timeout(TIMEOUT) public void testNoChangelogProducerStreamingRandom() throws Exception { ThreadLocalRandom random = ThreadLocalRandom.current(); TableEnvironment sEnv = @@ -360,14 +659,14 @@ public void testNoChangelogProducerStreamingRandom() throws Exception { } @Test - @Timeout(180) + @Timeout(TIMEOUT) public void testFullCompactionChangelogProducerBatchRandom() throws Exception { TableEnvironment bEnv = tableEnvironmentBuilder().batchMode().build(); testFullCompactionChangelogProducerRandom(bEnv, 1, false); } @Test - @Timeout(180) + @Timeout(TIMEOUT) public void testFullCompactionChangelogProducerStreamingRandom() throws Exception { ThreadLocalRandom random = ThreadLocalRandom.current(); TableEnvironment sEnv = @@ -380,7 +679,7 @@ public void testFullCompactionChangelogProducerStreamingRandom() throws Exceptio } @Test - @Timeout(180) + @Timeout(TIMEOUT) public void testStandAloneFullCompactJobRandom() throws Exception { ThreadLocalRandom random = ThreadLocalRandom.current(); TableEnvironment sEnv = @@ -393,14 +692,14 @@ public void testStandAloneFullCompactJobRandom() throws Exception { } @Test - @Timeout(180) + @Timeout(TIMEOUT) public void testLookupChangelogProducerBatchRandom() throws Exception { TableEnvironment bEnv = tableEnvironmentBuilder().batchMode().build(); testLookupChangelogProducerRandom(bEnv, 1, false); } @Test - @Timeout(180) + @Timeout(TIMEOUT) public void testLookupChangelogProducerStreamingRandom() throws Exception { ThreadLocalRandom random = ThreadLocalRandom.current(); TableEnvironment sEnv = @@ -413,7 +712,7 @@ public void testLookupChangelogProducerStreamingRandom() throws Exception { } @Test - @Timeout(180) + @Timeout(TIMEOUT) public void testStandAloneLookupJobRandom() throws Exception { ThreadLocalRandom random = ThreadLocalRandom.current(); TableEnvironment sEnv = @@ -488,15 +787,17 @@ private void testLookupChangelogProducerRandom( tEnv, numProducers, enableFailure, - "'bucket' = '4'," - + String.format( - "'write-buffer-size' = '%s'," - + "'changelog-producer' = 'lookup'," - + "'lookup-wait' = '%s'," - + "'deletion-vectors.enabled' = '%s'", - random.nextBoolean() ? "4mb" : "8mb", - random.nextBoolean(), - enableDeletionVectors)); + String.format( + "'bucket' = '4', " + + "'writer-buffer-size' = '%s', " + + "'changelog-producer' = 'lookup', " + + "'lookup-wait' = '%s', " + + "'deletion-vectors.enabled' = '%s', " + + "'changelog.precommit-compact' = '%s'", + random.nextBoolean() ? "4mb" : "8mb", + random.nextBoolean(), + enableDeletionVectors, + random.nextBoolean())); // sleep for a random amount of time to check // if we can first read complete records then read incremental records correctly @@ -595,7 +896,7 @@ private void checkChangelogTestResult(int numProducers) throws Exception { ResultChecker checker = new ResultChecker(); int endCnt = 0; - try (CloseableIterator it = sEnv.executeSql("SELECT * FROM T").collect()) { + try (CloseableIterator it = collect(sEnv.executeSql("SELECT * FROM T"))) { while (it.hasNext()) { Row row = it.next(); checker.addChangelog(row); @@ -713,7 +1014,7 @@ private void checkBatchResult(int numProducers) throws Exception { bEnv.executeSql("USE CATALOG testCatalog"); ResultChecker checker = new ResultChecker(); - try (CloseableIterator it = bEnv.executeSql("SELECT * FROM T").collect()) { + try (CloseableIterator it = collect(bEnv.executeSql("SELECT * FROM T"))) { while (it.hasNext()) { checker.addChangelog(it.next()); } diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/ReadWriteTableITCase.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/ReadWriteTableITCase.java index d1e9b23e1b54..732e96454236 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/ReadWriteTableITCase.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/ReadWriteTableITCase.java @@ -76,7 +76,6 @@ import static org.apache.paimon.CoreOptions.MergeEngine.FIRST_ROW; import static org.apache.paimon.CoreOptions.SOURCE_SPLIT_OPEN_FILE_COST; import static org.apache.paimon.CoreOptions.SOURCE_SPLIT_TARGET_SIZE; -import static org.apache.paimon.flink.AbstractFlinkTableFactory.buildPaimonTable; import static org.apache.paimon.flink.FlinkConnectorOptions.INFER_SCAN_MAX_PARALLELISM; import static org.apache.paimon.flink.FlinkConnectorOptions.INFER_SCAN_PARALLELISM; import static org.apache.paimon.flink.FlinkConnectorOptions.SCAN_PARALLELISM; @@ -802,6 +801,43 @@ public void testStreamingReadOverwriteWithoutPartitionedRecords() throws Excepti streamingItr.close(); } + @Test + public void testStreamingReadOverwriteWithDeleteRecords() throws Exception { + String table = + createTable( + Arrays.asList("currency STRING", "rate BIGINT", "dt STRING"), + Collections.singletonList("currency"), + Collections.emptyList(), + Collections.emptyList(), + streamingReadOverwrite); + + insertInto( + table, + "('US Dollar', 102, '2022-01-01')", + "('Yen', 1, '2022-01-02')", + "('Euro', 119, '2022-01-02')"); + + bEnv.executeSql(String.format("DELETE FROM %s WHERE currency = 'Euro'", table)).await(); + + checkFileStorePath(table, Collections.emptyList()); + + // test projection and filter + BlockingIterator streamingItr = + testStreamingRead( + buildQuery(table, "currency, rate", "WHERE dt = '2022-01-02'"), + Collections.singletonList(changelogRow("+I", "Yen", 1L))); + + insertOverwrite(table, "('US Dollar', 100, '2022-01-02')", "('Yen', 10, '2022-01-01')"); + + validateStreamingReadResult( + streamingItr, + Arrays.asList( + changelogRow("-D", "Yen", 1L), changelogRow("+I", "US Dollar", 100L))); + assertNoMoreRecords(streamingItr); + + streamingItr.close(); + } + @Test public void testUnsupportStreamingReadOverwriteWithoutPk() { assertThatThrownBy( @@ -1827,7 +1863,10 @@ private void testSinkParallelism(Integer configParallelism, int expectedParallel DynamicTableSink tableSink = new FlinkTableSink( - context.getObjectIdentifier(), buildPaimonTable(context), context, null); + context.getObjectIdentifier(), + new FlinkTableFactory().buildPaimonTable(context), + context, + null); assertThat(tableSink).isInstanceOf(FlinkTableSink.class); // 2. get sink provider diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/RescaleBucketITCase.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/RescaleBucketITCase.java index 08969bddfdb3..d5747d2e28d4 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/RescaleBucketITCase.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/RescaleBucketITCase.java @@ -26,7 +26,6 @@ import org.apache.flink.core.execution.JobClient; import org.apache.flink.core.execution.SavepointFormatType; -import org.apache.flink.runtime.jobgraph.SavepointConfigOptions; import org.apache.flink.types.Row; import org.junit.jupiter.api.Test; @@ -106,9 +105,10 @@ public void testSuspendAndRecoverAfterRescaleOverwrite() throws Exception { assertThat(batchSql("SELECT * FROM T3")).containsExactlyInAnyOrderElementsOf(committedData); // step5: resume streaming job + // use config string to stay compatible with flink 1.19- sEnv.getConfig() .getConfiguration() - .set(SavepointConfigOptions.SAVEPOINT_PATH, savepointPath); + .setString("execution.state-recovery.path", savepointPath); JobClient resumedJobClient = startJobAndCommitSnapshot(streamSql, snapshotAfterRescale.id()); // stop job diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/SchemaChangeITCase.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/SchemaChangeITCase.java index e1655bcb6ba9..a8e8332156b3 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/SchemaChangeITCase.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/SchemaChangeITCase.java @@ -25,8 +25,11 @@ import org.apache.flink.table.api.config.ExecutionConfigOptions; import org.apache.flink.types.Row; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; import java.time.format.DateTimeFormatter; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; @@ -35,6 +38,7 @@ import static org.apache.paimon.testutils.assertj.PaimonAssertions.anyCauseMatches; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatCode; import static org.assertj.core.api.Assertions.assertThatThrownBy; /** ITCase for schema changes. */ @@ -248,7 +252,8 @@ public void testModifyColumnTypeFromNumericToDecimal() { public void testModifyColumnTypeBooleanAndNumeric() { // boolean To numeric and numeric To boolean sql("CREATE TABLE T (a BOOLEAN, b BOOLEAN, c TINYINT, d INT, e BIGINT, f DOUBLE)"); - sql("INSERT INTO T VALUES(true, false, cast(0 as TINYINT), 1 , 123, 3.14)"); + sql( + "INSERT INTO T VALUES(true, false, cast(0 as TINYINT), 1 , -9223372036854775808, 3.14)"); sql("ALTER TABLE T MODIFY (a TINYINT, b INT, c BOOLEAN, d BOOLEAN, e BOOLEAN)"); List result = sql("SHOW CREATE TABLE T"); @@ -1007,6 +1012,33 @@ public void testAlterTableNonPhysicalColumn() { .doesNotContain("schema"); } + @Test + public void testSequenceFieldSortOrder() { + // test default condition which get the largest record + sql( + "CREATE TABLE T1 (a STRING PRIMARY KEY NOT ENFORCED, b STRING, c STRING) WITH ('sequence.field'='c')"); + sql("INSERT INTO T1 VALUES ('a', 'b', 'l')"); + sql("INSERT INTO T1 VALUES ('a', 'd', 'n')"); + sql("INSERT INTO T1 VALUES ('a', 'e', 'm')"); + assertThat(sql("select * from T1").toString()).isEqualTo("[+I[a, d, n]]"); + + // test for get small record + sql( + "CREATE TABLE T2 (a STRING PRIMARY KEY NOT ENFORCED, b STRING, c BIGINT) WITH ('sequence.field'='c', 'sequence.field.sort-order'='descending')"); + sql("INSERT INTO T2 VALUES ('a', 'b', 1)"); + sql("INSERT INTO T2 VALUES ('a', 'd', 3)"); + sql("INSERT INTO T2 VALUES ('a', 'e', 2)"); + assertThat(sql("select * from T2").toString()).isEqualTo("[+I[a, b, 1]]"); + + // test for get largest record + sql( + "CREATE TABLE T3 (a STRING PRIMARY KEY NOT ENFORCED, b STRING, c DOUBLE) WITH ('sequence.field'='c', 'sequence.field.sort-order'='ascending')"); + sql("INSERT INTO T3 VALUES ('a', 'b', 1.0)"); + sql("INSERT INTO T3 VALUES ('a', 'd', 3.0)"); + sql("INSERT INTO T3 VALUES ('a', 'e', 2.0)"); + assertThat(sql("select * from T3").toString()).isEqualTo("[+I[a, d, 3.0]]"); + } + @Test public void testAlterTableMetadataComment() { sql("CREATE TABLE T (a INT, name VARCHAR METADATA COMMENT 'header1', b INT)"); @@ -1059,4 +1091,112 @@ public void testAlterBucket() { UnsupportedOperationException.class, "Cannot change bucket to -1.")); } + + @ParameterizedTest() + @ValueSource(strings = {"orc", "avro", "parquet"}) + public void testUpdateNestedColumn(String formatType) { + sql( + "CREATE TABLE T " + + "( k INT, v ROW(f1 INT, f2 ROW(f1 STRING, f2 INT NOT NULL)), PRIMARY KEY (k) NOT ENFORCED ) " + + "WITH ( 'bucket' = '1', 'file.format' = '" + + formatType + + "' )"); + sql( + "INSERT INTO T VALUES (1, ROW(10, ROW('apple', 100))), (2, ROW(20, ROW('banana', 200)))"); + assertThat(sql("SELECT * FROM T")) + .containsExactlyInAnyOrder( + Row.of(1, Row.of(10, Row.of("apple", 100))), + Row.of(2, Row.of(20, Row.of("banana", 200)))); + + sql("ALTER TABLE T MODIFY (v ROW(f1 BIGINT, f2 ROW(f3 DOUBLE, f2 INT), f3 STRING))"); + sql( + "INSERT INTO T VALUES " + + "(1, ROW(1000000000001, ROW(101.0, 101), 'cat')), " + + "(3, ROW(3000000000001, ROW(301.0, CAST(NULL AS INT)), 'dog'))"); + assertThat(sql("SELECT * FROM T")) + .containsExactlyInAnyOrder( + Row.of(1, Row.of(1000000000001L, Row.of(101.0, 101), "cat")), + Row.of(2, Row.of(20L, Row.of(null, 200), null)), + Row.of(3, Row.of(3000000000001L, Row.of(301.0, null), "dog"))); + + sql( + "ALTER TABLE T MODIFY (v ROW(f1 BIGINT, f2 ROW(f3 DOUBLE, f1 STRING, f2 INT), f3 STRING))"); + sql( + "INSERT INTO T VALUES " + + "(1, ROW(1000000000002, ROW(102.0, 'APPLE', 102), 'cat')), " + + "(4, ROW(4000000000002, ROW(402.0, 'LEMON', 402), 'tiger'))"); + assertThat(sql("SELECT k, v.f2.f1, v.f3 FROM T")) + .containsExactlyInAnyOrder( + Row.of(1, "APPLE", "cat"), + Row.of(2, null, null), + Row.of(3, null, "dog"), + Row.of(4, "LEMON", "tiger")); + + assertThatCode(() -> sql("ALTER TABLE T MODIFY (v ROW(f1 BIGINT, f2 INT, f3 STRING))")) + .hasRootCauseMessage( + "Column v.f2 can only be updated to row type, and cannot be updated to INTEGER type"); + } + + @ParameterizedTest() + @ValueSource(strings = {"orc", "avro", "parquet"}) + public void testUpdateRowInArrayAndMap(String formatType) { + sql( + "CREATE TABLE T " + + "( k INT, v1 ARRAY, v2 MAP, PRIMARY KEY (k) NOT ENFORCED ) " + + "WITH ( 'bucket' = '1', 'file.format' = '" + + formatType + + "' )"); + sql( + "INSERT INTO T VALUES " + + "(1, ARRAY[ROW(100, 'apple'), ROW(101, 'banana')], MAP[100, ROW('cat', 1000), 101, ROW('dog', 1001)]), " + + "(2, ARRAY[ROW(200, 'pear'), ROW(201, 'grape')], MAP[200, ROW('tiger', 2000), 201, ROW('wolf', 2001)])"); + + Map map1 = new HashMap<>(); + map1.put(100, Row.of("cat", 1000)); + map1.put(101, Row.of("dog", 1001)); + Map map2 = new HashMap<>(); + map2.put(200, Row.of("tiger", 2000)); + map2.put(201, Row.of("wolf", 2001)); + assertThat(sql("SELECT * FROM T")) + .containsExactlyInAnyOrder( + Row.of(1, new Row[] {Row.of(100, "apple"), Row.of(101, "banana")}, map1), + Row.of(2, new Row[] {Row.of(200, "pear"), Row.of(201, "grape")}, map2)); + + sql( + "ALTER TABLE T MODIFY (v1 ARRAY, v2 MAP)"); + sql( + "INSERT INTO T VALUES " + + "(1, ARRAY[ROW(1000000000000, 'apple', 'A'), ROW(1000000000001, 'banana', 'B')], MAP[100, ROW(1000.0, 1000), 101, ROW(1001.0, 1001)]), " + + "(3, ARRAY[ROW(3000000000000, 'mango', 'M'), ROW(3000000000001, 'cherry', 'C')], MAP[300, ROW(3000.0, 3000), 301, ROW(3001.0, 3001)])"); + + map1.clear(); + map1.put(100, Row.of(1000.0, 1000)); + map1.put(101, Row.of(1001.0, 1001)); + map2.clear(); + map2.put(200, Row.of(null, 2000)); + map2.put(201, Row.of(null, 2001)); + Map map3 = new HashMap<>(); + map3.put(300, Row.of(3000.0, 3000)); + map3.put(301, Row.of(3001.0, 3001)); + assertThat(sql("SELECT v2, v1, k FROM T")) + .containsExactlyInAnyOrder( + Row.of( + map1, + new Row[] { + Row.of(1000000000000L, "apple", "A"), + Row.of(1000000000001L, "banana", "B") + }, + 1), + Row.of( + map2, + new Row[] {Row.of(200L, "pear", null), Row.of(201L, "grape", null)}, + 2), + Row.of( + map3, + new Row[] { + Row.of(3000000000000L, "mango", "M"), + Row.of(3000000000001L, "cherry", "C") + }, + 3)); + } } diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/SerializableRowData.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/SerializableRowData.java index 594affc124eb..75b96cbe02eb 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/SerializableRowData.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/SerializableRowData.java @@ -47,8 +47,10 @@ public SerializableRowData(RowData row, TypeSerializer serializer) { this.serializer = serializer; } - private void writeObject(ObjectOutputStream out) throws IOException { + private synchronized void writeObject(ObjectOutputStream out) throws IOException { out.defaultWriteObject(); + // This following invocation needs to be synchronized to avoid racing problems when the + // serializer is reused across multiple subtasks. serializer.serialize(row, new DataOutputViewStreamWrapper(out)); } diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/SystemTableITCase.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/SystemTableITCase.java new file mode 100644 index 000000000000..771f4acc5e58 --- /dev/null +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/SystemTableITCase.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.flink; + +import org.apache.paimon.utils.BlockingIterator; + +import org.apache.flink.types.Row; +import org.junit.jupiter.api.Test; + +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; + +/** ITCase for system table. */ +public class SystemTableITCase extends CatalogTableITCase { + + @Test + public void testBinlogTableStreamRead() throws Exception { + sql( + "CREATE TABLE T (a INT, b INT, primary key (a) NOT ENFORCED) with ('changelog-producer' = 'lookup', " + + "'bucket' = '2')"); + BlockingIterator iterator = + streamSqlBlockIter("SELECT * FROM T$binlog /*+ OPTIONS('scan.mode' = 'latest') */"); + sql("INSERT INTO T VALUES (1, 2)"); + sql("INSERT INTO T VALUES (1, 3)"); + sql("INSERT INTO T VALUES (2, 2)"); + List rows = iterator.collect(3); + assertThat(rows) + .containsExactly( + Row.of("+I", new Integer[] {1}, new Integer[] {2}), + Row.of("+U", new Integer[] {1, 1}, new Integer[] {2, 3}), + Row.of("+I", new Integer[] {2}, new Integer[] {2})); + iterator.close(); + } + + @Test + public void testBinlogTableBatchRead() throws Exception { + sql( + "CREATE TABLE T (a INT, b INT, primary key (a) NOT ENFORCED) with ('changelog-producer' = 'lookup', " + + "'bucket' = '2')"); + sql("INSERT INTO T VALUES (1, 2)"); + sql("INSERT INTO T VALUES (1, 3)"); + sql("INSERT INTO T VALUES (2, 2)"); + List rows = sql("SELECT * FROM T$binlog /*+ OPTIONS('scan.mode' = 'latest') */"); + assertThat(rows) + .containsExactly( + Row.of("+I", new Integer[] {1}, new Integer[] {3}), + Row.of("+I", new Integer[] {2}, new Integer[] {2})); + } +} diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/UnawareBucketAppendOnlyTableITCase.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/UnawareBucketAppendOnlyTableITCase.java index cb323542d4c1..fb8bee5d5962 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/UnawareBucketAppendOnlyTableITCase.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/UnawareBucketAppendOnlyTableITCase.java @@ -20,6 +20,9 @@ import org.apache.paimon.Snapshot; import org.apache.paimon.data.InternalRow; +import org.apache.paimon.flink.source.AbstractNonCoordinatedSource; +import org.apache.paimon.flink.source.AbstractNonCoordinatedSourceReader; +import org.apache.paimon.flink.source.SimpleSourceSplit; import org.apache.paimon.fs.Path; import org.apache.paimon.fs.local.LocalFileIO; import org.apache.paimon.reader.RecordReader; @@ -27,10 +30,16 @@ import org.apache.paimon.table.FileStoreTable; import org.apache.paimon.table.FileStoreTableFactory; import org.apache.paimon.utils.FailingFileIO; - +import org.apache.paimon.utils.TimeUtils; + +import org.apache.flink.api.common.eventtime.WatermarkStrategy; +import org.apache.flink.api.connector.source.Boundedness; +import org.apache.flink.api.connector.source.ReaderOutput; +import org.apache.flink.api.connector.source.SourceReader; +import org.apache.flink.api.connector.source.SourceReaderContext; +import org.apache.flink.core.io.InputStatus; import org.apache.flink.streaming.api.datastream.DataStream; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; -import org.apache.flink.streaming.api.functions.source.RichParallelSourceFunction; import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; import org.apache.flink.table.planner.factories.TestValuesTableFactory; import org.apache.flink.types.Row; @@ -49,7 +58,6 @@ import java.util.List; import java.util.Random; -import static org.apache.flink.streaming.api.environment.ExecutionCheckpointingOptions.CHECKPOINTING_INTERVAL; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -202,7 +210,11 @@ public void testCompactionInStreamingMode() throws Exception { batchSql("ALTER TABLE append_table SET ('compaction.early-max.file-num' = '4')"); batchSql("ALTER TABLE append_table SET ('continuous.discovery-interval' = '1 s')"); - sEnv.getConfig().getConfiguration().set(CHECKPOINTING_INTERVAL, Duration.ofMillis(500)); + sEnv.getConfig() + .getConfiguration() + .setString( + "execution.checkpointing.interval", + TimeUtils.formatWithHighestUnit(Duration.ofMillis(500))); sEnv.executeSql( "CREATE TEMPORARY TABLE Orders_in (\n" + " f0 INT,\n" @@ -223,7 +235,11 @@ public void testCompactionInStreamingModeWithMaxWatermark() throws Exception { batchSql("ALTER TABLE append_table SET ('compaction.early-max.file-num' = '4')"); batchSql("ALTER TABLE append_table SET ('continuous.discovery-interval' = '1 s')"); - sEnv.getConfig().getConfiguration().set(CHECKPOINTING_INTERVAL, Duration.ofMillis(500)); + sEnv.getConfig() + .getConfiguration() + .setString( + "execution.checkpointing.interval", + TimeUtils.formatWithHighestUnit(Duration.ofMillis(500))); sEnv.executeSql( "CREATE TEMPORARY TABLE Orders_in (\n" + " f0 INT,\n" @@ -371,7 +387,12 @@ public void testStatelessWriter() throws Exception { .checkpointIntervalMs(500) .build(); DataStream source = - env.addSource(new TestStatelessWriterSource(table)).setParallelism(2).forward(); + env.fromSource( + new TestStatelessWriterSource(table), + WatermarkStrategy.noWatermarks(), + "TestStatelessWriterSource") + .setParallelism(2) + .forward(); StreamTableEnvironment tEnv = StreamTableEnvironment.create(env); tEnv.registerCatalog("mycat", sEnv.getCatalog("PAIMON").get()); @@ -383,46 +404,59 @@ public void testStatelessWriter() throws Exception { .containsExactlyInAnyOrder(Row.of(1, "test"), Row.of(2, "test")); } - private static class TestStatelessWriterSource extends RichParallelSourceFunction { + private static class TestStatelessWriterSource extends AbstractNonCoordinatedSource { private final FileStoreTable table; - private volatile boolean isRunning = true; - private TestStatelessWriterSource(FileStoreTable table) { this.table = table; } @Override - public void run(SourceContext sourceContext) throws Exception { - int taskId = getRuntimeContext().getIndexOfThisSubtask(); - // wait some time in parallelism #2, - // so that it does not commit in the same checkpoint with parallelism #1 - int waitCount = (taskId == 0 ? 0 : 10); - - while (isRunning) { - synchronized (sourceContext.getCheckpointLock()) { - if (taskId == 0) { + public Boundedness getBoundedness() { + return Boundedness.CONTINUOUS_UNBOUNDED; + } + + @Override + public SourceReader createReader( + SourceReaderContext sourceReaderContext) throws Exception { + return new Reader(sourceReaderContext.getIndexOfSubtask()); + } + + private class Reader extends AbstractNonCoordinatedSourceReader { + private final int taskId; + private int waitCount; + + private Reader(int taskId) { + this.taskId = taskId; + this.waitCount = (taskId == 0 ? 0 : 10); + } + + @Override + public InputStatus pollNext(ReaderOutput readerOutput) throws Exception { + if (taskId == 0) { + if (waitCount == 0) { + readerOutput.collect(1); + } else if (countNumRecords() >= 1) { + // wait for the record to commit before exiting + Thread.sleep(1000); + return InputStatus.END_OF_INPUT; + } + } else { + int numRecords = countNumRecords(); + if (numRecords >= 1) { if (waitCount == 0) { - sourceContext.collect(1); - } else if (countNumRecords() >= 1) { - // wait for the record to commit before exiting - break; - } - } else { - int numRecords = countNumRecords(); - if (numRecords >= 1) { - if (waitCount == 0) { - sourceContext.collect(2); - } else if (countNumRecords() >= 2) { - // make sure the next checkpoint is successful - break; - } + readerOutput.collect(2); + } else if (countNumRecords() >= 2) { + // make sure the next checkpoint is successful + Thread.sleep(1000); + return InputStatus.END_OF_INPUT; } } - waitCount--; } + waitCount--; Thread.sleep(1000); + return InputStatus.MORE_AVAILABLE; } } @@ -438,11 +472,6 @@ private int countNumRecords() throws Exception { } return ret; } - - @Override - public void cancel() { - isRunning = false; - } } @Override diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/action/CloneActionITCase.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/action/CloneActionITCase.java index 71672551abcb..a55b01cc203b 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/action/CloneActionITCase.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/action/CloneActionITCase.java @@ -32,6 +32,7 @@ import org.apache.flink.table.api.config.TableConfigOptions; import org.apache.flink.types.Row; import org.apache.flink.util.CloseableIterator; +import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Timeout; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; @@ -44,8 +45,10 @@ import java.util.stream.Collectors; import java.util.stream.IntStream; +import static org.apache.paimon.testutils.assertj.PaimonAssertions.anyCauseMatches; import static org.apache.paimon.utils.Preconditions.checkState; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; /** IT cases for {@link CloneAction}. */ public class CloneActionITCase extends ActionITCaseBase { @@ -640,6 +643,46 @@ public void testCloneTableWithExpiration(String invoker) throws Exception { .isEqualTo(Collections.singletonList("+I[1]")); } + // ------------------------------------------------------------------------ + // Negative Tests + // ------------------------------------------------------------------------ + + @Test + public void testEmptySourceCatalog() { + String sourceWarehouse = getTempDirPath("source-ware"); + + TableEnvironment tEnv = tableEnvironmentBuilder().batchMode().parallelism(1).build(); + tEnv.executeSql( + "CREATE CATALOG sourcecat WITH (\n" + + " 'type' = 'paimon',\n" + + String.format(" 'warehouse' = '%s'\n", sourceWarehouse) + + ")"); + + String targetWarehouse = getTempDirPath("target-ware"); + + String[] args = + new String[] { + "clone", + "--warehouse", + sourceWarehouse, + "--target_warehouse", + targetWarehouse, + "--parallelism", + "1" + }; + CloneAction action = (CloneAction) ActionFactory.createAction(args).get(); + + StreamExecutionEnvironment env = + streamExecutionEnvironmentBuilder().streamingMode().allowRestart().build(); + action.withStreamExecutionEnvironment(env); + + assertThatThrownBy(action::run) + .satisfies( + anyCauseMatches( + IllegalStateException.class, + "Didn't find any table in source catalog.")); + } + // ------------------------------------------------------------------------ // Utils // ------------------------------------------------------------------------ diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/action/CompactActionITCase.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/action/CompactActionITCase.java index bc849f0a135f..2c4fb64f331c 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/action/CompactActionITCase.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/action/CompactActionITCase.java @@ -23,13 +23,9 @@ import org.apache.paimon.data.BinaryString; import org.apache.paimon.flink.FlinkConnectorOptions; import org.apache.paimon.table.FileStoreTable; -import org.apache.paimon.table.sink.StreamWriteBuilder; import org.apache.paimon.table.source.DataSplit; import org.apache.paimon.table.source.StreamTableScan; import org.apache.paimon.table.source.TableScan; -import org.apache.paimon.types.DataType; -import org.apache.paimon.types.DataTypes; -import org.apache.paimon.types.RowType; import org.apache.paimon.utils.CommonTestUtils; import org.apache.paimon.utils.SnapshotManager; @@ -56,12 +52,6 @@ /** IT cases for {@link CompactAction}. */ public class CompactActionITCase extends CompactActionITCaseBase { - private static final DataType[] FIELD_TYPES = - new DataType[] {DataTypes.INT(), DataTypes.INT(), DataTypes.INT(), DataTypes.STRING()}; - - private static final RowType ROW_TYPE = - RowType.of(FIELD_TYPES, new String[] {"k", "v", "hh", "dt"}); - @Test @Timeout(60) public void testBatchCompact() throws Exception { @@ -402,31 +392,6 @@ public void testWrongUsage() throws Exception { .hasMessage("sort compact do not support 'partition_idle_time'."); } - private FileStoreTable prepareTable( - List partitionKeys, - List primaryKeys, - List bucketKey, - Map tableOptions) - throws Exception { - FileStoreTable table = - createFileStoreTable(ROW_TYPE, partitionKeys, primaryKeys, bucketKey, tableOptions); - - StreamWriteBuilder streamWriteBuilder = - table.newStreamWriteBuilder().withCommitUser(commitUser); - write = streamWriteBuilder.newWrite(); - commit = streamWriteBuilder.newCommit(); - - return table; - } - - private void checkLatestSnapshot( - FileStoreTable table, long snapshotId, Snapshot.CommitKind commitKind) { - SnapshotManager snapshotManager = table.snapshotManager(); - Snapshot snapshot = snapshotManager.snapshot(snapshotManager.latestSnapshotId()); - assertThat(snapshot.id()).isEqualTo(snapshotId); - assertThat(snapshot.commitKind()).isEqualTo(commitKind); - } - private void runAction(boolean isStreaming) throws Exception { runAction(isStreaming, false); } diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/action/CompactActionITCaseBase.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/action/CompactActionITCaseBase.java index 4c646444cb72..41d01bdf7f35 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/action/CompactActionITCaseBase.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/action/CompactActionITCaseBase.java @@ -18,17 +18,22 @@ package org.apache.paimon.flink.action; +import org.apache.paimon.Snapshot; import org.apache.paimon.manifest.FileKind; import org.apache.paimon.manifest.ManifestEntry; import org.apache.paimon.operation.FileStoreScan; import org.apache.paimon.table.FileStoreTable; +import org.apache.paimon.table.sink.StreamWriteBuilder; import org.apache.paimon.table.source.StreamTableScan; import org.apache.paimon.table.source.TableScan; +import org.apache.paimon.types.DataType; +import org.apache.paimon.types.DataTypes; import org.apache.paimon.types.RowType; import org.apache.paimon.utils.SnapshotManager; import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.concurrent.TimeoutException; @@ -37,6 +42,12 @@ /** Base IT cases for {@link CompactAction} and {@link CompactDatabaseAction} . */ public class CompactActionITCaseBase extends ActionITCaseBase { + protected static final DataType[] FIELD_TYPES = + new DataType[] {DataTypes.INT(), DataTypes.INT(), DataTypes.INT(), DataTypes.STRING()}; + + protected static final RowType ROW_TYPE = + RowType.of(FIELD_TYPES, new String[] {"k", "v", "hh", "dt"}); + protected void validateResult( FileStoreTable table, RowType rowType, @@ -87,4 +98,29 @@ protected void checkFileAndRowSize( assertThat(files.size()).isEqualTo(fileNum); assertThat(count).isEqualTo(rowCount); } + + protected void checkLatestSnapshot( + FileStoreTable table, long snapshotId, Snapshot.CommitKind commitKind) { + SnapshotManager snapshotManager = table.snapshotManager(); + Snapshot snapshot = snapshotManager.snapshot(snapshotManager.latestSnapshotId()); + assertThat(snapshot.id()).isEqualTo(snapshotId); + assertThat(snapshot.commitKind()).isEqualTo(commitKind); + } + + protected FileStoreTable prepareTable( + List partitionKeys, + List primaryKeys, + List bucketKey, + Map tableOptions) + throws Exception { + FileStoreTable table = + createFileStoreTable(ROW_TYPE, partitionKeys, primaryKeys, bucketKey, tableOptions); + + StreamWriteBuilder streamWriteBuilder = + table.newStreamWriteBuilder().withCommitUser(commitUser); + write = streamWriteBuilder.newWrite(); + commit = streamWriteBuilder.newCommit(); + + return table; + } } diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/action/ConsumerActionITCase.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/action/ConsumerActionITCase.java index ef921ad666ca..6fb8c81eb744 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/action/ConsumerActionITCase.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/action/ConsumerActionITCase.java @@ -26,7 +26,11 @@ import org.apache.paimon.types.DataType; import org.apache.paimon.types.DataTypes; import org.apache.paimon.types.RowType; +import org.apache.paimon.utils.BlockingIterator; +import org.apache.flink.table.api.TableException; +import org.apache.flink.types.Row; +import org.junit.jupiter.api.Timeout; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; @@ -39,11 +43,13 @@ import static org.apache.paimon.flink.util.ReadWriteTableTestUtil.init; import static org.apache.paimon.flink.util.ReadWriteTableTestUtil.testStreamingRead; import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertThrows; /** IT cases for consumer management actions. */ public class ConsumerActionITCase extends ActionITCaseBase { @ParameterizedTest + @Timeout(60) @ValueSource(strings = {"action", "procedure_indexed", "procedure_named"}) public void testResetConsumer(String invoker) throws Exception { init(warehouse); @@ -70,18 +76,22 @@ public void testResetConsumer(String invoker) throws Exception { writeData(rowData(3L, BinaryString.fromString("Paimon"))); // use consumer streaming read table - testStreamingRead( + BlockingIterator iterator = + testStreamingRead( "SELECT * FROM `" + tableName + "` /*+ OPTIONS('consumer-id'='myid','consumer.expiration-time'='3h') */", Arrays.asList( changelogRow("+I", 1L, "Hi"), changelogRow("+I", 2L, "Hello"), - changelogRow("+I", 3L, "Paimon"))) - .close(); + changelogRow("+I", 3L, "Paimon"))); - Thread.sleep(1000); ConsumerManager consumerManager = new ConsumerManager(table.fileIO(), table.location()); + while (!consumerManager.consumer("myid").isPresent()) { + Thread.sleep(1000); + } + iterator.close(); + Optional consumer1 = consumerManager.consumer("myid"); assertThat(consumer1).isPresent(); assertThat(consumer1.get().nextSnapshot()).isEqualTo(4); @@ -144,9 +154,52 @@ public void testResetConsumer(String invoker) throws Exception { } Optional consumer3 = consumerManager.consumer("myid"); assertThat(consumer3).isNotPresent(); + + // reset consumer to a not exist snapshot id + List args1 = + Arrays.asList( + "reset_consumer", + "--warehouse", + warehouse, + "--database", + database, + "--table", + tableName, + "--consumer_id", + "myid", + "--next_snapshot", + "10"); + switch (invoker) { + case "action": + assertThrows( + RuntimeException.class, + () -> createAction(ResetConsumerAction.class, args1).run()); + break; + case "procedure_indexed": + assertThrows( + TableException.class, + () -> + executeSQL( + String.format( + "CALL sys.reset_consumer('%s.%s', 'myid', 10)", + database, tableName))); + break; + case "procedure_named": + assertThrows( + TableException.class, + () -> + executeSQL( + String.format( + "CALL sys.reset_consumer(`table` => '%s.%s', consumer_id => 'myid', next_snapshot_id => cast(10 as bigint))", + database, tableName))); + break; + default: + throw new UnsupportedOperationException(invoker); + } } @ParameterizedTest + @Timeout(60) @ValueSource(strings = {"action", "procedure_indexed", "procedure_named"}) public void testResetBranchConsumer(String invoker) throws Exception { init(warehouse); @@ -178,18 +231,23 @@ public void testResetBranchConsumer(String invoker) throws Exception { String branchTableName = tableName + "$branch_b1"; // use consumer streaming read table - testStreamingRead( + BlockingIterator iterator = + testStreamingRead( "SELECT * FROM `" + branchTableName + "` /*+ OPTIONS('consumer-id'='myid','consumer.expiration-time'='3h') */", Arrays.asList( changelogRow("+I", 1L, "Hi"), changelogRow("+I", 2L, "Hello"), - changelogRow("+I", 3L, "Paimon"))) - .close(); + changelogRow("+I", 3L, "Paimon"))); ConsumerManager consumerManager = new ConsumerManager(table.fileIO(), table.location(), branchName); + while (!consumerManager.consumer("myid").isPresent()) { + Thread.sleep(1000); + } + iterator.close(); + Optional consumer1 = consumerManager.consumer("myid"); assertThat(consumer1).isPresent(); assertThat(consumer1.get().nextSnapshot()).isEqualTo(4); @@ -206,7 +264,7 @@ public void testResetBranchConsumer(String invoker) throws Exception { "--consumer_id", "myid", "--next_snapshot", - "1"); + "3"); // reset consumer switch (invoker) { case "action": @@ -215,13 +273,13 @@ public void testResetBranchConsumer(String invoker) throws Exception { case "procedure_indexed": executeSQL( String.format( - "CALL sys.reset_consumer('%s.%s', 'myid', 1)", + "CALL sys.reset_consumer('%s.%s', 'myid', 3)", database, branchTableName)); break; case "procedure_named": executeSQL( String.format( - "CALL sys.reset_consumer(`table` => '%s.%s', consumer_id => 'myid', next_snapshot_id => cast(1 as bigint))", + "CALL sys.reset_consumer(`table` => '%s.%s', consumer_id => 'myid', next_snapshot_id => cast(3 as bigint))", database, branchTableName)); break; default: @@ -229,7 +287,7 @@ public void testResetBranchConsumer(String invoker) throws Exception { } Optional consumer2 = consumerManager.consumer("myid"); assertThat(consumer2).isPresent(); - assertThat(consumer2.get().nextSnapshot()).isEqualTo(1); + assertThat(consumer2.get().nextSnapshot()).isEqualTo(3); // delete consumer switch (invoker) { diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/action/ExpireTagsActionTest.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/action/ExpireTagsActionTest.java new file mode 100644 index 000000000000..5a156ced25be --- /dev/null +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/action/ExpireTagsActionTest.java @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.flink.action; + +import org.apache.paimon.data.Timestamp; +import org.apache.paimon.table.FileStoreTable; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.time.LocalDateTime; + +import static org.apache.paimon.flink.util.ReadWriteTableTestUtil.bEnv; +import static org.apache.paimon.flink.util.ReadWriteTableTestUtil.init; +import static org.assertj.core.api.Assertions.assertThat; + +/** IT cases for {@link ExpireTagsAction}. */ +public class ExpireTagsActionTest extends ActionITCaseBase { + + @BeforeEach + public void setUp() { + init(warehouse); + } + + @Test + public void testExpireTags() throws Exception { + bEnv.executeSql( + "CREATE TABLE T (id STRING, name STRING," + + " PRIMARY KEY (id) NOT ENFORCED)" + + " WITH ('bucket'='1', 'write-only'='true')"); + + FileStoreTable table = getFileStoreTable("T"); + + // generate 5 snapshots + for (int i = 1; i <= 5; i++) { + bEnv.executeSql("INSERT INTO T VALUES ('" + i + "', '" + i + "')").await(); + } + assertThat(table.snapshotManager().snapshotCount()).isEqualTo(5); + + bEnv.executeSql("CALL sys.create_tag('default.T', 'tag-1', 1)").await(); + bEnv.executeSql("CALL sys.create_tag('default.T', 'tag-2', 2, '1h')").await(); + bEnv.executeSql("CALL sys.create_tag('default.T', 'tag-3', 3, '1h')").await(); + assertThat(table.tagManager().tags().size()).isEqualTo(3); + + createAction( + ExpireTagsAction.class, + "expire_tags", + "--warehouse", + warehouse, + "--table", + database + ".T") + .run(); + // no tags expired + assertThat(table.tagManager().tags().size()).isEqualTo(3); + + bEnv.executeSql("CALL sys.create_tag('default.T', 'tag-4', 4, '1s')").await(); + bEnv.executeSql("CALL sys.create_tag('default.T', 'tag-5', 5, '1s')").await(); + assertThat(table.tagManager().tags().size()).isEqualTo(5); + + Thread.sleep(2000); + createAction( + ExpireTagsAction.class, + "expire_tags", + "--warehouse", + warehouse, + "--table", + database + ".T") + .run(); + // tag-4,tag-5 expires + assertThat(table.tagManager().tags().size()).isEqualTo(3); + assertThat(table.tagManager().tagExists("tag-4")).isFalse(); + assertThat(table.tagManager().tagExists("tag-5")).isFalse(); + + // tag-3 as the base older_than time + LocalDateTime olderThanTime = table.tagManager().tag("tag-3").getTagCreateTime(); + java.sql.Timestamp timestamp = + new java.sql.Timestamp(Timestamp.fromLocalDateTime(olderThanTime).getMillisecond()); + createAction( + ExpireTagsAction.class, + "expire_tags", + "--warehouse", + warehouse, + "--table", + database + ".T", + "--older_than", + timestamp.toString()) + .run(); + // tag-1,tag-2 expires. tag-1 expired by its file creation time. + assertThat(table.tagManager().tags().size()).isEqualTo(1); + assertThat(table.tagManager().tagExists("tag-3")).isTrue(); + } +} diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/action/MergeIntoActionITCase.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/action/MergeIntoActionITCase.java index 41d607fac5f6..3907c0398532 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/action/MergeIntoActionITCase.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/action/MergeIntoActionITCase.java @@ -571,6 +571,123 @@ public void testIllegalSourceNameSqlCase() { .satisfies(anyCauseMatches(ValidationException.class, "Object 'S' not found")); } + @ParameterizedTest + @MethodSource("testArguments") + public void testNotMatchedBySourceUpsert(boolean qualified, String invoker) throws Exception { + sEnv.executeSql("DROP TABLE T"); + prepareTargetTable(CoreOptions.ChangelogProducer.INPUT); + + // build MergeIntoAction + MergeIntoActionBuilder action = new MergeIntoActionBuilder(warehouse, database, "T"); + action.withSourceSqls("CREATE TEMPORARY VIEW SS AS SELECT k, v, 'unknown', dt FROM S") + .withSourceTable(qualified ? "default.SS" : "SS") + .withMergeCondition("T.k = SS.k AND T.dt = SS.dt") + .withNotMatchedBySourceUpsert( + "dt < '02-28'", "v = v || '_nmu', last_action = 'not_matched_upsert'"); + + String procedureStatement = ""; + if ("procedure_indexed".equals(invoker)) { + procedureStatement = + String.format( + "CALL sys.merge_into('%s.T', '', '%s', '%s', 'T.k = SS.k AND T.dt = SS.dt', '', '', '', '', '', 'dt < ''02-28''', 'v = v || ''_nmu'', last_action = ''not_matched_upsert''')", + database, + "CREATE TEMPORARY VIEW SS AS SELECT k, v, ''unknown'', dt FROM S", + qualified ? "default.SS" : "SS"); + } else if ("procedure_named".equals(invoker)) { + procedureStatement = + String.format( + "CALL sys.merge_into(" + + "target_table => '%s.T', " + + "source_sqls => '%s', " + + "source_table => '%s', " + + "merge_condition => 'T.k = SS.k AND T.dt = SS.dt', " + + "not_matched_by_source_upsert_condition => 'dt < ''02-28'''," + + "not_matched_by_source_upsert_setting => 'v = v || ''_nmu'', last_action = ''not_matched_upsert''')", + database, + "CREATE TEMPORARY VIEW SS AS SELECT k, v, ''unknown'', dt FROM S", + qualified ? "default.SS" : "SS"); + } + + List streamingExpected = + Arrays.asList( + changelogRow("+U", 2, "v_2_nmu", "not_matched_upsert", "02-27"), + changelogRow("+U", 3, "v_3_nmu", "not_matched_upsert", "02-27")); + + List batchExpected = + Arrays.asList( + changelogRow("+I", 1, "v_1", "creation", "02-27"), + changelogRow("+I", 2, "v_2_nmu", "not_matched_upsert", "02-27"), + changelogRow("+I", 3, "v_3_nmu", "not_matched_upsert", "02-27"), + changelogRow("+I", 4, "v_4", "creation", "02-27"), + changelogRow("+I", 5, "v_5", "creation", "02-28"), + changelogRow("+I", 6, "v_6", "creation", "02-28"), + changelogRow("+I", 7, "v_7", "creation", "02-28"), + changelogRow("+I", 8, "v_8", "creation", "02-28"), + changelogRow("+I", 9, "v_9", "creation", "02-28"), + changelogRow("+I", 10, "v_10", "creation", "02-28")); + + if ("action".equals(invoker)) { + validateActionRunResult(action.build(), streamingExpected, batchExpected); + } else { + validateProcedureResult(procedureStatement, streamingExpected, batchExpected); + } + } + + @ParameterizedTest + @MethodSource("testArguments") + public void testNotMatchedBySourceDelete(boolean qualified, String invoker) throws Exception { + // build MergeIntoAction + MergeIntoActionBuilder action = new MergeIntoActionBuilder(warehouse, database, "T"); + action.withSourceSqls("CREATE TEMPORARY VIEW SS AS SELECT k, v, 'unknown', dt FROM S") + .withSourceTable(qualified ? "default.SS" : "SS") + .withMergeCondition("T.k = SS.k AND T.dt = SS.dt") + .withNotMatchedBySourceDelete(null); + + String procedureStatement = ""; + if ("procedure_indexed".equals(invoker)) { + procedureStatement = + String.format( + "CALL sys.merge_into('%s.T', '', '%s', '%s', 'T.k = SS.k AND T.dt = SS.dt', '', '', '', '', '', '', '', 'TRUE')", + database, + "CREATE TEMPORARY VIEW SS AS SELECT k, v, ''unknown'', dt FROM S", + qualified ? "default.SS" : "SS"); + } else if ("procedure_named".equals(invoker)) { + procedureStatement = + String.format( + "CALL sys.merge_into(" + + "target_table => '%s.T', " + + "source_sqls => '%s', " + + "source_table => '%s', " + + "merge_condition => 'T.k = SS.k AND T.dt = SS.dt', " + + "not_matched_by_source_delete_condition => 'TRUE')", + database, + "CREATE TEMPORARY VIEW SS AS SELECT k, v, ''unknown'', dt FROM S", + qualified ? "default.SS" : "SS"); + } + + List streamingExpected = + Arrays.asList( + changelogRow("-D", 2, "v_2", "creation", "02-27"), + changelogRow("-D", 3, "v_3", "creation", "02-27"), + changelogRow("-D", 5, "v_5", "creation", "02-28"), + changelogRow("-D", 6, "v_6", "creation", "02-28"), + changelogRow("-D", 9, "v_9", "creation", "02-28"), + changelogRow("-D", 10, "v_10", "creation", "02-28")); + + List batchExpected = + Arrays.asList( + changelogRow("+I", 1, "v_1", "creation", "02-27"), + changelogRow("+I", 4, "v_4", "creation", "02-27"), + changelogRow("+I", 7, "v_7", "creation", "02-28"), + changelogRow("+I", 8, "v_8", "creation", "02-28")); + + if ("action".equals(invoker)) { + validateActionRunResult(action.build(), streamingExpected, batchExpected); + } else { + validateProcedureResult(procedureStatement, streamingExpected, batchExpected); + } + } + private void validateActionRunResult( MergeIntoAction action, List streamingExpected, List batchExpected) throws Exception { diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/action/MinorCompactActionITCase.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/action/MinorCompactActionITCase.java new file mode 100644 index 000000000000..0373eb01a2d9 --- /dev/null +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/action/MinorCompactActionITCase.java @@ -0,0 +1,205 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.flink.action; + +import org.apache.paimon.CoreOptions; +import org.apache.paimon.Snapshot; +import org.apache.paimon.data.BinaryString; +import org.apache.paimon.table.FileStoreTable; +import org.apache.paimon.table.source.DataSplit; + +import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; +import org.assertj.core.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; + +/** IT cases for compact strategy {@link CompactAction}. */ +public class MinorCompactActionITCase extends CompactActionITCaseBase { + + @Test + @Timeout(60) + public void testBatchMinorCompactStrategy() throws Exception { + FileStoreTable table = + prepareTable( + Arrays.asList("dt", "hh"), + Arrays.asList("dt", "hh", "k"), + Collections.emptyList(), + Collections.singletonMap(CoreOptions.WRITE_ONLY.key(), "true")); + + writeData( + rowData(1, 100, 15, BinaryString.fromString("20221208")), + rowData(1, 100, 16, BinaryString.fromString("20221208"))); + + writeData( + rowData(2, 100, 15, BinaryString.fromString("20221208")), + rowData(2, 100, 16, BinaryString.fromString("20221208"))); + + checkLatestSnapshot(table, 2, Snapshot.CommitKind.APPEND); + + CompactAction action = + createAction( + CompactAction.class, + "compact", + "--warehouse", + warehouse, + "--database", + database, + "--table", + tableName, + "--compact_strategy", + "minor", + "--table_conf", + CoreOptions.NUM_SORTED_RUNS_COMPACTION_TRIGGER.key() + "=3"); + StreamExecutionEnvironment env = streamExecutionEnvironmentBuilder().batchMode().build(); + action.withStreamExecutionEnvironment(env).build(); + env.execute(); + + // Due to the limitation of parameter 'num-sorted-run.compaction-trigger', so compact is not + // performed. + checkLatestSnapshot(table, 2, Snapshot.CommitKind.APPEND); + + // Make par-15 has 3 datafile and par-16 has 2 datafile, so par-16 will not be picked out to + // compact. + writeData(rowData(2, 100, 15, BinaryString.fromString("20221208"))); + + env = streamExecutionEnvironmentBuilder().batchMode().build(); + action.withStreamExecutionEnvironment(env).build(); + env.execute(); + + checkLatestSnapshot(table, 4, Snapshot.CommitKind.COMPACT); + + List splits = table.newSnapshotReader().read().dataSplits(); + assertThat(splits.size()).isEqualTo(2); + for (DataSplit split : splits) { + // Par-16 is not compacted. + assertThat(split.dataFiles().size()) + .isEqualTo(split.partition().getInt(1) == 16 ? 2 : 1); + } + } + + @Test + @Timeout(60) + public void testBatchFullCompactStrategy() throws Exception { + FileStoreTable table = + prepareTable( + Arrays.asList("dt", "hh"), + Arrays.asList("dt", "hh", "k"), + Collections.emptyList(), + Collections.singletonMap(CoreOptions.WRITE_ONLY.key(), "true")); + + writeData( + rowData(1, 100, 15, BinaryString.fromString("20221208")), + rowData(1, 100, 16, BinaryString.fromString("20221208"))); + + writeData( + rowData(2, 100, 15, BinaryString.fromString("20221208")), + rowData(2, 100, 16, BinaryString.fromString("20221208"))); + + checkLatestSnapshot(table, 2, Snapshot.CommitKind.APPEND); + + CompactAction action = + createAction( + CompactAction.class, + "compact", + "--warehouse", + warehouse, + "--database", + database, + "--table", + tableName, + "--compact_strategy", + "full", + "--table_conf", + CoreOptions.NUM_SORTED_RUNS_COMPACTION_TRIGGER.key() + "=3"); + StreamExecutionEnvironment env = streamExecutionEnvironmentBuilder().batchMode().build(); + action.withStreamExecutionEnvironment(env).build(); + env.execute(); + + checkLatestSnapshot(table, 3, Snapshot.CommitKind.COMPACT); + + List splits = table.newSnapshotReader().read().dataSplits(); + assertThat(splits.size()).isEqualTo(2); + for (DataSplit split : splits) { + assertThat(split.dataFiles().size()).isEqualTo(1); + } + } + + @Test + @Timeout(60) + public void testStreamingFullCompactStrategy() throws Exception { + prepareTable( + Arrays.asList("dt", "hh"), + Arrays.asList("dt", "hh", "k"), + Collections.emptyList(), + Collections.singletonMap(CoreOptions.WRITE_ONLY.key(), "true")); + CompactAction action = + createAction( + CompactAction.class, + "compact", + "--warehouse", + warehouse, + "--database", + database, + "--table", + tableName, + "--compact_strategy", + "full", + "--table_conf", + CoreOptions.NUM_SORTED_RUNS_COMPACTION_TRIGGER.key() + "=3"); + StreamExecutionEnvironment env = + streamExecutionEnvironmentBuilder().streamingMode().build(); + Assertions.assertThatThrownBy(() -> action.withStreamExecutionEnvironment(env).build()) + .hasMessage( + "The full compact strategy is only supported in batch mode. Please add -Dexecution.runtime-mode=BATCH."); + } + + @Test + @Timeout(60) + public void testCompactStrategyWithWrongUsage() throws Exception { + prepareTable( + Arrays.asList("dt", "hh"), + Arrays.asList("dt", "hh", "k"), + Collections.emptyList(), + Collections.singletonMap(CoreOptions.WRITE_ONLY.key(), "true")); + Assertions.assertThatThrownBy( + () -> + createAction( + CompactAction.class, + "compact", + "--warehouse", + warehouse, + "--database", + database, + "--table", + tableName, + "--compact_strategy", + "wrong_usage", + "--table_conf", + CoreOptions.NUM_SORTED_RUNS_COMPACTION_TRIGGER.key() + + "=3")) + .hasMessage( + "The compact strategy only supports 'full' or 'minor', but 'wrong_usage' is configured."); + } +} diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/action/RemoveOrphanFilesActionITCase.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/action/RemoveOrphanFilesActionITCase.java index 938a8ce1be7a..a92e529aa2cf 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/action/RemoveOrphanFilesActionITCase.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/action/RemoveOrphanFilesActionITCase.java @@ -18,324 +18,5 @@ package org.apache.paimon.flink.action; -import org.apache.paimon.CoreOptions; -import org.apache.paimon.data.BinaryString; -import org.apache.paimon.data.GenericRow; -import org.apache.paimon.fs.FileIO; -import org.apache.paimon.fs.Path; -import org.apache.paimon.options.Options; -import org.apache.paimon.schema.SchemaChange; -import org.apache.paimon.schema.SchemaManager; -import org.apache.paimon.schema.TableSchema; -import org.apache.paimon.table.FileStoreTable; -import org.apache.paimon.table.FileStoreTableFactory; -import org.apache.paimon.table.sink.StreamTableCommit; -import org.apache.paimon.table.sink.StreamTableWrite; -import org.apache.paimon.table.sink.StreamWriteBuilder; -import org.apache.paimon.types.DataType; -import org.apache.paimon.types.DataTypes; -import org.apache.paimon.types.RowType; - -import org.apache.paimon.shade.guava30.com.google.common.collect.ImmutableList; - -import org.apache.flink.types.Row; -import org.apache.flink.util.CloseableIterator; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.ValueSource; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.UUID; -import java.util.concurrent.ThreadLocalRandom; - -import static org.apache.paimon.CoreOptions.SCAN_FALLBACK_BRANCH; -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatCode; - -/** IT cases for {@link RemoveOrphanFilesAction}. */ -public class RemoveOrphanFilesActionITCase extends ActionITCaseBase { - - private static final String ORPHAN_FILE_1 = "bucket-0/orphan_file1"; - private static final String ORPHAN_FILE_2 = "bucket-0/orphan_file2"; - - private FileStoreTable createTableAndWriteData(String tableName) throws Exception { - RowType rowType = - RowType.of( - new DataType[] {DataTypes.BIGINT(), DataTypes.STRING()}, - new String[] {"k", "v"}); - - FileStoreTable table = - createFileStoreTable( - tableName, - rowType, - Collections.emptyList(), - Collections.singletonList("k"), - Collections.emptyList(), - Collections.emptyMap()); - - StreamWriteBuilder writeBuilder = table.newStreamWriteBuilder().withCommitUser(commitUser); - write = writeBuilder.newWrite(); - commit = writeBuilder.newCommit(); - - writeData(rowData(1L, BinaryString.fromString("Hi"))); - - Path orphanFile1 = getOrphanFilePath(table, ORPHAN_FILE_1); - Path orphanFile2 = getOrphanFilePath(table, ORPHAN_FILE_2); - - FileIO fileIO = table.fileIO(); - fileIO.writeFile(orphanFile1, "a", true); - Thread.sleep(2000); - fileIO.writeFile(orphanFile2, "b", true); - - return table; - } - - private Path getOrphanFilePath(FileStoreTable table, String orphanFile) { - return new Path(table.location(), orphanFile); - } - - @ParameterizedTest - @ValueSource(booleans = {true, false}) - public void testRunWithoutException(boolean isNamedArgument) throws Exception { - createTableAndWriteData(tableName); - - List args = - new ArrayList<>( - Arrays.asList( - "remove_orphan_files", - "--warehouse", - warehouse, - "--database", - database, - "--table", - tableName)); - RemoveOrphanFilesAction action1 = createAction(RemoveOrphanFilesAction.class, args); - assertThatCode(action1::run).doesNotThrowAnyException(); - - args.add("--older_than"); - args.add("2023-12-31 23:59:59"); - RemoveOrphanFilesAction action2 = createAction(RemoveOrphanFilesAction.class, args); - assertThatCode(action2::run).doesNotThrowAnyException(); - - String withoutOlderThan = - String.format( - isNamedArgument - ? "CALL sys.remove_orphan_files(`table` => '%s.%s')" - : "CALL sys.remove_orphan_files('%s.%s')", - database, - tableName); - CloseableIterator withoutOlderThanCollect = executeSQL(withoutOlderThan); - assertThat(ImmutableList.copyOf(withoutOlderThanCollect)).containsOnly(Row.of("0")); - - String withDryRun = - String.format( - isNamedArgument - ? "CALL sys.remove_orphan_files(`table` => '%s.%s', older_than => '2999-12-31 23:59:59', dry_run => true)" - : "CALL sys.remove_orphan_files('%s.%s', '2999-12-31 23:59:59', true)", - database, - tableName); - ImmutableList actualDryRunDeleteFile = ImmutableList.copyOf(executeSQL(withDryRun)); - assertThat(actualDryRunDeleteFile).containsOnly(Row.of("2")); - - String withOlderThan = - String.format( - isNamedArgument - ? "CALL sys.remove_orphan_files(`table` => '%s.%s', older_than => '2999-12-31 23:59:59')" - : "CALL sys.remove_orphan_files('%s.%s', '2999-12-31 23:59:59')", - database, - tableName); - ImmutableList actualDeleteFile = ImmutableList.copyOf(executeSQL(withOlderThan)); - - assertThat(actualDeleteFile).containsExactlyInAnyOrder(Row.of("2")); - } - - @ParameterizedTest - @ValueSource(booleans = {true, false}) - public void testRemoveDatabaseOrphanFilesITCase(boolean isNamedArgument) throws Exception { - createTableAndWriteData("tableName1"); - createTableAndWriteData("tableName2"); - - List args = - new ArrayList<>( - Arrays.asList( - "remove_orphan_files", - "--warehouse", - warehouse, - "--database", - database, - "--table", - "*")); - RemoveOrphanFilesAction action1 = createAction(RemoveOrphanFilesAction.class, args); - assertThatCode(action1::run).doesNotThrowAnyException(); - - args.add("--older_than"); - args.add("2023-12-31 23:59:59"); - RemoveOrphanFilesAction action2 = createAction(RemoveOrphanFilesAction.class, args); - assertThatCode(action2::run).doesNotThrowAnyException(); - - args.add("--parallelism"); - args.add("5"); - RemoveOrphanFilesAction action3 = createAction(RemoveOrphanFilesAction.class, args); - assertThatCode(action3::run).doesNotThrowAnyException(); - - String withoutOlderThan = - String.format( - isNamedArgument - ? "CALL sys.remove_orphan_files(`table` => '%s.%s')" - : "CALL sys.remove_orphan_files('%s.%s')", - database, - "*"); - CloseableIterator withoutOlderThanCollect = executeSQL(withoutOlderThan); - assertThat(ImmutableList.copyOf(withoutOlderThanCollect)).containsOnly(Row.of("0")); - - String withParallelism = - String.format("CALL sys.remove_orphan_files('%s.%s','',true,5)", database, "*"); - CloseableIterator withParallelismCollect = executeSQL(withParallelism); - assertThat(ImmutableList.copyOf(withParallelismCollect)).containsOnly(Row.of("0")); - - String withDryRun = - String.format( - isNamedArgument - ? "CALL sys.remove_orphan_files(`table` => '%s.%s', older_than => '2999-12-31 23:59:59', dry_run => true)" - : "CALL sys.remove_orphan_files('%s.%s', '2999-12-31 23:59:59', true)", - database, - "*"); - ImmutableList actualDryRunDeleteFile = ImmutableList.copyOf(executeSQL(withDryRun)); - assertThat(actualDryRunDeleteFile).containsOnly(Row.of("4")); - - String withOlderThan = - String.format( - isNamedArgument - ? "CALL sys.remove_orphan_files(`table` => '%s.%s', older_than => '2999-12-31 23:59:59')" - : "CALL sys.remove_orphan_files('%s.%s', '2999-12-31 23:59:59')", - database, - "*"); - ImmutableList actualDeleteFile = ImmutableList.copyOf(executeSQL(withOlderThan)); - - assertThat(actualDeleteFile).containsOnly(Row.of("4")); - } - - @ParameterizedTest - @ValueSource(booleans = {true, false}) - public void testCleanWithBranch(boolean isNamedArgument) throws Exception { - // create main branch - FileStoreTable table = createTableAndWriteData(tableName); - - // create first branch and write some data - table.createBranch("br"); - SchemaManager schemaManager = new SchemaManager(table.fileIO(), table.location(), "br"); - TableSchema branchSchema = - schemaManager.commitChanges(SchemaChange.addColumn("v2", DataTypes.INT())); - Options branchOptions = new Options(branchSchema.options()); - branchOptions.set(CoreOptions.BRANCH, "br"); - branchSchema = branchSchema.copy(branchOptions.toMap()); - FileStoreTable branchTable = - FileStoreTableFactory.create(table.fileIO(), table.location(), branchSchema); - - String commitUser = UUID.randomUUID().toString(); - StreamTableWrite write = branchTable.newWrite(commitUser); - StreamTableCommit commit = branchTable.newCommit(commitUser); - write.write(GenericRow.of(2L, BinaryString.fromString("Hello"), 20)); - commit.commit(1, write.prepareCommit(false, 1)); - write.close(); - commit.close(); - - // create orphan file in snapshot directory of first branch - Path orphanFile3 = new Path(table.location(), "branch/branch-br/snapshot/orphan_file3"); - branchTable.fileIO().writeFile(orphanFile3, "x", true); - - // create second branch, which is empty - table.createBranch("br2"); - - // create orphan file in snapshot directory of second branch - Path orphanFile4 = new Path(table.location(), "branch/branch-br2/snapshot/orphan_file4"); - branchTable.fileIO().writeFile(orphanFile4, "y", true); - - if (ThreadLocalRandom.current().nextBoolean()) { - executeSQL( - String.format( - "ALTER TABLE `%s`.`%s` SET ('%s' = 'br')", - database, tableName, SCAN_FALLBACK_BRANCH.key()), - false, - true); - } - String procedure = - String.format( - isNamedArgument - ? "CALL sys.remove_orphan_files(`table` => '%s.%s', older_than => '2999-12-31 23:59:59')" - : "CALL sys.remove_orphan_files('%s.%s', '2999-12-31 23:59:59')", - database, - "*"); - ImmutableList actualDeleteFile = ImmutableList.copyOf(executeSQL(procedure)); - assertThat(actualDeleteFile).containsOnly(Row.of("4")); - } - - @ParameterizedTest - @ValueSource(booleans = {true, false}) - public void testRunWithMode(boolean isNamedArgument) throws Exception { - createTableAndWriteData(tableName); - - List args = - new ArrayList<>( - Arrays.asList( - "remove_orphan_files", - "--warehouse", - warehouse, - "--database", - database, - "--table", - tableName)); - RemoveOrphanFilesAction action1 = createAction(RemoveOrphanFilesAction.class, args); - assertThatCode(action1::run).doesNotThrowAnyException(); - - args.add("--older_than"); - args.add("2023-12-31 23:59:59"); - RemoveOrphanFilesAction action2 = createAction(RemoveOrphanFilesAction.class, args); - assertThatCode(action2::run).doesNotThrowAnyException(); - - String withoutOlderThan = - String.format( - isNamedArgument - ? "CALL sys.remove_orphan_files(`table` => '%s.%s')" - : "CALL sys.remove_orphan_files('%s.%s')", - database, - tableName); - CloseableIterator withoutOlderThanCollect = executeSQL(withoutOlderThan); - assertThat(ImmutableList.copyOf(withoutOlderThanCollect)).containsOnly(Row.of("0")); - - String withLocalMode = - String.format( - isNamedArgument - ? "CALL sys.remove_orphan_files(`table` => '%s.%s', older_than => '2999-12-31 23:59:59', dry_run => true, parallelism => 5, mode => 'local')" - : "CALL sys.remove_orphan_files('%s.%s', '2999-12-31 23:59:59', true, 5, 'local')", - database, - tableName); - ImmutableList actualLocalRunDeleteFile = - ImmutableList.copyOf(executeSQL(withLocalMode)); - assertThat(actualLocalRunDeleteFile).containsOnly(Row.of("2")); - - String withDistributedMode = - String.format( - isNamedArgument - ? "CALL sys.remove_orphan_files(`table` => '%s.%s', older_than => '2999-12-31 23:59:59', dry_run => true, parallelism => 5, mode => 'distributed')" - : "CALL sys.remove_orphan_files('%s.%s', '2999-12-31 23:59:59', true, 5, 'distributed')", - database, - tableName); - ImmutableList actualDistributedRunDeleteFile = - ImmutableList.copyOf(executeSQL(withDistributedMode)); - assertThat(actualDistributedRunDeleteFile).containsOnly(Row.of("2")); - - String withInvalidMode = - String.format( - isNamedArgument - ? "CALL sys.remove_orphan_files(`table` => '%s.%s', older_than => '2999-12-31 23:59:59', dry_run => true, parallelism => 5, mode => 'unknown')" - : "CALL sys.remove_orphan_files('%s.%s', '2999-12-31 23:59:59', true, 5, 'unknown')", - database, - tableName); - assertThatCode(() -> executeSQL(withInvalidMode)) - .isInstanceOf(RuntimeException.class) - .hasMessageContaining("Unknown mode"); - } -} +/** IT cases base for {@link RemoveOrphanFilesAction} in Flink Common. */ +public class RemoveOrphanFilesActionITCase extends RemoveOrphanFilesActionITCaseBase {} diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/action/RemoveOrphanFilesActionITCaseBase.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/action/RemoveOrphanFilesActionITCaseBase.java new file mode 100644 index 000000000000..77f3be2f0c76 --- /dev/null +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/action/RemoveOrphanFilesActionITCaseBase.java @@ -0,0 +1,341 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.flink.action; + +import org.apache.paimon.CoreOptions; +import org.apache.paimon.data.BinaryString; +import org.apache.paimon.data.GenericRow; +import org.apache.paimon.fs.FileIO; +import org.apache.paimon.fs.Path; +import org.apache.paimon.options.Options; +import org.apache.paimon.schema.SchemaChange; +import org.apache.paimon.schema.SchemaManager; +import org.apache.paimon.schema.TableSchema; +import org.apache.paimon.table.FileStoreTable; +import org.apache.paimon.table.FileStoreTableFactory; +import org.apache.paimon.table.sink.StreamTableCommit; +import org.apache.paimon.table.sink.StreamTableWrite; +import org.apache.paimon.table.sink.StreamWriteBuilder; +import org.apache.paimon.types.DataType; +import org.apache.paimon.types.DataTypes; +import org.apache.paimon.types.RowType; + +import org.apache.paimon.shade.guava30.com.google.common.collect.ImmutableList; + +import org.apache.flink.types.Row; +import org.apache.flink.util.CloseableIterator; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.ThreadLocalRandom; + +import static org.apache.paimon.CoreOptions.SCAN_FALLBACK_BRANCH; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatCode; + +/** IT cases base for {@link RemoveOrphanFilesAction}. */ +public abstract class RemoveOrphanFilesActionITCaseBase extends ActionITCaseBase { + + private static final String ORPHAN_FILE_1 = "bucket-0/orphan_file1"; + private static final String ORPHAN_FILE_2 = "bucket-0/orphan_file2"; + + private FileStoreTable createTableAndWriteData(String tableName) throws Exception { + RowType rowType = + RowType.of( + new DataType[] {DataTypes.BIGINT(), DataTypes.STRING()}, + new String[] {"k", "v"}); + + FileStoreTable table = + createFileStoreTable( + tableName, + rowType, + Collections.emptyList(), + Collections.singletonList("k"), + Collections.emptyList(), + Collections.emptyMap()); + + StreamWriteBuilder writeBuilder = table.newStreamWriteBuilder().withCommitUser(commitUser); + write = writeBuilder.newWrite(); + commit = writeBuilder.newCommit(); + + writeData(rowData(1L, BinaryString.fromString("Hi"))); + + Path orphanFile1 = getOrphanFilePath(table, ORPHAN_FILE_1); + Path orphanFile2 = getOrphanFilePath(table, ORPHAN_FILE_2); + + FileIO fileIO = table.fileIO(); + fileIO.writeFile(orphanFile1, "a", true); + Thread.sleep(2000); + fileIO.writeFile(orphanFile2, "b", true); + + return table; + } + + private Path getOrphanFilePath(FileStoreTable table, String orphanFile) { + return new Path(table.location(), orphanFile); + } + + @ParameterizedTest + @ValueSource(booleans = {true, false}) + public void testRunWithoutException(boolean isNamedArgument) throws Exception { + createTableAndWriteData(tableName); + + List args = + new ArrayList<>( + Arrays.asList( + "remove_orphan_files", + "--warehouse", + warehouse, + "--database", + database, + "--table", + tableName)); + RemoveOrphanFilesAction action1 = createAction(RemoveOrphanFilesAction.class, args); + assertThatCode(action1::run).doesNotThrowAnyException(); + + args.add("--older_than"); + args.add("2023-12-31 23:59:59"); + RemoveOrphanFilesAction action2 = createAction(RemoveOrphanFilesAction.class, args); + assertThatCode(action2::run).doesNotThrowAnyException(); + + String withoutOlderThan = + String.format( + isNamedArgument + ? "CALL sys.remove_orphan_files(`table` => '%s.%s')" + : "CALL sys.remove_orphan_files('%s.%s')", + database, + tableName); + CloseableIterator withoutOlderThanCollect = executeSQL(withoutOlderThan); + assertThat(ImmutableList.copyOf(withoutOlderThanCollect)).containsOnly(Row.of("0")); + + String withDryRun = + String.format( + isNamedArgument + ? "CALL sys.remove_orphan_files(`table` => '%s.%s', older_than => '2999-12-31 23:59:59', dry_run => true)" + : "CALL sys.remove_orphan_files('%s.%s', '2999-12-31 23:59:59', true)", + database, + tableName); + ImmutableList actualDryRunDeleteFile = ImmutableList.copyOf(executeSQL(withDryRun)); + assertThat(actualDryRunDeleteFile).containsOnly(Row.of("2")); + + String withOlderThan = + String.format( + isNamedArgument + ? "CALL sys.remove_orphan_files(`table` => '%s.%s', older_than => '2999-12-31 23:59:59')" + : "CALL sys.remove_orphan_files('%s.%s', '2999-12-31 23:59:59')", + database, + tableName); + ImmutableList actualDeleteFile = ImmutableList.copyOf(executeSQL(withOlderThan)); + + assertThat(actualDeleteFile).containsExactlyInAnyOrder(Row.of("2"), Row.of("2")); + } + + @ParameterizedTest + @ValueSource(booleans = {true, false}) + public void testRemoveDatabaseOrphanFilesITCase(boolean isNamedArgument) throws Exception { + createTableAndWriteData("tableName1"); + createTableAndWriteData("tableName2"); + + List args = + new ArrayList<>( + Arrays.asList( + "remove_orphan_files", + "--warehouse", + warehouse, + "--database", + database, + "--table", + "*")); + RemoveOrphanFilesAction action1 = createAction(RemoveOrphanFilesAction.class, args); + assertThatCode(action1::run).doesNotThrowAnyException(); + + args.add("--older_than"); + args.add("2023-12-31 23:59:59"); + RemoveOrphanFilesAction action2 = createAction(RemoveOrphanFilesAction.class, args); + assertThatCode(action2::run).doesNotThrowAnyException(); + + args.add("--parallelism"); + args.add("5"); + RemoveOrphanFilesAction action3 = createAction(RemoveOrphanFilesAction.class, args); + assertThatCode(action3::run).doesNotThrowAnyException(); + + String withoutOlderThan = + String.format( + isNamedArgument + ? "CALL sys.remove_orphan_files(`table` => '%s.%s')" + : "CALL sys.remove_orphan_files('%s.%s')", + database, + "*"); + CloseableIterator withoutOlderThanCollect = executeSQL(withoutOlderThan); + assertThat(ImmutableList.copyOf(withoutOlderThanCollect)).containsOnly(Row.of("0")); + + String withParallelism = + String.format("CALL sys.remove_orphan_files('%s.%s','',true,5)", database, "*"); + CloseableIterator withParallelismCollect = executeSQL(withParallelism); + assertThat(ImmutableList.copyOf(withParallelismCollect)).containsOnly(Row.of("0")); + + String withDryRun = + String.format( + isNamedArgument + ? "CALL sys.remove_orphan_files(`table` => '%s.%s', older_than => '2999-12-31 23:59:59', dry_run => true)" + : "CALL sys.remove_orphan_files('%s.%s', '2999-12-31 23:59:59', true)", + database, + "*"); + ImmutableList actualDryRunDeleteFile = ImmutableList.copyOf(executeSQL(withDryRun)); + assertThat(actualDryRunDeleteFile).containsOnly(Row.of("4")); + + String withOlderThan = + String.format( + isNamedArgument + ? "CALL sys.remove_orphan_files(`table` => '%s.%s', older_than => '2999-12-31 23:59:59')" + : "CALL sys.remove_orphan_files('%s.%s', '2999-12-31 23:59:59')", + database, + "*"); + ImmutableList actualDeleteFile = ImmutableList.copyOf(executeSQL(withOlderThan)); + + assertThat(actualDeleteFile).containsOnly(Row.of("4")); + } + + @ParameterizedTest + @ValueSource(booleans = {true, false}) + public void testCleanWithBranch(boolean isNamedArgument) throws Exception { + // create main branch + FileStoreTable table = createTableAndWriteData(tableName); + + // create first branch and write some data + table.createBranch("br"); + SchemaManager schemaManager = new SchemaManager(table.fileIO(), table.location(), "br"); + TableSchema branchSchema = + schemaManager.commitChanges(SchemaChange.addColumn("v2", DataTypes.INT())); + Options branchOptions = new Options(branchSchema.options()); + branchOptions.set(CoreOptions.BRANCH, "br"); + branchSchema = branchSchema.copy(branchOptions.toMap()); + FileStoreTable branchTable = + FileStoreTableFactory.create(table.fileIO(), table.location(), branchSchema); + + String commitUser = UUID.randomUUID().toString(); + StreamTableWrite write = branchTable.newWrite(commitUser); + StreamTableCommit commit = branchTable.newCommit(commitUser); + write.write(GenericRow.of(2L, BinaryString.fromString("Hello"), 20)); + commit.commit(1, write.prepareCommit(false, 1)); + write.close(); + commit.close(); + + // create orphan file in snapshot directory of first branch + Path orphanFile3 = new Path(table.location(), "branch/branch-br/snapshot/orphan_file3"); + branchTable.fileIO().writeFile(orphanFile3, "x", true); + + // create second branch, which is empty + table.createBranch("br2"); + + // create orphan file in snapshot directory of second branch + Path orphanFile4 = new Path(table.location(), "branch/branch-br2/snapshot/orphan_file4"); + branchTable.fileIO().writeFile(orphanFile4, "y", true); + + if (ThreadLocalRandom.current().nextBoolean()) { + executeSQL( + String.format( + "ALTER TABLE `%s`.`%s` SET ('%s' = 'br')", + database, tableName, SCAN_FALLBACK_BRANCH.key()), + false, + true); + } + String procedure = + String.format( + isNamedArgument + ? "CALL sys.remove_orphan_files(`table` => '%s.%s', older_than => '2999-12-31 23:59:59')" + : "CALL sys.remove_orphan_files('%s.%s', '2999-12-31 23:59:59')", + database, + "*"); + ImmutableList actualDeleteFile = ImmutableList.copyOf(executeSQL(procedure)); + assertThat(actualDeleteFile).containsOnly(Row.of("4")); + } + + @ParameterizedTest + @ValueSource(booleans = {true, false}) + public void testRunWithMode(boolean isNamedArgument) throws Exception { + createTableAndWriteData(tableName); + + List args = + new ArrayList<>( + Arrays.asList( + "remove_orphan_files", + "--warehouse", + warehouse, + "--database", + database, + "--table", + tableName)); + RemoveOrphanFilesAction action1 = createAction(RemoveOrphanFilesAction.class, args); + assertThatCode(action1::run).doesNotThrowAnyException(); + + args.add("--older_than"); + args.add("2023-12-31 23:59:59"); + RemoveOrphanFilesAction action2 = createAction(RemoveOrphanFilesAction.class, args); + assertThatCode(action2::run).doesNotThrowAnyException(); + + String withoutOlderThan = + String.format( + isNamedArgument + ? "CALL sys.remove_orphan_files(`table` => '%s.%s')" + : "CALL sys.remove_orphan_files('%s.%s')", + database, + tableName); + CloseableIterator withoutOlderThanCollect = executeSQL(withoutOlderThan); + assertThat(ImmutableList.copyOf(withoutOlderThanCollect)).containsOnly(Row.of("0")); + + String withLocalMode = + String.format( + isNamedArgument + ? "CALL sys.remove_orphan_files(`table` => '%s.%s', older_than => '2999-12-31 23:59:59', dry_run => true, parallelism => 5, mode => 'local')" + : "CALL sys.remove_orphan_files('%s.%s', '2999-12-31 23:59:59', true, 5, 'local')", + database, + tableName); + ImmutableList actualLocalRunDeleteFile = + ImmutableList.copyOf(executeSQL(withLocalMode)); + assertThat(actualLocalRunDeleteFile).containsOnly(Row.of("2")); + + String withDistributedMode = + String.format( + isNamedArgument + ? "CALL sys.remove_orphan_files(`table` => '%s.%s', older_than => '2999-12-31 23:59:59', dry_run => true, parallelism => 5, mode => 'distributed')" + : "CALL sys.remove_orphan_files('%s.%s', '2999-12-31 23:59:59', true, 5, 'distributed')", + database, + tableName); + ImmutableList actualDistributedRunDeleteFile = + ImmutableList.copyOf(executeSQL(withDistributedMode)); + assertThat(actualDistributedRunDeleteFile).containsOnly(Row.of("2")); + + String withInvalidMode = + String.format( + isNamedArgument + ? "CALL sys.remove_orphan_files(`table` => '%s.%s', older_than => '2999-12-31 23:59:59', dry_run => true, parallelism => 5, mode => 'unknown')" + : "CALL sys.remove_orphan_files('%s.%s', '2999-12-31 23:59:59', true, 5, 'unknown')", + database, + tableName); + assertThatCode(() -> executeSQL(withInvalidMode)) + .isInstanceOf(RuntimeException.class) + .hasMessageContaining("Unknown mode"); + } +} diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/action/ReplaceTagActionTest.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/action/ReplaceTagActionTest.java new file mode 100644 index 000000000000..00b43b9e11c9 --- /dev/null +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/action/ReplaceTagActionTest.java @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.flink.action; + +import org.apache.paimon.table.FileStoreTable; +import org.apache.paimon.utils.TagManager; + +import org.assertj.core.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import static org.apache.paimon.flink.util.ReadWriteTableTestUtil.bEnv; +import static org.apache.paimon.flink.util.ReadWriteTableTestUtil.init; +import static org.assertj.core.api.Assertions.assertThat; + +/** IT cases for {@link ReplaceTagAction}. */ +public class ReplaceTagActionTest extends ActionITCaseBase { + + @BeforeEach + public void setUp() { + init(warehouse); + } + + @Test + public void testReplaceTag() throws Exception { + bEnv.executeSql( + "CREATE TABLE T (id INT, name STRING," + + " PRIMARY KEY (id) NOT ENFORCED)" + + " WITH ('bucket'='1')"); + + FileStoreTable table = getFileStoreTable("T"); + TagManager tagManager = table.tagManager(); + + bEnv.executeSql("INSERT INTO T VALUES (1, 'a')").await(); + bEnv.executeSql("INSERT INTO T VALUES (2, 'b')").await(); + assertThat(table.snapshotManager().snapshotCount()).isEqualTo(2); + + Assertions.assertThatThrownBy( + () -> + bEnv.executeSql( + "CALL sys.replace_tag(`table` => 'default.T', tag => 'test_tag')")) + .hasMessageContaining("Tag name 'test_tag' does not exist."); + + bEnv.executeSql("CALL sys.create_tag(`table` => 'default.T', tag => 'test_tag')"); + assertThat(tagManager.tagExists("test_tag")).isEqualTo(true); + assertThat(tagManager.tag("test_tag").trimToSnapshot().id()).isEqualTo(2); + assertThat(tagManager.tag("test_tag").getTagTimeRetained()).isEqualTo(null); + + // replace tag with new time_retained + createAction( + ReplaceTagAction.class, + "replace_tag", + "--warehouse", + warehouse, + "--database", + database, + "--table", + "T", + "--tag_name", + "test_tag", + "--time_retained", + "1 d") + .run(); + assertThat(tagManager.tag("test_tag").getTagTimeRetained().toHours()).isEqualTo(24); + + // replace tag with new snapshot and time_retained + createAction( + ReplaceTagAction.class, + "replace_tag", + "--warehouse", + warehouse, + "--database", + database, + "--table", + "T", + "--tag_name", + "test_tag", + "--snapshot", + "1", + "--time_retained", + "2 d") + .run(); + assertThat(tagManager.tag("test_tag").trimToSnapshot().id()).isEqualTo(1); + assertThat(tagManager.tag("test_tag").getTagTimeRetained().toHours()).isEqualTo(48); + } +} diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/action/RollbackToActionITCase.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/action/RollbackToActionITCase.java index 859f8deda4d1..7dc9fec643cf 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/action/RollbackToActionITCase.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/action/RollbackToActionITCase.java @@ -160,4 +160,59 @@ public void rollbackToTagTest(String invoker) throws Exception { "SELECT * FROM `" + tableName + "`", Arrays.asList(Row.of(1L, "Hi"), Row.of(2L, "Apache"))); } + + @ParameterizedTest + @ValueSource(strings = {"action", "procedure_named", "procedure_indexed"}) + public void rollbackToTimestampTest(String invoker) throws Exception { + FileStoreTable table = + createFileStoreTable( + ROW_TYPE, + Collections.emptyList(), + Collections.singletonList("k"), + Collections.emptyList(), + Collections.emptyMap()); + StreamWriteBuilder writeBuilder = table.newStreamWriteBuilder().withCommitUser(commitUser); + write = writeBuilder.newWrite(); + commit = writeBuilder.newCommit(); + + writeData(rowData(1L, BinaryString.fromString("Hi"))); + writeData(rowData(2L, BinaryString.fromString("Apache"))); + long timestamp = System.currentTimeMillis(); + writeData(rowData(2L, BinaryString.fromString("Paimon"))); + + switch (invoker) { + case "action": + createAction( + RollbackToTimestampAction.class, + "rollback_to_timestamp", + "--warehouse", + warehouse, + "--database", + database, + "--table", + tableName, + "--timestamp", + timestamp + "") + .run(); + break; + case "procedure_indexed": + executeSQL( + String.format( + "CALL sys.rollback_to_timestamp('%s.%s', %s)", + database, tableName, timestamp)); + break; + case "procedure_named": + executeSQL( + String.format( + "CALL sys.rollback_to_timestamp(`table` => '%s.%s', `timestamp` => %s)", + database, tableName, timestamp)); + break; + default: + throw new UnsupportedOperationException(invoker); + } + + testBatchRead( + "SELECT * FROM `" + tableName + "`", + Arrays.asList(Row.of(1L, "Hi"), Row.of(2L, "Apache"))); + } } diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/compact/UnawareBucketCompactorTest.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/compact/UnawareBucketCompactorTest.java index 69229ddce2f6..1ec26afabc55 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/compact/UnawareBucketCompactorTest.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/compact/UnawareBucketCompactorTest.java @@ -34,8 +34,10 @@ import org.apache.paimon.types.DataTypes; import org.apache.paimon.utils.ExecutorThreadFactory; +import org.apache.flink.metrics.Counter; import org.apache.flink.metrics.Gauge; import org.apache.flink.metrics.MetricGroup; +import org.apache.flink.metrics.SimpleCounter; import org.assertj.core.api.Assertions; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -48,6 +50,8 @@ import java.util.concurrent.Executors; import static org.apache.paimon.operation.metrics.CompactionMetrics.AVG_COMPACTION_TIME; +import static org.apache.paimon.operation.metrics.CompactionMetrics.COMPACTION_COMPLETED_COUNT; +import static org.apache.paimon.operation.metrics.CompactionMetrics.COMPACTION_QUEUED_COUNT; import static org.apache.paimon.operation.metrics.CompactionMetrics.COMPACTION_THREAD_BUSY; /** Test for {@link UnawareBucketCompactor}. */ @@ -65,7 +69,8 @@ public void testGaugeCollection() throws Exception { Executors.newSingleThreadScheduledExecutor( new ExecutorThreadFactory( Thread.currentThread().getName() + "-append-only-compact-worker")); - Map map = new HashMap<>(); + Map gaugeMap = new HashMap<>(); + Map counterMap = new HashMap<>(); UnawareBucketCompactor unawareBucketCompactor = new UnawareBucketCompactor( (FileStoreTable) catalog.getTable(identifier()), @@ -74,7 +79,7 @@ public void testGaugeCollection() throws Exception { new FileStoreSourceReaderTest.DummyMetricGroup() { @Override public > G gauge(String name, G gauge) { - map.put(name, gauge); + gaugeMap.put(name, gauge); return null; } @@ -87,6 +92,13 @@ public MetricGroup addGroup(String name) { public MetricGroup addGroup(String key, String value) { return this; } + + @Override + public Counter counter(String name) { + SimpleCounter counter = new SimpleCounter(); + counterMap.put(name, counter); + return counter; + } }); for (int i = 0; i < 320; i++) { @@ -94,11 +106,15 @@ public MetricGroup addGroup(String key, String value) { Thread.sleep(250); } - double compactionThreadBusy = (double) map.get(COMPACTION_THREAD_BUSY).getValue(); - double compactionAvrgTime = (double) map.get(AVG_COMPACTION_TIME).getValue(); + double compactionThreadBusy = (double) gaugeMap.get(COMPACTION_THREAD_BUSY).getValue(); + double compactionAvgTime = (double) gaugeMap.get(AVG_COMPACTION_TIME).getValue(); + long compactionsCompletedCount = counterMap.get(COMPACTION_COMPLETED_COUNT).getCount(); + long compactionsQueuedCount = counterMap.get(COMPACTION_QUEUED_COUNT).getCount(); Assertions.assertThat(compactionThreadBusy).isGreaterThan(45).isLessThan(55); - Assertions.assertThat(compactionAvrgTime).isGreaterThan(120).isLessThan(140); + Assertions.assertThat(compactionAvgTime).isGreaterThan(120).isLessThan(140); + Assertions.assertThat(compactionsCompletedCount).isEqualTo(320L); + Assertions.assertThat(compactionsQueuedCount).isEqualTo(0L); } protected Catalog getCatalog() { diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/compact/changelog/ChangelogCompactTaskSerializerTest.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/compact/changelog/ChangelogCompactTaskSerializerTest.java new file mode 100644 index 000000000000..906fac850973 --- /dev/null +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/compact/changelog/ChangelogCompactTaskSerializerTest.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.flink.compact.changelog; + +import org.apache.paimon.data.BinaryRow; +import org.apache.paimon.data.BinaryRowWriter; +import org.apache.paimon.io.DataFileMeta; +import org.apache.paimon.manifest.FileSource; + +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.UUID; + +import static org.apache.paimon.mergetree.compact.MergeTreeCompactManagerTest.row; +import static org.apache.paimon.stats.StatsTestUtils.newSimpleStats; +import static org.assertj.core.api.Assertions.assertThat; + +/** Test for {@link ChangelogCompactTaskSerializer}. */ +public class ChangelogCompactTaskSerializerTest { + private final ChangelogCompactTaskSerializer serializer = new ChangelogCompactTaskSerializer(); + + @Test + public void testSerializer() throws Exception { + BinaryRow partition = new BinaryRow(1); + BinaryRowWriter writer = new BinaryRowWriter(partition); + writer.writeInt(0, 0); + writer.complete(); + + ChangelogCompactTask task = + new ChangelogCompactTask( + 1L, + partition, + new HashMap>() { + { + put(0, newFiles(20)); + put(1, newFiles(20)); + } + }, + new HashMap>() { + { + put(0, newFiles(10)); + put(1, newFiles(10)); + } + }); + ChangelogCompactTask serializeTask = serializer.deserialize(1, serializer.serialize(task)); + assertThat(task).isEqualTo(serializeTask); + } + + private List newFiles(int num) { + List list = new ArrayList<>(); + for (int i = 0; i < num; i++) { + list.add(newFile()); + } + return list; + } + + private DataFileMeta newFile() { + return new DataFileMeta( + UUID.randomUUID().toString(), + 0, + 1, + row(0), + row(0), + newSimpleStats(0, 1), + newSimpleStats(0, 1), + 0, + 1, + 0, + 0, + 0L, + null, + FileSource.APPEND, + null); + } +} diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/iceberg/FlinkIcebergITCaseBase.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/iceberg/FlinkIcebergITCaseBase.java new file mode 100644 index 000000000000..9202cfb8fefb --- /dev/null +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/iceberg/FlinkIcebergITCaseBase.java @@ -0,0 +1,471 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.flink.iceberg; + +import org.apache.paimon.flink.util.AbstractTestBase; + +import org.apache.flink.table.api.TableEnvironment; +import org.apache.flink.table.api.TableResult; +import org.apache.flink.types.Row; +import org.apache.flink.util.CloseableIterator; +import org.apache.hadoop.conf.Configuration; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.data.IcebergGenerics; +import org.apache.iceberg.data.Record; +import org.apache.iceberg.expressions.Expressions; +import org.apache.iceberg.hadoop.HadoopCatalog; +import org.apache.iceberg.io.CloseableIterable; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import java.util.ArrayList; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; + +/** IT cases for Paimon Iceberg compatibility. */ +public abstract class FlinkIcebergITCaseBase extends AbstractTestBase { + + @ParameterizedTest + @ValueSource(strings = {"orc", "parquet", "avro"}) + public void testPrimaryKeyTable(String format) throws Exception { + String warehouse = getTempDirPath(); + TableEnvironment tEnv = tableEnvironmentBuilder().batchMode().parallelism(2).build(); + tEnv.executeSql( + "CREATE CATALOG paimon WITH (\n" + + " 'type' = 'paimon',\n" + + " 'warehouse' = '" + + warehouse + + "'\n" + + ")"); + tEnv.executeSql( + "CREATE TABLE paimon.`default`.T (\n" + + " pt INT,\n" + + " k INT,\n" + + " v1 INT,\n" + + " v2 STRING,\n" + + " PRIMARY KEY (pt, k) NOT ENFORCED\n" + + ") PARTITIONED BY (pt) WITH (\n" + + " 'metadata.iceberg.storage' = 'hadoop-catalog',\n" + // make sure all changes are visible in iceberg metadata + + " 'full-compaction.delta-commits' = '1',\n" + + " 'file.format' = '" + + format + + "'\n" + + ")"); + tEnv.executeSql( + "INSERT INTO paimon.`default`.T VALUES " + + "(1, 10, 100, 'apple'), " + + "(1, 11, 110, 'banana'), " + + "(2, 20, 200, 'cat'), " + + "(2, 21, 210, 'dog')") + .await(); + + tEnv.executeSql( + "CREATE CATALOG iceberg WITH (\n" + + " 'type' = 'iceberg',\n" + + " 'catalog-type' = 'hadoop',\n" + + " 'warehouse' = '" + + warehouse + + "/iceberg',\n" + + " 'cache-enabled' = 'false'\n" + + ")"); + assertThat( + collect( + tEnv.executeSql( + "SELECT v1, k, v2, pt FROM iceberg.`default`.T ORDER BY pt, k"))) + .containsExactly( + Row.of(100, 10, "apple", 1), + Row.of(110, 11, "banana", 1), + Row.of(200, 20, "cat", 2), + Row.of(210, 21, "dog", 2)); + + tEnv.executeSql( + "INSERT INTO paimon.`default`.T VALUES " + + "(1, 10, 101, 'red'), " + + "(1, 12, 121, 'green'), " + + "(2, 20, 201, 'blue'), " + + "(2, 22, 221, 'yellow')") + .await(); + assertThat( + collect( + tEnv.executeSql( + "SELECT v1, k, v2, pt FROM iceberg.`default`.T ORDER BY pt, k"))) + .containsExactly( + Row.of(101, 10, "red", 1), + Row.of(110, 11, "banana", 1), + Row.of(121, 12, "green", 1), + Row.of(201, 20, "blue", 2), + Row.of(210, 21, "dog", 2), + Row.of(221, 22, "yellow", 2)); + } + + @ParameterizedTest + @ValueSource(strings = {"orc", "parquet", "avro"}) + public void testAppendOnlyTable(String format) throws Exception { + String warehouse = getTempDirPath(); + TableEnvironment tEnv = tableEnvironmentBuilder().batchMode().parallelism(2).build(); + tEnv.executeSql( + "CREATE CATALOG paimon WITH (\n" + + " 'type' = 'paimon',\n" + + " 'warehouse' = '" + + warehouse + + "'\n" + + ")"); + tEnv.executeSql( + "CREATE TABLE paimon.`default`.cities (\n" + + " country STRING,\n" + + " name STRING\n" + + ") WITH (\n" + + " 'metadata.iceberg.storage' = 'hadoop-catalog',\n" + + " 'file.format' = '" + + format + + "'\n" + + ")"); + tEnv.executeSql( + "INSERT INTO paimon.`default`.cities VALUES " + + "('usa', 'new york'), " + + "('germany', 'berlin'), " + + "('usa', 'chicago'), " + + "('germany', 'hamburg')") + .await(); + + tEnv.executeSql( + "CREATE CATALOG iceberg WITH (\n" + + " 'type' = 'iceberg',\n" + + " 'catalog-type' = 'hadoop',\n" + + " 'warehouse' = '" + + warehouse + + "/iceberg',\n" + + " 'cache-enabled' = 'false'\n" + + ")"); + assertThat(collect(tEnv.executeSql("SELECT name, country FROM iceberg.`default`.cities"))) + .containsExactlyInAnyOrder( + Row.of("new york", "usa"), + Row.of("chicago", "usa"), + Row.of("berlin", "germany"), + Row.of("hamburg", "germany")); + + tEnv.executeSql( + "INSERT INTO paimon.`default`.cities VALUES " + + "('usa', 'houston'), " + + "('germany', 'munich')") + .await(); + assertThat( + collect( + tEnv.executeSql( + "SELECT name FROM iceberg.`default`.cities WHERE country = 'germany'"))) + .containsExactlyInAnyOrder(Row.of("berlin"), Row.of("hamburg"), Row.of("munich")); + } + + @ParameterizedTest + @ValueSource(strings = {"orc", "parquet", "avro"}) + public void testFilterAllTypes(String format) throws Exception { + String warehouse = getTempDirPath(); + TableEnvironment tEnv = tableEnvironmentBuilder().batchMode().parallelism(2).build(); + tEnv.executeSql( + "CREATE CATALOG paimon WITH (\n" + + " 'type' = 'paimon',\n" + + " 'warehouse' = '" + + warehouse + + "'\n" + + ")"); + tEnv.executeSql( + "CREATE TABLE paimon.`default`.T (\n" + + " pt INT,\n" + + " id INT," + + " v_int INT,\n" + + " v_boolean BOOLEAN,\n" + + " v_bigint BIGINT,\n" + + " v_float FLOAT,\n" + + " v_double DOUBLE,\n" + + " v_decimal DECIMAL(8, 3),\n" + + " v_varchar STRING,\n" + + " v_varbinary VARBINARY(20),\n" + + " v_date DATE,\n" + // it seems that Iceberg Flink connector has some bug when filtering a + // timestamp_ltz, so we don't test it here + + " v_timestamp TIMESTAMP(6)\n" + + ") PARTITIONED BY (pt) WITH (\n" + + " 'metadata.iceberg.storage' = 'hadoop-catalog',\n" + + " 'file.format' = '" + + format + + "'\n" + + ")"); + tEnv.executeSql( + "INSERT INTO paimon.`default`.T VALUES " + + "(1, 1, 1, true, 10, CAST(100.0 AS FLOAT), 1000.0, 123.456, 'cat', CAST('B_cat' AS VARBINARY(20)), DATE '2024-10-10', TIMESTAMP '2024-10-10 11:22:33.123456'), " + + "(2, 2, 2, false, 20, CAST(200.0 AS FLOAT), 2000.0, 234.567, 'dog', CAST('B_dog' AS VARBINARY(20)), DATE '2024-10-20', TIMESTAMP '2024-10-20 11:22:33.123456'), " + + "(3, 3, CAST(NULL AS INT), CAST(NULL AS BOOLEAN), CAST(NULL AS BIGINT), CAST(NULL AS FLOAT), CAST(NULL AS DOUBLE), CAST(NULL AS DECIMAL(8, 3)), CAST(NULL AS STRING), CAST(NULL AS VARBINARY(20)), CAST(NULL AS DATE), CAST(NULL AS TIMESTAMP(6)))") + .await(); + + tEnv.executeSql( + "CREATE CATALOG iceberg WITH (\n" + + " 'type' = 'iceberg',\n" + + " 'catalog-type' = 'hadoop',\n" + + " 'warehouse' = '" + + warehouse + + "/iceberg',\n" + + " 'cache-enabled' = 'false'\n" + + ")"); + tEnv.executeSql("USE CATALOG iceberg"); + assertThat(collect(tEnv.executeSql("SELECT id FROM T where pt = 1"))) + .containsExactly(Row.of(1)); + assertThat(collect(tEnv.executeSql("SELECT id FROM T where v_int = 1"))) + .containsExactly(Row.of(1)); + assertThat(collect(tEnv.executeSql("SELECT id FROM T where v_boolean = true"))) + .containsExactly(Row.of(1)); + assertThat(collect(tEnv.executeSql("SELECT id FROM T where v_bigint = 10"))) + .containsExactly(Row.of(1)); + assertThat(collect(tEnv.executeSql("SELECT id FROM T where v_float = 100.0"))) + .containsExactly(Row.of(1)); + assertThat(collect(tEnv.executeSql("SELECT id FROM T where v_double = 1000.0"))) + .containsExactly(Row.of(1)); + assertThat(collect(tEnv.executeSql("SELECT id FROM T where v_decimal = 123.456"))) + .containsExactly(Row.of(1)); + assertThat(collect(tEnv.executeSql("SELECT id FROM T where v_varchar = 'cat'"))) + .containsExactly(Row.of(1)); + assertThat(collect(tEnv.executeSql("SELECT id FROM T where v_date = '2024-10-10'"))) + .containsExactly(Row.of(1)); + assertThat( + collect( + tEnv.executeSql( + "SELECT id FROM T where v_timestamp = TIMESTAMP '2024-10-10 11:22:33.123456'"))) + .containsExactly(Row.of(1)); + assertThat(collect(tEnv.executeSql("SELECT id FROM T where v_int IS NULL"))) + .containsExactly(Row.of(3)); + assertThat(collect(tEnv.executeSql("SELECT id FROM T where v_boolean IS NULL"))) + .containsExactly(Row.of(3)); + assertThat(collect(tEnv.executeSql("SELECT id FROM T where v_bigint IS NULL"))) + .containsExactly(Row.of(3)); + assertThat(collect(tEnv.executeSql("SELECT id FROM T where v_float IS NULL"))) + .containsExactly(Row.of(3)); + assertThat(collect(tEnv.executeSql("SELECT id FROM T where v_double IS NULL"))) + .containsExactly(Row.of(3)); + assertThat(collect(tEnv.executeSql("SELECT id FROM T where v_decimal IS NULL"))) + .containsExactly(Row.of(3)); + assertThat(collect(tEnv.executeSql("SELECT id FROM T where v_varchar IS NULL"))) + .containsExactly(Row.of(3)); + assertThat(collect(tEnv.executeSql("SELECT id FROM T where v_varbinary IS NULL"))) + .containsExactly(Row.of(3)); + assertThat(collect(tEnv.executeSql("SELECT id FROM T where v_date IS NULL"))) + .containsExactly(Row.of(3)); + assertThat(collect(tEnv.executeSql("SELECT id FROM T where v_timestamp IS NULL"))) + .containsExactly(Row.of(3)); + } + + @ParameterizedTest + // orc writer does not write timestamp_ltz correctly, however we won't fix it due to + // compatibility concern, so we don't test orc here + @ValueSource(strings = {"parquet"}) + public void testFilterTimestampLtz(String format) throws Exception { + String warehouse = getTempDirPath(); + TableEnvironment tEnv = tableEnvironmentBuilder().batchMode().parallelism(2).build(); + tEnv.executeSql( + "CREATE CATALOG paimon WITH (\n" + + " 'type' = 'paimon',\n" + + " 'warehouse' = '" + + warehouse + + "'\n" + + ")"); + tEnv.executeSql( + "CREATE TABLE paimon.`default`.T (\n" + + " id INT," + + " v_timestampltz TIMESTAMP_LTZ(6)\n" + + ") WITH (\n" + + " 'metadata.iceberg.storage' = 'hadoop-catalog',\n" + + " 'file.format' = '" + + format + + "'\n" + + ")"); + tEnv.executeSql( + "INSERT INTO paimon.`default`.T VALUES " + + "(1, CAST(TO_TIMESTAMP_LTZ(1100000000321, 3) AS TIMESTAMP_LTZ(6))), " + + "(2, CAST(TO_TIMESTAMP_LTZ(1200000000321, 3) AS TIMESTAMP_LTZ(6))), " + + "(3, CAST(NULL AS TIMESTAMP_LTZ(6)))") + .await(); + + HadoopCatalog icebergCatalog = + new HadoopCatalog(new Configuration(), warehouse + "/iceberg"); + TableIdentifier icebergIdentifier = TableIdentifier.of("default", "T"); + org.apache.iceberg.Table icebergTable = icebergCatalog.loadTable(icebergIdentifier); + + CloseableIterable result = + IcebergGenerics.read(icebergTable) + .where(Expressions.equal("v_timestampltz", 1100000000321000L)) + .build(); + List actual = new ArrayList<>(); + for (Record record : result) { + actual.add(record.get(0)); + } + result.close(); + assertThat(actual).containsExactly(1); + + result = + IcebergGenerics.read(icebergTable) + .where(Expressions.isNull("v_timestampltz")) + .build(); + actual = new ArrayList<>(); + for (Record record : result) { + actual.add(record.get(0)); + } + result.close(); + assertThat(actual).containsExactly(3); + } + + @ParameterizedTest + @ValueSource(strings = {"orc", "parquet", "avro"}) + public void testDropAndRecreateTable(String format) throws Exception { + String warehouse = getTempDirPath(); + TableEnvironment tEnv = tableEnvironmentBuilder().batchMode().parallelism(2).build(); + tEnv.executeSql( + "CREATE CATALOG paimon WITH (\n" + + " 'type' = 'paimon',\n" + + " 'warehouse' = '" + + warehouse + + "'\n" + + ")"); + String createTableDdl = + "CREATE TABLE paimon.`default`.cities (\n" + + " country STRING,\n" + + " name STRING\n" + + ") WITH (\n" + + " 'metadata.iceberg.storage' = 'hadoop-catalog',\n" + + " 'file.format' = '" + + format + + "'\n" + + ")"; + tEnv.executeSql(createTableDdl); + tEnv.executeSql( + "INSERT INTO paimon.`default`.cities VALUES " + + "('usa', 'new york'), " + + "('germany', 'berlin')") + .await(); + + tEnv.executeSql( + "CREATE CATALOG iceberg WITH (\n" + + " 'type' = 'iceberg',\n" + + " 'catalog-type' = 'hadoop',\n" + + " 'warehouse' = '" + + warehouse + + "/iceberg',\n" + + " 'cache-enabled' = 'false'\n" + + ")"); + assertThat(collect(tEnv.executeSql("SELECT name, country FROM iceberg.`default`.cities"))) + .containsExactlyInAnyOrder(Row.of("new york", "usa"), Row.of("berlin", "germany")); + + tEnv.executeSql( + "INSERT INTO paimon.`default`.cities VALUES " + + "('usa', 'chicago'), " + + "('germany', 'hamburg')") + .await(); + assertThat(collect(tEnv.executeSql("SELECT name, country FROM iceberg.`default`.cities"))) + .containsExactlyInAnyOrder( + Row.of("new york", "usa"), + Row.of("chicago", "usa"), + Row.of("berlin", "germany"), + Row.of("hamburg", "germany")); + + tEnv.executeSql("DROP TABLE paimon.`default`.cities"); + tEnv.executeSql(createTableDdl); + tEnv.executeSql( + "INSERT INTO paimon.`default`.cities VALUES " + + "('usa', 'houston'), " + + "('germany', 'munich')") + .await(); + assertThat(collect(tEnv.executeSql("SELECT name, country FROM iceberg.`default`.cities"))) + .containsExactlyInAnyOrder(Row.of("houston", "usa"), Row.of("munich", "germany")); + + tEnv.executeSql( + "INSERT INTO paimon.`default`.cities VALUES " + + "('usa', 'san francisco'), " + + "('germany', 'cologne')") + .await(); + assertThat( + collect( + tEnv.executeSql( + "SELECT name FROM iceberg.`default`.cities WHERE country = 'germany'"))) + .containsExactlyInAnyOrder(Row.of("munich"), Row.of("cologne")); + } + + @ParameterizedTest + @ValueSource(strings = {"orc", "parquet", "avro"}) + public void testNestedTypes(String format) throws Exception { + String warehouse = getTempDirPath(); + TableEnvironment tEnv = tableEnvironmentBuilder().batchMode().parallelism(2).build(); + tEnv.executeSql( + "CREATE CATALOG paimon WITH (\n" + + " 'type' = 'paimon',\n" + + " 'warehouse' = '" + + warehouse + + "'\n" + + ")"); + tEnv.executeSql( + "CREATE TABLE paimon.`default`.T (\n" + + " k INT,\n" + + " v MAP>,\n" + + " v2 BIGINT\n" + + ") WITH (\n" + + " 'metadata.iceberg.storage' = 'hadoop-catalog',\n" + + " 'file.format' = '" + + format + + "'\n" + + ")"); + tEnv.executeSql( + "INSERT INTO paimon.`default`.T VALUES " + + "(1, MAP[10, ARRAY[ROW('apple', 100), ROW('banana', 101)], 20, ARRAY[ROW('cat', 102), ROW('dog', 103)]], 1000), " + + "(2, MAP[10, ARRAY[ROW('cherry', 200), ROW('pear', 201)], 20, ARRAY[ROW('tiger', 202), ROW('wolf', 203)]], 2000)") + .await(); + + tEnv.executeSql( + "CREATE CATALOG iceberg WITH (\n" + + " 'type' = 'iceberg',\n" + + " 'catalog-type' = 'hadoop',\n" + + " 'warehouse' = '" + + warehouse + + "/iceberg',\n" + + " 'cache-enabled' = 'false'\n" + + ")"); + assertThat(collect(tEnv.executeSql("SELECT k, v[10], v2 FROM iceberg.`default`.T"))) + .containsExactlyInAnyOrder( + Row.of(1, new Row[] {Row.of("apple", 100), Row.of("banana", 101)}, 1000L), + Row.of(2, new Row[] {Row.of("cherry", 200), Row.of("pear", 201)}, 2000L)); + + tEnv.executeSql( + "INSERT INTO paimon.`default`.T VALUES " + + "(3, MAP[10, ARRAY[ROW('mango', 300), ROW('watermelon', 301)], 20, ARRAY[ROW('rabbit', 302), ROW('lion', 303)]], 3000)") + .await(); + assertThat( + collect( + tEnv.executeSql( + "SELECT k, v[10][2].f1, v2 FROM iceberg.`default`.T WHERE v[20][1].f2 > 200"))) + .containsExactlyInAnyOrder( + Row.of(2, "pear", 2000L), Row.of(3, "watermelon", 3000L)); + } + + private List collect(TableResult result) throws Exception { + List rows = new ArrayList<>(); + try (CloseableIterator it = result.collect()) { + while (it.hasNext()) { + rows.add(it.next()); + } + } + return rows; + } +} diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/lookup/FileStoreLookupFunctionTest.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/lookup/FileStoreLookupFunctionTest.java index f8c8adcb2bf3..2b219f6a712a 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/lookup/FileStoreLookupFunctionTest.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/lookup/FileStoreLookupFunctionTest.java @@ -34,6 +34,7 @@ import org.apache.paimon.service.ServiceManager; import org.apache.paimon.table.FileStoreTable; import org.apache.paimon.table.FileStoreTableFactory; +import org.apache.paimon.table.Table; import org.apache.paimon.table.sink.CommitMessage; import org.apache.paimon.table.sink.StreamTableWrite; import org.apache.paimon.table.sink.TableCommitImpl; @@ -52,6 +53,9 @@ import java.net.InetSocketAddress; import java.nio.file.Path; import java.time.Duration; +import java.time.Instant; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -59,8 +63,11 @@ import java.util.Random; import java.util.UUID; +import static org.apache.paimon.flink.FlinkConnectorOptions.LOOKUP_REFRESH_TIME_PERIODS_BLACKLIST; import static org.apache.paimon.service.ServiceManager.PRIMARY_KEY_LOOKUP; +import static org.apache.paimon.testutils.assertj.PaimonAssertions.anyCauseMatches; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.AssertionsForClassTypes.assertThatThrownBy; /** Tests for {@link FileStoreLookupFunction}. */ public class FileStoreLookupFunctionTest { @@ -91,6 +98,18 @@ private void createLookupFunction( boolean dynamicPartition, boolean refreshAsync) throws Exception { + table = createFileStoreTable(isPartition, dynamicPartition, refreshAsync); + lookupFunction = createLookupFunction(table, joinEqualPk); + lookupFunction.open(tempDir.toString()); + } + + private FileStoreLookupFunction createLookupFunction(Table table, boolean joinEqualPk) { + return new FileStoreLookupFunction( + table, new int[] {0, 1}, joinEqualPk ? new int[] {0, 1} : new int[] {1}, null); + } + + private FileStoreTable createFileStoreTable( + boolean isPartition, boolean dynamicPartition, boolean refreshAsync) throws Exception { SchemaManager schemaManager = new SchemaManager(fileIO, tablePath); Options conf = new Options(); conf.set(FlinkConnectorOptions.LOOKUP_REFRESH_ASYNC, refreshAsync); @@ -106,7 +125,6 @@ private void createLookupFunction( RowType.of( new DataType[] {DataTypes.INT(), DataTypes.INT(), DataTypes.BIGINT()}, new String[] {"pt", "k", "v"}); - Schema schema = new Schema( rowType.getFields(), @@ -115,17 +133,8 @@ private void createLookupFunction( conf.toMap(), ""); TableSchema tableSchema = schemaManager.createTable(schema); - table = - FileStoreTableFactory.create( - fileIO, new org.apache.paimon.fs.Path(tempDir.toString()), tableSchema); - - lookupFunction = - new FileStoreLookupFunction( - table, - new int[] {0, 1}, - joinEqualPk ? new int[] {0, 1} : new int[] {1}, - null); - lookupFunction.open(tempDir.toString()); + return FileStoreTableFactory.create( + fileIO, new org.apache.paimon.fs.Path(tempDir.toString()), tableSchema); } @AfterEach @@ -214,6 +223,68 @@ public void testLookupDynamicPartition() throws Exception { .isEqualTo(0); } + @Test + public void testParseWrongTimePeriodsBlacklist() throws Exception { + Table table = createFileStoreTable(false, false, false); + + Table table1 = + table.copy( + Collections.singletonMap( + LOOKUP_REFRESH_TIME_PERIODS_BLACKLIST.key(), + "2024-10-31 12:00,2024-10-31 16:00")); + assertThatThrownBy(() -> createLookupFunction(table1, true)) + .satisfies( + anyCauseMatches( + IllegalArgumentException.class, + "Incorrect time periods format: [2024-10-31 12:00,2024-10-31 16:00].")); + + Table table2 = + table.copy( + Collections.singletonMap( + LOOKUP_REFRESH_TIME_PERIODS_BLACKLIST.key(), + "20241031 12:00->20241031 16:00")); + assertThatThrownBy(() -> createLookupFunction(table2, true)) + .satisfies( + anyCauseMatches( + IllegalArgumentException.class, + "Date time format error: [20241031 12:00]")); + + Table table3 = + table.copy( + Collections.singletonMap( + LOOKUP_REFRESH_TIME_PERIODS_BLACKLIST.key(), + "2024-10-31 12:00->2024-10-31 16:00,2024-10-31 20:00->2024-10-31 18:00")); + assertThatThrownBy(() -> createLookupFunction(table3, true)) + .satisfies( + anyCauseMatches( + IllegalArgumentException.class, + "Incorrect time period: [2024-10-31 20:00->2024-10-31 18:00]")); + } + + @Test + public void testCheckRefreshInBlacklist() throws Exception { + Instant now = Instant.now(); + Instant start = Instant.ofEpochSecond(now.getEpochSecond() / 60 * 60); + Instant end = start.plusSeconds(30 * 60); + + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm"); + String left = start.atZone(ZoneId.systemDefault()).format(formatter); + String right = end.atZone(ZoneId.systemDefault()).format(formatter); + + Table table = + createFileStoreTable(false, false, false) + .copy( + Collections.singletonMap( + LOOKUP_REFRESH_TIME_PERIODS_BLACKLIST.key(), + left + "->" + right)); + + FileStoreLookupFunction lookupFunction = createLookupFunction(table, true); + + lookupFunction.tryRefresh(); + + assertThat(lookupFunction.nextBlacklistCheckTime()).isEqualTo(end.toEpochMilli() + 1); + } + private void commit(List messages) throws Exception { TableCommitImpl commit = table.newCommit(commitUser); commit.commit(messages); diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/lookup/LookupTableTest.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/lookup/LookupTableTest.java index 619cb4c1d620..46c61a15bd8a 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/lookup/LookupTableTest.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/lookup/LookupTableTest.java @@ -559,6 +559,129 @@ public void testNoPrimaryKeyTableFilter() throws Exception { assertRow(result.get(1), 1, 11, 111); } + @Test + public void testPkTableWithCacheRowFilter() throws Exception { + FileStoreTable storeTable = createTable(singletonList("f0"), new Options()); + writeWithBucketAssigner( + storeTable, row -> 0, GenericRow.of(1, 11, 111), GenericRow.of(2, 22, 222)); + + FullCacheLookupTable.Context context = + new FullCacheLookupTable.Context( + storeTable, + new int[] {0, 1, 2}, + null, + null, + tempDir.toFile(), + singletonList("f0"), + null); + table = FullCacheLookupTable.create(context, ThreadLocalRandom.current().nextInt(2) * 10); + assertThat(table).isInstanceOf(PrimaryKeyLookupTable.class); + table.specifyCacheRowFilter(row -> row.getInt(0) < 2); + table.open(); + + List res = table.get(GenericRow.of(1)); + assertThat(res).hasSize(1); + assertRow(res.get(0), 1, 11, 111); + + res = table.get(GenericRow.of(2)); + assertThat(res).isEmpty(); + + writeWithBucketAssigner( + storeTable, row -> 0, GenericRow.of(0, 0, 0), GenericRow.of(3, 33, 333)); + res = table.get(GenericRow.of(0)); + assertThat(res).isEmpty(); + + table.refresh(); + res = table.get(GenericRow.of(0)); + assertThat(res).hasSize(1); + assertRow(res.get(0), 0, 0, 0); + + res = table.get(GenericRow.of(3)); + assertThat(res).isEmpty(); + } + + @Test + public void testNoPkTableWithCacheRowFilter() throws Exception { + FileStoreTable storeTable = createTable(emptyList(), new Options()); + writeWithBucketAssigner( + storeTable, row -> 0, GenericRow.of(1, 11, 111), GenericRow.of(2, 22, 222)); + + FullCacheLookupTable.Context context = + new FullCacheLookupTable.Context( + storeTable, + new int[] {0, 1, 2}, + null, + null, + tempDir.toFile(), + singletonList("f0"), + null); + table = FullCacheLookupTable.create(context, ThreadLocalRandom.current().nextInt(2) * 10); + assertThat(table).isInstanceOf(NoPrimaryKeyLookupTable.class); + table.specifyCacheRowFilter(row -> row.getInt(0) < 2); + table.open(); + + List res = table.get(GenericRow.of(1)); + assertThat(res).hasSize(1); + assertRow(res.get(0), 1, 11, 111); + + res = table.get(GenericRow.of(2)); + assertThat(res).isEmpty(); + + writeWithBucketAssigner( + storeTable, row -> 0, GenericRow.of(0, 0, 0), GenericRow.of(3, 33, 333)); + res = table.get(GenericRow.of(0)); + assertThat(res).isEmpty(); + + table.refresh(); + res = table.get(GenericRow.of(0)); + assertThat(res).hasSize(1); + assertRow(res.get(0), 0, 0, 0); + + res = table.get(GenericRow.of(3)); + assertThat(res).isEmpty(); + } + + @Test + public void testSecKeyTableWithCacheRowFilter() throws Exception { + FileStoreTable storeTable = createTable(singletonList("f0"), new Options()); + writeWithBucketAssigner( + storeTable, row -> 0, GenericRow.of(1, 11, 111), GenericRow.of(2, 22, 222)); + + FullCacheLookupTable.Context context = + new FullCacheLookupTable.Context( + storeTable, + new int[] {0, 1, 2}, + null, + null, + tempDir.toFile(), + singletonList("f1"), + null); + table = FullCacheLookupTable.create(context, ThreadLocalRandom.current().nextInt(2) * 10); + assertThat(table).isInstanceOf(SecondaryIndexLookupTable.class); + table.specifyCacheRowFilter(row -> row.getInt(1) < 22); + table.open(); + + List res = table.get(GenericRow.of(11)); + assertThat(res).hasSize(1); + assertRow(res.get(0), 1, 11, 111); + + res = table.get(GenericRow.of(22)); + assertThat(res).isEmpty(); + + writeWithBucketAssigner( + storeTable, row -> 0, GenericRow.of(0, 0, 0), GenericRow.of(3, 33, 333)); + res = table.get(GenericRow.of(0)); + assertThat(res).isEmpty(); + + table.refresh(); + res = table.get(GenericRow.of(0)); + assertThat(res).hasSize(1); + assertRow(res.get(0), 0, 0, 0); + + res = table.get(GenericRow.of(33)); + assertThat(res).isEmpty(); + } + @Test public void testPartialLookupTable() throws Exception { FileStoreTable dimTable = createDimTable(); @@ -592,6 +715,27 @@ public void testPartialLookupTable() throws Exception { assertThat(result).hasSize(0); } + @Test + public void testPartialLookupTableWithRowFilter() throws Exception { + Options options = new Options(); + options.set(CoreOptions.BUCKET.key(), "2"); + options.set(CoreOptions.BUCKET_KEY.key(), "f0"); + FileStoreTable dimTable = createTable(singletonList("f0"), options); + write(dimTable, GenericRow.of(1, 11, 111), GenericRow.of(2, 22, 222)); + + PrimaryKeyPartialLookupTable table = + PrimaryKeyPartialLookupTable.createLocalTable( + dimTable, new int[] {0, 2}, tempDir.toFile(), ImmutableList.of("f0"), null); + table.specifyCacheRowFilter(row -> row.getInt(0) < 2); + table.open(); + + List result = table.get(row(1, 11)); + assertThat(result).hasSize(1); + + result = table.get(row(2, 22)); + assertThat(result).isEmpty(); + } + @Test public void testPartialLookupTableWithProjection() throws Exception { FileStoreTable dimTable = createDimTable(); diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/procedure/CompactManifestProcedureITCase.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/procedure/CompactManifestProcedureITCase.java new file mode 100644 index 000000000000..89cdc48d85a0 --- /dev/null +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/procedure/CompactManifestProcedureITCase.java @@ -0,0 +1,133 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.flink.procedure; + +import org.apache.paimon.flink.CatalogITCaseBase; + +import org.assertj.core.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.Objects; + +/** IT Case for {@link CompactManifestProcedure}. */ +public class CompactManifestProcedureITCase extends CatalogITCaseBase { + + @Test + public void testManifestCompactProcedure() { + sql( + "CREATE TABLE T (" + + " k INT," + + " v STRING," + + " hh INT," + + " dt STRING" + + ") PARTITIONED BY (dt, hh) WITH (" + + " 'write-only' = 'true'," + + " 'file.format' = 'parquet'," + + " 'manifest.full-compaction-threshold-size' = '10000 T'," + + " 'bucket' = '-1'" + + ")"); + + sql( + "INSERT INTO T VALUES (1, '10', 15, '20221208'), (4, '100', 16, '20221208'), (5, '1000', 15, '20221209')"); + + sql( + "INSERT OVERWRITE T VALUES (1, '10', 15, '20221208'), (4, '100', 16, '20221208'), (5, '1000', 15, '20221209')"); + + sql( + "INSERT OVERWRITE T VALUES (1, '10', 15, '20221208'), (4, '100', 16, '20221208'), (5, '1000', 15, '20221209')"); + + sql( + "INSERT OVERWRITE T VALUES (1, '101', 15, '20221208'), (4, '1001', 16, '20221208'), (5, '10001', 15, '20221209')"); + + Assertions.assertThat( + sql("SELECT sum(num_deleted_files) FROM T$manifests").get(0).getField(0)) + .isEqualTo(9L); + + Assertions.assertThat( + Objects.requireNonNull( + sql("CALL sys.compact_manifest(`table` => 'default.T')") + .get(0) + .getField(0)) + .toString()) + .isEqualTo("success"); + + Assertions.assertThat( + sql("SELECT sum(num_deleted_files) FROM T$manifests").get(0).getField(0)) + .isEqualTo(0L); + + Assertions.assertThat(sql("SELECT * FROM T ORDER BY k").toString()) + .isEqualTo( + "[+I[1, 101, 15, 20221208], +I[4, 1001, 16, 20221208], +I[5, 10001, 15, 20221209]]"); + } + + @Test + public void testManifestCompactProcedureWithBranch() { + sql( + "CREATE TABLE T (" + + " k INT," + + " v STRING," + + " hh INT," + + " dt STRING" + + ") PARTITIONED BY (dt, hh) WITH (" + + " 'write-only' = 'true'," + + " 'manifest.full-compaction-threshold-size' = '10000 T'," + + " 'bucket' = '-1'" + + ")"); + + sql( + "INSERT INTO `T` VALUES (1, '10', 15, '20221208'), (4, '100', 16, '20221208'), (5, '1000', 15, '20221209')"); + + sql("CALL sys.create_tag('default.T', 'tag1', 1)"); + + sql("call sys.create_branch('default.T', 'branch1', 'tag1')"); + + sql( + "INSERT OVERWRITE T$branch_branch1 VALUES (1, '10', 15, '20221208'), (4, '100', 16, '20221208'), (5, '1000', 15, '20221209')"); + + sql( + "INSERT OVERWRITE T$branch_branch1 VALUES (1, '10', 15, '20221208'), (4, '100', 16, '20221208'), (5, '1000', 15, '20221209')"); + + sql( + "INSERT OVERWRITE T$branch_branch1 VALUES (1, '101', 15, '20221208'), (4, '1001', 16, '20221208'), (5, '10001', 15, '20221209')"); + + Assertions.assertThat( + sql("SELECT sum(num_deleted_files) FROM T$branch_branch1$manifests") + .get(0) + .getField(0)) + .isEqualTo(9L); + + Assertions.assertThat( + Objects.requireNonNull( + sql("CALL sys.compact_manifest(`table` => 'default.T$branch_branch1')") + .get(0) + .getField(0)) + .toString()) + .isEqualTo("success"); + + Assertions.assertThat( + sql("SELECT sum(num_deleted_files) FROM T$branch_branch1$manifests") + .get(0) + .getField(0)) + .isEqualTo(0L); + + Assertions.assertThat(sql("SELECT * FROM T$branch_branch1 ORDER BY k").toString()) + .isEqualTo( + "[+I[1, 101, 15, 20221208], +I[4, 1001, 16, 20221208], +I[5, 10001, 15, 20221209]]"); + } +} diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/procedure/CompactProcedureITCase.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/procedure/CompactProcedureITCase.java index bec669acd30d..d79d13f0260c 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/procedure/CompactProcedureITCase.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/procedure/CompactProcedureITCase.java @@ -31,6 +31,7 @@ import org.apache.flink.table.api.config.TableConfigOptions; import org.apache.flink.types.Row; import org.apache.flink.types.RowKind; +import org.assertj.core.api.Assertions; import org.junit.jupiter.api.Test; import java.util.List; @@ -240,6 +241,117 @@ public void testDynamicBucketSortCompact() throws Exception { checkLatestSnapshot(table, 21, Snapshot.CommitKind.OVERWRITE); } + // ----------------------- Minor Compact ----------------------- + + @Test + public void testBatchMinorCompactStrategy() throws Exception { + sql( + "CREATE TABLE T (" + + " k INT," + + " v INT," + + " hh INT," + + " dt STRING," + + " PRIMARY KEY (k, dt, hh) NOT ENFORCED" + + ") PARTITIONED BY (dt, hh) WITH (" + + " 'write-only' = 'true'," + + " 'bucket' = '1'" + + ")"); + FileStoreTable table = paimonTable("T"); + tEnv.getConfig().set(TableConfigOptions.TABLE_DML_SYNC, true); + + sql("INSERT INTO T VALUES (1, 100, 15, '20221208'), (1, 100, 16, '20221208')"); + sql("INSERT INTO T VALUES (2, 100, 15, '20221208'), (2, 100, 16, '20221208')"); + + checkLatestSnapshot(table, 2, Snapshot.CommitKind.APPEND); + + sql( + "CALL sys.compact(`table` => 'default.T', compact_strategy => 'minor', " + + "options => 'num-sorted-run.compaction-trigger=3')"); + + // Due to the limitation of parameter 'num-sorted-run.compaction-trigger' = 3, so compact is + // not + // performed. + checkLatestSnapshot(table, 2, Snapshot.CommitKind.APPEND); + + // Make par-15 has 3 datafile and par-16 has 2 datafile, so par-16 will not be picked out to + // compact. + sql("INSERT INTO T VALUES (1, 100, 15, '20221208')"); + + sql( + "CALL sys.compact(`table` => 'default.T', compact_strategy => 'minor', " + + "options => 'num-sorted-run.compaction-trigger=3')"); + + checkLatestSnapshot(table, 4, Snapshot.CommitKind.COMPACT); + + List splits = table.newSnapshotReader().read().dataSplits(); + assertThat(splits.size()).isEqualTo(2); + for (DataSplit split : splits) { + // Par-16 is not compacted. + assertThat(split.dataFiles().size()) + .isEqualTo(split.partition().getInt(1) == 16 ? 2 : 1); + } + } + + @Test + public void testBatchFullCompactStrategy() throws Exception { + sql( + "CREATE TABLE T (" + + " k INT," + + " v INT," + + " hh INT," + + " dt STRING," + + " PRIMARY KEY (k, dt, hh) NOT ENFORCED" + + ") PARTITIONED BY (dt, hh) WITH (" + + " 'write-only' = 'true'," + + " 'bucket' = '1'" + + ")"); + FileStoreTable table = paimonTable("T"); + tEnv.getConfig().set(TableConfigOptions.TABLE_DML_SYNC, true); + + sql("INSERT INTO T VALUES (1, 100, 15, '20221208'), (1, 100, 16, '20221208')"); + sql("INSERT INTO T VALUES (2, 100, 15, '20221208'), (2, 100, 16, '20221208')"); + + checkLatestSnapshot(table, 2, Snapshot.CommitKind.APPEND); + + sql( + "CALL sys.compact(`table` => 'default.T', compact_strategy => 'full', " + + "options => 'num-sorted-run.compaction-trigger=3')"); + + checkLatestSnapshot(table, 3, Snapshot.CommitKind.COMPACT); + + List splits = table.newSnapshotReader().read().dataSplits(); + assertThat(splits.size()).isEqualTo(2); + for (DataSplit split : splits) { + // Par-16 is not compacted. + assertThat(split.dataFiles().size()).isEqualTo(1); + } + } + + @Test + public void testStreamFullCompactStrategy() throws Exception { + sql( + "CREATE TABLE T (" + + " k INT," + + " v INT," + + " hh INT," + + " dt STRING," + + " PRIMARY KEY (k, dt, hh) NOT ENFORCED" + + ") PARTITIONED BY (dt, hh) WITH (" + + " 'write-only' = 'true'," + + " 'bucket' = '1'" + + ")"); + tEnv.getConfig().set(TableConfigOptions.TABLE_DML_SYNC, true); + + Assertions.assertThatThrownBy( + () -> + streamSqlIter( + "CALL sys.compact(`table` => 'default.T', compact_strategy => 'full', " + + "options => 'num-sorted-run.compaction-trigger=3')") + .close()) + .hasMessageContaining( + "The full compact strategy is only supported in batch mode. Please add -Dexecution.runtime-mode=BATCH."); + } + private void checkLatestSnapshot( FileStoreTable table, long snapshotId, Snapshot.CommitKind commitKind) { SnapshotManager snapshotManager = table.snapshotManager(); diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/procedure/ExpirePartitionsProcedureITCase.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/procedure/ExpirePartitionsProcedureITCase.java index bc2e84902f35..a40968e067bc 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/procedure/ExpirePartitionsProcedureITCase.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/procedure/ExpirePartitionsProcedureITCase.java @@ -402,6 +402,56 @@ public void testSortAndLimitExpirePartition() throws Exception { .containsExactlyInAnyOrder("4:2024-06-03:01:00", "Never-expire:9999-09-09:99:99"); } + @Test + public void testNullPartitionExpire() { + sql("CREATE TABLE T (k INT, ds STRING) PARTITIONED BY (ds);"); + sql("INSERT INTO T VALUES (1, CAST (NULL AS STRING))"); + assertThat( + callExpirePartitions( + "CALL sys.expire_partitions(" + + "`table` => 'default.T'" + + ", expiration_time => '1 d'" + + ", timestamp_formatter => 'yyyyMMdd')")) + .containsExactly("No expired partitions."); + } + + @Test + public void testExpirePartitionsWithDefaultNum() throws Exception { + sql( + "CREATE TABLE T (" + + " k STRING," + + " dt STRING," + + " PRIMARY KEY (k, dt) NOT ENFORCED" + + ") PARTITIONED BY (dt) WITH (" + + " 'bucket' = '1'," + + " 'partition.expiration-max-num'='2'" + + ")"); + FileStoreTable table = paimonTable("T"); + + sql("INSERT INTO T VALUES ('a', '2024-06-01')"); + sql("INSERT INTO T VALUES ('b', '2024-06-02')"); + sql("INSERT INTO T VALUES ('c', '2024-06-03')"); + // This partition never expires. + sql("INSERT INTO T VALUES ('Never-expire', '9999-09-09')"); + Function consumerReadResult = + (InternalRow row) -> row.getString(0) + ":" + row.getString(1); + + assertThat(read(table, consumerReadResult)) + .containsExactlyInAnyOrder( + "a:2024-06-01", "b:2024-06-02", "c:2024-06-03", "Never-expire:9999-09-09"); + + assertThat( + callExpirePartitions( + "CALL sys.expire_partitions(" + + "`table` => 'default.T'" + + ", expiration_time => '1 d'" + + ", timestamp_formatter => 'yyyy-MM-dd')")) + .containsExactlyInAnyOrder("dt=2024-06-01", "dt=2024-06-02"); + + assertThat(read(table, consumerReadResult)) + .containsExactlyInAnyOrder("c:2024-06-03", "Never-expire:9999-09-09"); + } + /** Return a list of expired partitions. */ public List callExpirePartitions(String callSql) { return sql(callSql).stream() diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/procedure/ExpireTagsProcedureITCase.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/procedure/ExpireTagsProcedureITCase.java new file mode 100644 index 000000000000..4a89531b22a0 --- /dev/null +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/procedure/ExpireTagsProcedureITCase.java @@ -0,0 +1,141 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.flink.procedure; + +import org.apache.paimon.data.Timestamp; +import org.apache.paimon.flink.CatalogITCaseBase; +import org.apache.paimon.table.FileStoreTable; +import org.apache.paimon.utils.SnapshotManager; + +import org.apache.flink.types.Row; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.time.LocalDateTime; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; + +/** IT Case for {@link ExpireTagsProcedure}. */ +public class ExpireTagsProcedureITCase extends CatalogITCaseBase { + + @Test + public void testExpireTagsByTagCreateTimeAndTagTimeRetained() throws Exception { + sql( + "CREATE TABLE T (id STRING, name STRING," + + " PRIMARY KEY (id) NOT ENFORCED)" + + " WITH ('bucket'='1', 'write-only'='true')"); + + FileStoreTable table = paimonTable("T"); + SnapshotManager snapshotManager = table.snapshotManager(); + + // generate 5 snapshots + for (int i = 1; i <= 5; i++) { + sql("INSERT INTO T VALUES ('" + i + "', '" + i + "')"); + } + checkSnapshots(snapshotManager, 1, 5); + + sql("CALL sys.create_tag(`table` => 'default.T', tag => 'tag-1', snapshot_id => 1)"); + sql( + "CALL sys.create_tag(`table` => 'default.T', tag => 'tag-2', snapshot_id => 2, time_retained => '1h')"); + + // no tags expired + assertThat(sql("CALL sys.expire_tags(`table` => 'default.T')")) + .containsExactly(Row.of("No expired tags.")); + + sql( + "CALL sys.create_tag(`table` => 'default.T', tag => 'tag-3', snapshot_id => 3, time_retained => '1s')"); + sql( + "CALL sys.create_tag(`table` => 'default.T', tag => 'tag-4', snapshot_id => 4, time_retained => '1s')"); + + Thread.sleep(2000); + // tag-3,tag-4 expired + assertThat(sql("CALL sys.expire_tags(`table` => 'default.T')")) + .containsExactlyInAnyOrder(Row.of("tag-3"), Row.of("tag-4")); + } + + @Test + public void testExpireTagsByOlderThanTime() throws Exception { + sql( + "CREATE TABLE T (id STRING, name STRING," + + " PRIMARY KEY (id) NOT ENFORCED)" + + " WITH ('bucket'='1', 'write-only'='true')"); + + FileStoreTable table = paimonTable("T"); + SnapshotManager snapshotManager = table.snapshotManager(); + + // generate 5 snapshots + for (int i = 1; i <= 5; i++) { + sql("INSERT INTO T VALUES ('" + i + "', '" + i + "')"); + } + checkSnapshots(snapshotManager, 1, 5); + + sql("CALL sys.create_tag(`table` => 'default.T', tag => 'tag-1', snapshot_id => 1)"); + sql( + "CALL sys.create_tag(`table` => 'default.T', tag => 'tag-2', snapshot_id => 2, time_retained => '1d')"); + sql( + "CALL sys.create_tag(`table` => 'default.T', tag => 'tag-3', snapshot_id => 3, time_retained => '1d')"); + sql( + "CALL sys.create_tag(`table` => 'default.T', tag => 'tag-4', snapshot_id => 4, time_retained => '1d')"); + List sql = sql("select count(tag_name) from `T$tags`"); + assertThat(sql("select count(tag_name) from `T$tags`")).containsExactly(Row.of(4L)); + + // no tags expired + assertThat(sql("CALL sys.expire_tags(`table` => 'default.T')")) + .containsExactlyInAnyOrder(Row.of("No expired tags.")); + + // tag-2 as the base older_than time. + // tag-1 expired by its file creation time. + LocalDateTime olderThanTime1 = table.tagManager().tag("tag-2").getTagCreateTime(); + java.sql.Timestamp timestamp1 = + new java.sql.Timestamp( + Timestamp.fromLocalDateTime(olderThanTime1).getMillisecond()); + assertThat( + sql( + "CALL sys.expire_tags(`table` => 'default.T', older_than => '" + + timestamp1.toString() + + "')")) + .containsExactlyInAnyOrder(Row.of("tag-1")); + + sql( + "CALL sys.create_tag(`table` => 'default.T', tag => 'tag-5', snapshot_id => 5, time_retained => '1s')"); + Thread.sleep(1000); + + // tag-4 as the base older_than time. + // tag-2,tag-3,tag-5 expired, tag-5 reached its tagTimeRetained. + LocalDateTime olderThanTime2 = table.tagManager().tag("tag-4").getTagCreateTime(); + java.sql.Timestamp timestamp2 = + new java.sql.Timestamp( + Timestamp.fromLocalDateTime(olderThanTime2).getMillisecond()); + assertThat( + sql( + "CALL sys.expire_tags(`table` => 'default.T', older_than => '" + + timestamp2.toString() + + "')")) + .containsExactlyInAnyOrder(Row.of("tag-2"), Row.of("tag-3"), Row.of("tag-5")); + + assertThat(sql("select tag_name from `T$tags`")).containsExactly(Row.of("tag-4")); + } + + private void checkSnapshots(SnapshotManager sm, int earliest, int latest) throws IOException { + assertThat(sm.snapshotCount()).isEqualTo(latest - earliest + 1); + assertThat(sm.earliestSnapshotId()).isEqualTo(earliest); + assertThat(sm.latestSnapshotId()).isEqualTo(latest); + } +} diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/procedure/ProcedureTest.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/procedure/ProcedureTest.java index c24d4105a557..74e3aeeac53b 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/procedure/ProcedureTest.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/procedure/ProcedureTest.java @@ -98,7 +98,8 @@ public void testProcedureHasNamedArgument() { } private Method getMethodFromName(Class clazz, String methodName) { - Method[] methods = clazz.getDeclaredMethods(); + // get all methods of current and parent class + Method[] methods = clazz.getMethods(); for (Method method : methods) { if (method.getName().equals(methodName)) { diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/procedure/PurgeFilesProcedureITCase.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/procedure/PurgeFilesProcedureITCase.java new file mode 100644 index 000000000000..9eb9aad2969e --- /dev/null +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/procedure/PurgeFilesProcedureITCase.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.flink.procedure; + +import org.apache.paimon.flink.CatalogITCaseBase; + +import org.apache.flink.types.Row; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +/** IT Case for {@link PurgeFilesProcedure}. */ +public class PurgeFilesProcedureITCase extends CatalogITCaseBase { + + @Test + public void testPurgeFiles() throws Exception { + sql( + "CREATE TABLE T (id INT, name STRING," + + " PRIMARY KEY (id) NOT ENFORCED)" + + " WITH ('bucket'='1')"); + + sql("INSERT INTO T VALUES (1, 'a')"); + assertThat(sql("select * from `T`")).containsExactly(Row.of(1, "a")); + + sql("INSERT INTO T VALUES (1, 'a')"); + sql("CALL sys.purge_files(`table` => 'default.T')"); + assertThat(sql("select * from `T`")).containsExactly(); + + sql("INSERT INTO T VALUES (2, 'a')"); + assertThat(sql("select * from `T`")).containsExactly(Row.of(2, "a")); + } +} diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/procedure/ReplaceTagProcedureITCase.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/procedure/ReplaceTagProcedureITCase.java new file mode 100644 index 000000000000..8a4eb791a6ad --- /dev/null +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/procedure/ReplaceTagProcedureITCase.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.flink.procedure; + +import org.apache.paimon.flink.CatalogITCaseBase; + +import org.apache.flink.types.Row; +import org.assertj.core.api.Assertions; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +/** IT Case for {@link ReplaceTagProcedure}. */ +public class ReplaceTagProcedureITCase extends CatalogITCaseBase { + + @Test + public void testExpireTagsByTagCreateTimeAndTagTimeRetained() throws Exception { + sql( + "CREATE TABLE T (id INT, name STRING," + + " PRIMARY KEY (id) NOT ENFORCED)" + + " WITH ('bucket'='1')"); + + sql("INSERT INTO T VALUES (1, 'a')"); + sql("INSERT INTO T VALUES (2, 'b')"); + assertThat(paimonTable("T").snapshotManager().snapshotCount()).isEqualTo(2); + + Assertions.assertThatThrownBy( + () -> + sql( + "CALL sys.replace_tag(`table` => 'default.T', tag => 'test_tag')")) + .hasMessageContaining("Tag name 'test_tag' does not exist."); + + sql("CALL sys.create_tag(`table` => 'default.T', tag => 'test_tag')"); + assertThat(sql("select tag_name,snapshot_id,time_retained from `T$tags`")) + .containsExactly(Row.of("test_tag", 2L, null)); + + // replace tag with new time_retained + sql( + "CALL sys.replace_tag(`table` => 'default.T', tag => 'test_tag'," + + " time_retained => '1 d')"); + assertThat(sql("select tag_name,snapshot_id,time_retained from `T$tags`")) + .containsExactly(Row.of("test_tag", 2L, "PT24H")); + + // replace tag with new snapshot and time_retained + sql( + "CALL sys.replace_tag(`table` => 'default.T', tag => 'test_tag'," + + " snapshot => 1, time_retained => '2 d')"); + assertThat(sql("select tag_name,snapshot_id,time_retained from `T$tags`")) + .containsExactly(Row.of("test_tag", 2L, "PT48H")); + } +} diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/procedure/RollbackToWatermarkProcedureITCase.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/procedure/RollbackToWatermarkProcedureITCase.java new file mode 100644 index 000000000000..f87ecd24756b --- /dev/null +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/procedure/RollbackToWatermarkProcedureITCase.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.flink.procedure; + +import org.apache.paimon.flink.CatalogITCaseBase; +import org.apache.paimon.table.FileStoreTable; + +import org.apache.flink.types.Row; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +/** IT Case for {@link RollbackToWatermarkProcedure}. */ +public class RollbackToWatermarkProcedureITCase extends CatalogITCaseBase { + + @Test + public void testCreateTagsFromSnapshotsWatermark() throws Exception { + sql( + "CREATE TABLE T (" + + " k STRING," + + " dt STRING," + + " PRIMARY KEY (k, dt) NOT ENFORCED" + + ") PARTITIONED BY (dt) WITH (" + + " 'bucket' = '1'" + + ")"); + + // create snapshot 1 with watermark 1000. + sql( + "insert into T/*+ OPTIONS('end-input.watermark'= '1000') */ values('k1', '2024-12-02')"); + // create snapshot 2 with watermark 2000. + sql( + "insert into T/*+ OPTIONS('end-input.watermark'= '2000') */ values('k2', '2024-12-02')"); + // create snapshot 3 with watermark 3000. + sql( + "insert into T/*+ OPTIONS('end-input.watermark'= '3000') */ values('k3', '2024-12-02')"); + + FileStoreTable table = paimonTable("T"); + + long watermark1 = table.snapshotManager().snapshot(1).watermark(); + long watermark2 = table.snapshotManager().snapshot(2).watermark(); + long watermark3 = table.snapshotManager().snapshot(3).watermark(); + + assertThat(watermark1 == 1000).isTrue(); + assertThat(watermark2 == 2000).isTrue(); + assertThat(watermark3 == 3000).isTrue(); + + assertThat(sql("select * from T").stream().map(Row::toString)) + .containsExactlyInAnyOrder( + "+I[k1, 2024-12-02]", "+I[k2, 2024-12-02]", "+I[k3, 2024-12-02]"); + + sql("CALL sys.rollback_to_watermark(`table` => 'default.T',`watermark` => 2001)"); + + // check for snapshot 2 + assertThat(sql("select * from T").stream().map(Row::toString)) + .containsExactlyInAnyOrder("+I[k1, 2024-12-02]", "+I[k2, 2024-12-02]"); + + sql("CALL sys.rollback_to_watermark(`table` => 'default.T',`watermark` => 1001)"); + + // check for snapshot 1 + assertThat(sql("select * from T").stream().map(Row::toString)) + .containsExactlyInAnyOrder("+I[k1, 2024-12-02]"); + } +} diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/AppendOnlyMultiTableCompactionWorkerOperatorTest.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/AppendOnlyMultiTableCompactionWorkerOperatorTest.java index d589459d9b96..949c2c7a66a3 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/AppendOnlyMultiTableCompactionWorkerOperatorTest.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/AppendOnlyMultiTableCompactionWorkerOperatorTest.java @@ -25,7 +25,13 @@ import org.apache.paimon.table.sink.CommitMessage; import org.apache.paimon.table.sink.CommitMessageImpl; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.runtime.operators.testutils.DummyEnvironment; +import org.apache.flink.streaming.api.operators.StreamOperatorParameters; import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; +import org.apache.flink.streaming.runtime.tasks.SourceOperatorStreamTask; +import org.apache.flink.streaming.util.MockOutput; +import org.apache.flink.streaming.util.MockStreamConfig; import org.assertj.core.api.Assertions; import org.junit.jupiter.api.Test; @@ -45,8 +51,17 @@ public class AppendOnlyMultiTableCompactionWorkerOperatorTest extends TableTestB public void testAsyncCompactionWorks() throws Exception { AppendOnlyMultiTableCompactionWorkerOperator workerOperator = - new AppendOnlyMultiTableCompactionWorkerOperator( - () -> catalog, "user", new Options()); + new AppendOnlyMultiTableCompactionWorkerOperator.Factory( + () -> catalog, "user", new Options()) + .createStreamOperator( + new StreamOperatorParameters<>( + new SourceOperatorStreamTask( + new DummyEnvironment()), + new MockStreamConfig(new Configuration(), 1), + new MockOutput<>(new ArrayList<>()), + null, + null, + null)); List> records = new ArrayList<>(); // create table and write diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/AppendOnlySingleTableCompactionWorkerOperatorTest.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/AppendOnlySingleTableCompactionWorkerOperatorTest.java index d04032817cf0..6238a9cbf3ea 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/AppendOnlySingleTableCompactionWorkerOperatorTest.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/AppendOnlySingleTableCompactionWorkerOperatorTest.java @@ -32,7 +32,13 @@ import org.apache.paimon.table.sink.CommitMessageImpl; import org.apache.paimon.types.DataTypes; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.runtime.operators.testutils.DummyEnvironment; +import org.apache.flink.streaming.api.operators.StreamOperatorParameters; import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; +import org.apache.flink.streaming.runtime.tasks.SourceOperatorStreamTask; +import org.apache.flink.streaming.util.MockOutput; +import org.apache.flink.streaming.util.MockStreamConfig; import org.assertj.core.api.Assertions; import org.junit.jupiter.api.Test; @@ -49,7 +55,16 @@ public class AppendOnlySingleTableCompactionWorkerOperatorTest extends TableTest public void testAsyncCompactionWorks() throws Exception { createTableDefault(); AppendOnlySingleTableCompactionWorkerOperator workerOperator = - new AppendOnlySingleTableCompactionWorkerOperator(getTableDefault(), "user"); + new AppendOnlySingleTableCompactionWorkerOperator.Factory(getTableDefault(), "user") + .createStreamOperator( + new StreamOperatorParameters<>( + new SourceOperatorStreamTask( + new DummyEnvironment()), + new MockStreamConfig(new Configuration(), 1), + new MockOutput<>(new ArrayList<>()), + null, + null, + null)); // write 200 files List commitMessages = writeDataDefault(200, 20); @@ -102,7 +117,16 @@ public void testAsyncCompactionWorks() throws Exception { public void testAsyncCompactionFileDeletedWhenShutdown() throws Exception { createTableDefault(); AppendOnlySingleTableCompactionWorkerOperator workerOperator = - new AppendOnlySingleTableCompactionWorkerOperator(getTableDefault(), "user"); + new AppendOnlySingleTableCompactionWorkerOperator.Factory(getTableDefault(), "user") + .createStreamOperator( + new StreamOperatorParameters<>( + new SourceOperatorStreamTask( + new DummyEnvironment()), + new MockStreamConfig(new Configuration(), 1), + new MockOutput<>(new ArrayList<>()), + null, + null, + null)); // write 200 files List commitMessages = writeDataDefault(200, 40); diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/AutoTagForSavepointCommitterOperatorTest.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/AutoTagForSavepointCommitterOperatorTest.java index 3b58c24d16b1..ee930a06fc3d 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/AutoTagForSavepointCommitterOperatorTest.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/AutoTagForSavepointCommitterOperatorTest.java @@ -32,7 +32,7 @@ import org.apache.flink.runtime.checkpoint.OperatorSubtaskState; import org.apache.flink.runtime.checkpoint.SavepointType; import org.apache.flink.runtime.state.StateInitializationContext; -import org.apache.flink.streaming.api.operators.OneInputStreamOperator; +import org.apache.flink.streaming.api.operators.OneInputStreamOperatorFactory; import org.apache.flink.streaming.util.OneInputStreamOperatorTestHarness; import org.junit.jupiter.api.Test; @@ -198,13 +198,15 @@ private void processCommittable( } @Override - protected OneInputStreamOperator createCommitterOperator( - FileStoreTable table, - String commitUser, - CommittableStateManager committableStateManager) { - return new AutoTagForSavepointCommitterOperator<>( - (CommitterOperator) - super.createCommitterOperator(table, commitUser, committableStateManager), + protected OneInputStreamOperatorFactory + createCommitterOperatorFactory( + FileStoreTable table, + String commitUser, + CommittableStateManager committableStateManager) { + return new AutoTagForSavepointCommitterOperatorFactory<>( + (CommitterOperatorFactory) + super.createCommitterOperatorFactory( + table, commitUser, committableStateManager), table::snapshotManager, table::tagManager, () -> table.store().newTagDeletion(), @@ -213,14 +215,15 @@ protected OneInputStreamOperator createCommitterOperat } @Override - protected OneInputStreamOperator createCommitterOperator( - FileStoreTable table, - String commitUser, - CommittableStateManager committableStateManager, - ThrowingConsumer initializeFunction) { - return new AutoTagForSavepointCommitterOperator<>( - (CommitterOperator) - super.createCommitterOperator( + protected OneInputStreamOperatorFactory + createCommitterOperatorFactory( + FileStoreTable table, + String commitUser, + CommittableStateManager committableStateManager, + ThrowingConsumer initializeFunction) { + return new AutoTagForSavepointCommitterOperatorFactory<>( + (CommitterOperatorFactory) + super.createCommitterOperatorFactory( table, commitUser, committableStateManager, initializeFunction), table::snapshotManager, table::tagManager, diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/BatchWriteGeneratorTagOperatorTest.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/BatchWriteGeneratorTagOperatorTest.java index 147110637aef..68162832eac9 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/BatchWriteGeneratorTagOperatorTest.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/BatchWriteGeneratorTagOperatorTest.java @@ -27,13 +27,21 @@ import org.apache.paimon.utils.SnapshotManager; import org.apache.paimon.utils.TagManager; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.runtime.operators.testutils.DummyEnvironment; import org.apache.flink.streaming.api.operators.OneInputStreamOperator; +import org.apache.flink.streaming.api.operators.OneInputStreamOperatorFactory; +import org.apache.flink.streaming.api.operators.StreamOperatorParameters; +import org.apache.flink.streaming.runtime.tasks.SourceOperatorStreamTask; +import org.apache.flink.streaming.util.MockOutput; +import org.apache.flink.streaming.util.MockStreamConfig; import org.junit.jupiter.api.Test; import java.time.Instant; import java.time.LocalDateTime; import java.time.ZoneId; import java.time.format.DateTimeFormatter; +import java.util.ArrayList; import java.util.HashMap; import java.util.Objects; @@ -54,12 +62,23 @@ public void testBatchWriteGeneratorTag() throws Exception { StreamTableWrite write = table.newStreamWriteBuilder().withCommitUser(initialCommitUser).newWrite(); - OneInputStreamOperator committerOperator = - createCommitterOperator( + OneInputStreamOperatorFactory committerOperatorFactory = + createCommitterOperatorFactory( table, initialCommitUser, new RestoreAndFailCommittableStateManager<>( ManifestCommittableSerializer::new)); + + OneInputStreamOperator committerOperator = + committerOperatorFactory.createStreamOperator( + new StreamOperatorParameters<>( + new SourceOperatorStreamTask(new DummyEnvironment()), + new MockStreamConfig(new Configuration(), 1), + new MockOutput<>(new ArrayList<>()), + null, + null, + null)); + committerOperator.open(); TableCommitImpl tableCommit = table.newCommit(initialCommitUser); @@ -106,13 +125,15 @@ public void testBatchWriteGeneratorTag() throws Exception { } @Override - protected OneInputStreamOperator createCommitterOperator( - FileStoreTable table, - String commitUser, - CommittableStateManager committableStateManager) { - return new BatchWriteGeneratorTagOperator<>( - (CommitterOperator) - super.createCommitterOperator(table, commitUser, committableStateManager), + protected OneInputStreamOperatorFactory + createCommitterOperatorFactory( + FileStoreTable table, + String commitUser, + CommittableStateManager committableStateManager) { + return new BatchWriteGeneratorTagOperatorFactory<>( + (CommitterOperatorFactory) + super.createCommitterOperatorFactory( + table, commitUser, committableStateManager), table); } } diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/CommitterOperatorTest.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/CommitterOperatorTest.java index b2764fc37c6e..28c93ca79be0 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/CommitterOperatorTest.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/CommitterOperatorTest.java @@ -22,6 +22,7 @@ import org.apache.paimon.Snapshot; import org.apache.paimon.data.BinaryRow; import org.apache.paimon.data.GenericRow; +import org.apache.paimon.flink.FlinkConnectorOptions; import org.apache.paimon.flink.utils.TestingMetricUtils; import org.apache.paimon.fs.local.LocalFileIO; import org.apache.paimon.io.CompactIncrement; @@ -37,6 +38,7 @@ import org.apache.paimon.utils.SnapshotManager; import org.apache.paimon.utils.ThrowingConsumer; +import org.apache.paimon.shade.guava30.com.google.common.collect.ImmutableMap; import org.apache.paimon.shade.guava30.com.google.common.collect.Lists; import org.apache.flink.api.common.ExecutionConfig; @@ -49,10 +51,13 @@ import org.apache.flink.runtime.checkpoint.OperatorSubtaskState; import org.apache.flink.runtime.jobgraph.OperatorID; import org.apache.flink.runtime.state.StateInitializationContext; -import org.apache.flink.streaming.api.operators.OneInputStreamOperator; +import org.apache.flink.streaming.api.operators.OneInputStreamOperatorFactory; +import org.apache.flink.streaming.api.operators.StreamOperator; +import org.apache.flink.streaming.api.operators.StreamOperatorParameters; import org.apache.flink.streaming.api.watermark.Watermark; import org.apache.flink.streaming.util.AbstractStreamOperatorTestHarness; import org.apache.flink.streaming.util.OneInputStreamOperatorTestHarness; +import org.apache.flink.util.Preconditions; import org.assertj.core.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -257,8 +262,8 @@ public void testRestoreCommitUser() throws Exception { // 3. Check whether success List actual = new ArrayList<>(); - OneInputStreamOperator operator = - createCommitterOperator( + OneInputStreamOperatorFactory operatorFactory = + createCommitterOperatorFactory( table, initialCommitUser, new NoopCommittableStateManager(), @@ -272,7 +277,7 @@ public void testRestoreCommitUser() throws Exception { }); OneInputStreamOperatorTestHarness testHarness1 = - createTestHarness(operator); + createTestHarness(operatorFactory); testHarness1.initializeState(snapshot); testHarness1.close(); @@ -281,14 +286,43 @@ public void testRestoreCommitUser() throws Exception { Assertions.assertThat(actual).hasSameElementsAs(Lists.newArrayList(commitUser)); } + @Test + public void testRestoreEmptyMarkDoneState() throws Exception { + FileStoreTable table = createFileStoreTable(o -> {}, Collections.singletonList("b")); + + String commitUser = UUID.randomUUID().toString(); + + // 1. Generate operatorSubtaskState + OperatorSubtaskState snapshot; + { + OneInputStreamOperatorTestHarness testHarness = + createLossyTestHarness(table, commitUser); + + testHarness.open(); + snapshot = writeAndSnapshot(table, commitUser, 1, 1, testHarness); + testHarness.close(); + } + // 2. enable mark done. + table = + table.copy( + ImmutableMap.of( + FlinkConnectorOptions.PARTITION_IDLE_TIME_TO_DONE.key(), "1h")); + + // 3. restore from state. + OneInputStreamOperatorTestHarness testHarness = + createLossyTestHarness(table); + testHarness.initializeState(snapshot); + } + @Test public void testCommitInputEnd() throws Exception { FileStoreTable table = createFileStoreTable(); String commitUser = UUID.randomUUID().toString(); - OneInputStreamOperator operator = - createCommitterOperator(table, commitUser, new NoopCommittableStateManager()); + OneInputStreamOperatorFactory operatorFactory = + createCommitterOperatorFactory( + table, commitUser, new NoopCommittableStateManager()); OneInputStreamOperatorTestHarness testHarness = - createTestHarness(operator); + createTestHarness(operatorFactory); testHarness.open(); Assertions.assertThatCode( () -> { @@ -348,10 +382,10 @@ public void testCommitInputEnd() throws Exception { }) .doesNotThrowAnyException(); - if (operator instanceof CommitterOperator) { + if (operatorFactory instanceof CommitterOperator) { Assertions.assertThat( ((ManifestCommittable) - ((CommitterOperator) operator) + ((CommitterOperator) operatorFactory) .committablesPerCheckpoint.get(Long.MAX_VALUE)) .fileCommittables() .size()) @@ -565,7 +599,7 @@ public void testCalcDataBytesSend() throws Exception { table, commit, Committer.createContext("", metricGroup, true, false, null)); committer.commit(Collections.singletonList(manifestCommittable)); CommitterMetrics metrics = committer.getCommitterMetrics(); - assertThat(metrics.getNumBytesOutCounter().getCount()).isEqualTo(529); + assertThat(metrics.getNumBytesOutCounter().getCount()).isEqualTo(533); assertThat(metrics.getNumRecordsOutCounter().getCount()).isEqualTo(2); committer.close(); } @@ -574,14 +608,14 @@ public void testCalcDataBytesSend() throws Exception { public void testCommitMetrics() throws Exception { FileStoreTable table = createFileStoreTable(); - OneInputStreamOperator operator = - createCommitterOperator( + OneInputStreamOperatorFactory operatorFactory = + createCommitterOperatorFactory( table, null, new RestoreAndFailCommittableStateManager<>( ManifestCommittableSerializer::new)); OneInputStreamOperatorTestHarness testHarness = - createTestHarness(operator); + createTestHarness(operatorFactory); testHarness.open(); long timestamp = 0; StreamTableWrite write = @@ -597,7 +631,9 @@ public void testCommitMetrics() throws Exception { testHarness.notifyOfCompletedCheckpoint(cpId); MetricGroup commitMetricGroup = - operator.getMetricGroup() + testHarness + .getOneInputOperator() + .getMetricGroup() .addGroup("paimon") .addGroup("table", table.name()) .addGroup("commit"); @@ -655,10 +691,11 @@ public void testCommitMetrics() throws Exception { public void testParallelism() throws Exception { FileStoreTable table = createFileStoreTable(); String commitUser = UUID.randomUUID().toString(); - OneInputStreamOperator operator = - createCommitterOperator(table, commitUser, new NoopCommittableStateManager()); + OneInputStreamOperatorFactory operatorFactory = + createCommitterOperatorFactory( + table, commitUser, new NoopCommittableStateManager()); try (OneInputStreamOperatorTestHarness testHarness = - createTestHarness(operator, 10, 10, 3)) { + createTestHarness(operatorFactory, 10, 10, 3)) { Assertions.assertThatCode(testHarness::open) .hasMessage("Committer Operator parallelism in paimon MUST be one."); } @@ -670,13 +707,13 @@ public void testParallelism() throws Exception { protected OneInputStreamOperatorTestHarness createRecoverableTestHarness(FileStoreTable table) throws Exception { - OneInputStreamOperator operator = - createCommitterOperator( + OneInputStreamOperatorFactory operatorFactory = + createCommitterOperatorFactory( table, null, new RestoreAndFailCommittableStateManager<>( ManifestCommittableSerializer::new)); - return createTestHarness(operator); + return createTestHarness(operatorFactory); } private OneInputStreamOperatorTestHarness createLossyTestHarness( @@ -686,18 +723,20 @@ private OneInputStreamOperatorTestHarness createLossyT private OneInputStreamOperatorTestHarness createLossyTestHarness( FileStoreTable table, String commitUser) throws Exception { - OneInputStreamOperator operator = - createCommitterOperator(table, commitUser, new NoopCommittableStateManager()); - return createTestHarness(operator); + OneInputStreamOperatorFactory operatorFactory = + createCommitterOperatorFactory( + table, commitUser, new NoopCommittableStateManager()); + return createTestHarness(operatorFactory); } private OneInputStreamOperatorTestHarness createTestHarness( - OneInputStreamOperator operator) throws Exception { - return createTestHarness(operator, 1, 1, 0); + OneInputStreamOperatorFactory operatorFactory) + throws Exception { + return createTestHarness(operatorFactory, 1, 1, 0); } private OneInputStreamOperatorTestHarness createTestHarness( - OneInputStreamOperator operator, + OneInputStreamOperatorFactory operatorFactory, int maxParallelism, int parallelism, int subTaskIndex) @@ -706,22 +745,23 @@ private OneInputStreamOperatorTestHarness createTestHa new CommittableTypeInfo().createSerializer(new ExecutionConfig()); OneInputStreamOperatorTestHarness harness = new OneInputStreamOperatorTestHarness<>( - operator, + operatorFactory, maxParallelism, parallelism, subTaskIndex, - serializer, new OperatorID()); + harness.getStreamConfig().setupNetworkInputs(Preconditions.checkNotNull(serializer)); + harness.getStreamConfig().serializeAllConfigs(); harness.setup(serializer); return harness; } - protected OneInputStreamOperator createCommitterOperator( - FileStoreTable table, - String commitUser, - CommittableStateManager committableStateManager) { - return new CommitterOperator<>( - true, + protected OneInputStreamOperatorFactory + createCommitterOperatorFactory( + FileStoreTable table, + String commitUser, + CommittableStateManager committableStateManager) { + return new CommitterOperatorFactory<>( true, true, commitUser == null ? initialCommitUser : commitUser, @@ -735,13 +775,13 @@ protected OneInputStreamOperator createCommitterOperat committableStateManager); } - protected OneInputStreamOperator createCommitterOperator( - FileStoreTable table, - String commitUser, - CommittableStateManager committableStateManager, - ThrowingConsumer initializeFunction) { - return new CommitterOperator( - true, + protected OneInputStreamOperatorFactory + createCommitterOperatorFactory( + FileStoreTable table, + String commitUser, + CommittableStateManager committableStateManager, + ThrowingConsumer initializeFunction) { + return new CommitterOperatorFactory( true, true, commitUser == null ? initialCommitUser : commitUser, @@ -754,8 +794,24 @@ protected OneInputStreamOperator createCommitterOperat context), committableStateManager) { @Override - public void initializeState(StateInitializationContext context) throws Exception { - initializeFunction.accept(context); + @SuppressWarnings("unchecked") + public > T createStreamOperator( + StreamOperatorParameters parameters) { + return (T) + new CommitterOperator( + parameters, + streamingCheckpointEnabled, + forceSingleParallelism, + initialCommitUser, + committerFactory, + committableStateManager, + endInputWatermark) { + @Override + public void initializeState(StateInitializationContext context) + throws Exception { + initializeFunction.accept(context); + } + }; } }; } diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/CommitterOperatorTestBase.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/CommitterOperatorTestBase.java index 77b53ba7069d..a69f8dbd3a35 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/CommitterOperatorTestBase.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/CommitterOperatorTestBase.java @@ -88,10 +88,15 @@ protected void assertResults(FileStoreTable table, String... expected) { } protected FileStoreTable createFileStoreTable() throws Exception { - return createFileStoreTable(options -> {}); + return createFileStoreTable(options -> {}, Collections.emptyList()); } protected FileStoreTable createFileStoreTable(Consumer setOptions) throws Exception { + return createFileStoreTable(setOptions, Collections.emptyList()); + } + + protected FileStoreTable createFileStoreTable( + Consumer setOptions, List partitionKeys) throws Exception { Options conf = new Options(); conf.set(CoreOptions.PATH, tablePath.toString()); conf.setString("bucket", "1"); @@ -101,7 +106,7 @@ protected FileStoreTable createFileStoreTable(Consumer setOptions) thro schemaManager.createTable( new Schema( ROW_TYPE.getFields(), - Collections.emptyList(), + partitionKeys, Collections.emptyList(), conf.toMap(), "")); diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/CompactionTaskSimpleSerializerTest.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/CompactionTaskSimpleSerializerTest.java index 7cacb6c2931f..9f6e8159a1be 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/CompactionTaskSimpleSerializerTest.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/CompactionTaskSimpleSerializerTest.java @@ -79,6 +79,7 @@ private DataFileMeta newFile() { 0, 0L, null, - FileSource.APPEND); + FileSource.APPEND, + null); } } diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/CompactorSinkITCase.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/CompactorSinkITCase.java index a5f260fb25a5..d487d75925eb 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/CompactorSinkITCase.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/CompactorSinkITCase.java @@ -132,7 +132,7 @@ public void testCompact() throws Exception { .withContinuousMode(false) .withPartitionPredicate(predicate) .build(); - new CompactorSinkBuilder(table).withInput(source).build(); + new CompactorSinkBuilder(table, true).withInput(source).build(); env.execute(); snapshot = snapshotManager.snapshot(snapshotManager.latestSnapshotId()); @@ -181,7 +181,8 @@ public void testCompactParallelism() throws Exception { FlinkConnectorOptions.SINK_PARALLELISM.key(), String.valueOf(sinkParalellism)); } - })) + }), + false) .withInput(source) .build(); @@ -253,8 +254,8 @@ private OneInputStreamOperatorTestHarness createTestHarnes return harness; } - protected StoreCompactOperator createCompactOperator(FileStoreTable table) { - return new StoreCompactOperator( + protected StoreCompactOperator.Factory createCompactOperator(FileStoreTable table) { + return new StoreCompactOperator.Factory( table, (t, commitUser, state, ioManager, memoryPool, metricGroup) -> new StoreSinkWriteImpl( @@ -267,13 +268,20 @@ protected StoreCompactOperator createCompactOperator(FileStoreTable table) { false, memoryPool, metricGroup), - "test"); + "test", + true); } - protected MultiTablesStoreCompactOperator createMultiTablesCompactOperator( + protected MultiTablesStoreCompactOperator.Factory createMultiTablesCompactOperator( Catalog.Loader catalogLoader) throws Exception { - return new MultiTablesStoreCompactOperator( - catalogLoader, commitUser, new CheckpointConfig(), false, false, new Options()); + return new MultiTablesStoreCompactOperator.Factory( + catalogLoader, + commitUser, + new CheckpointConfig(), + false, + false, + true, + new Options()); } private static byte[] partition(String dt, int hh) { diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/FlinkSinkTest.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/FlinkSinkTest.java index c335568344b3..5f21858e61a5 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/FlinkSinkTest.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/FlinkSinkTest.java @@ -42,7 +42,7 @@ import org.apache.flink.streaming.api.datastream.DataStream; import org.apache.flink.streaming.api.datastream.DataStreamSource; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; -import org.apache.flink.streaming.api.operators.SimpleOperatorFactory; +import org.apache.flink.streaming.api.operators.OneInputStreamOperatorFactory; import org.apache.flink.streaming.api.transformations.OneInputTransformation; import org.apache.flink.streaming.util.OneInputStreamOperatorTestHarness; import org.junit.jupiter.api.Test; @@ -82,20 +82,22 @@ private boolean testSpillable( Collections.singletonList(GenericRow.of(1, 1))); FlinkSink flinkSink = new FixedBucketSink(fileStoreTable, null, null); DataStream written = flinkSink.doWrite(source, "123", 1); - RowDataStoreWriteOperator operator = - ((RowDataStoreWriteOperator) - ((SimpleOperatorFactory) - ((OneInputTransformation) written.getTransformation()) - .getOperatorFactory()) - .getOperator()); + OneInputStreamOperatorFactory operatorFactory = + (OneInputStreamOperatorFactory) + ((OneInputTransformation) + written.getTransformation()) + .getOperatorFactory(); TypeSerializer serializer = new CommittableTypeInfo().createSerializer(new ExecutionConfig()); OneInputStreamOperatorTestHarness harness = - new OneInputStreamOperatorTestHarness<>(operator); + new OneInputStreamOperatorTestHarness<>(operatorFactory); harness.setup(serializer); harness.initializeEmptyState(); + RowDataStoreWriteOperator operator = + (RowDataStoreWriteOperator) harness.getOneInputOperator(); + return ((KeyValueFileStoreWrite) ((StoreSinkWriteImpl) operator.write).write.getWrite()) .bufferSpillable(); } diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/LocalMergeOperatorTest.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/LocalMergeOperatorTest.java new file mode 100644 index 000000000000..fc45eceb3fd5 --- /dev/null +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/LocalMergeOperatorTest.java @@ -0,0 +1,228 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.flink.sink; + +import org.apache.paimon.data.GenericRow; +import org.apache.paimon.data.InternalRow; +import org.apache.paimon.mergetree.localmerge.HashMapLocalMerger; +import org.apache.paimon.schema.TableSchema; +import org.apache.paimon.types.DataTypes; +import org.apache.paimon.types.RowKind; +import org.apache.paimon.types.RowType; + +import org.apache.flink.configuration.Configuration; +import org.apache.flink.runtime.operators.testutils.DummyEnvironment; +import org.apache.flink.streaming.api.operators.Output; +import org.apache.flink.streaming.api.operators.StreamOperatorParameters; +import org.apache.flink.streaming.api.watermark.Watermark; +import org.apache.flink.streaming.runtime.streamrecord.LatencyMarker; +import org.apache.flink.streaming.runtime.streamrecord.RecordAttributes; +import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; +import org.apache.flink.streaming.runtime.tasks.SourceOperatorStreamTask; +import org.apache.flink.streaming.runtime.watermarkstatus.WatermarkStatus; +import org.apache.flink.streaming.util.MockOutput; +import org.apache.flink.streaming.util.MockStreamConfig; +import org.apache.flink.util.OutputTag; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.function.Consumer; + +import static org.apache.paimon.CoreOptions.LOCAL_MERGE_BUFFER_SIZE; +import static org.apache.paimon.CoreOptions.SEQUENCE_FIELD; +import static org.apache.paimon.data.BinaryString.fromString; +import static org.apache.paimon.types.RowKind.DELETE; +import static org.assertj.core.api.Assertions.assertThat; + +class LocalMergeOperatorTest { + + private LocalMergeOperator operator; + + @Test + public void testHashNormal() throws Exception { + prepareHashOperator(); + List result = new ArrayList<>(); + setOutput(result); + + // first test + processElement("a", 1); + processElement("b", 1); + processElement("a", 2); + processElement(DELETE, "b", 2); + operator.prepareSnapshotPreBarrier(0); + assertThat(result).containsExactlyInAnyOrder("+I:a->2", "-D:b->2"); + result.clear(); + + // second test + processElement("c", 1); + processElement("d", 1); + operator.prepareSnapshotPreBarrier(0); + assertThat(result).containsExactlyInAnyOrder("+I:c->1", "+I:d->1"); + result.clear(); + + // large records + Map expected = new HashMap<>(); + Random rnd = new Random(); + int records = 10_000; + for (int i = 0; i < records; i++) { + String key = rnd.nextInt(records) + ""; + expected.put(key, "+I:" + key + "->" + i); + processElement(key, i); + } + + operator.prepareSnapshotPreBarrier(0); + assertThat(result).containsExactlyInAnyOrderElementsOf(expected.values()); + result.clear(); + } + + @Test + public void testUserDefineSequence() throws Exception { + Map options = new HashMap<>(); + options.put(SEQUENCE_FIELD.key(), "f1"); + prepareHashOperator(options); + + List result = new ArrayList<>(); + setOutput(result); + + processElement("a", 2); + processElement("b", 1); + processElement("a", 1); + operator.prepareSnapshotPreBarrier(0); + assertThat(result).containsExactlyInAnyOrder("+I:a->2", "+I:b->1"); + result.clear(); + } + + @Test + public void testHashSpill() throws Exception { + Map options = new HashMap<>(); + options.put(LOCAL_MERGE_BUFFER_SIZE.key(), "2 m"); + prepareHashOperator(options); + List result = new ArrayList<>(); + setOutput(result); + + Map expected = new HashMap<>(); + for (int i = 0; i < 30_000; i++) { + String key = i + ""; + expected.put(key, "+I:" + key + "->" + i); + processElement(key, i); + } + + operator.prepareSnapshotPreBarrier(0); + assertThat(result).containsExactlyInAnyOrderElementsOf(expected.values()); + result.clear(); + } + + private void prepareHashOperator() throws Exception { + prepareHashOperator(new HashMap<>()); + } + + private void prepareHashOperator(Map options) throws Exception { + if (!options.containsKey(LOCAL_MERGE_BUFFER_SIZE.key())) { + options.put(LOCAL_MERGE_BUFFER_SIZE.key(), "10 m"); + } + RowType rowType = + RowType.of( + DataTypes.STRING(), + DataTypes.INT(), + DataTypes.INT(), + DataTypes.INT(), + DataTypes.INT()); + TableSchema schema = + new TableSchema( + 0L, + rowType.getFields(), + rowType.getFieldCount(), + Collections.emptyList(), + Collections.singletonList("f0"), + options, + null); + operator = + new LocalMergeOperator.Factory(schema) + .createStreamOperator( + new StreamOperatorParameters<>( + new SourceOperatorStreamTask( + new DummyEnvironment()), + new MockStreamConfig(new Configuration(), 1), + new MockOutput<>(new ArrayList<>()), + null, + null, + null)); + operator.open(); + assertThat(operator.merger()).isInstanceOf(HashMapLocalMerger.class); + } + + private void setOutput(List result) { + operator.setOutput( + new TestOutput( + row -> + result.add( + row.getRowKind().shortString() + + ":" + + row.getString(0) + + "->" + + row.getInt(1)))); + } + + private void processElement(String key, int value) throws Exception { + processElement(RowKind.INSERT, key, value); + } + + private void processElement(RowKind rowKind, String key, int value) throws Exception { + operator.processElement( + new StreamRecord<>( + GenericRow.ofKind(rowKind, fromString(key), value, value, value, value))); + } + + private static class TestOutput implements Output> { + + private final Consumer consumer; + + private TestOutput(Consumer consumer) { + this.consumer = consumer; + } + + @Override + public void emitWatermark(Watermark mark) {} + + @Override + public void emitWatermarkStatus(WatermarkStatus watermarkStatus) {} + + @Override + public void collect(OutputTag outputTag, StreamRecord record) {} + + @Override + public void emitLatencyMarker(LatencyMarker latencyMarker) {} + + @Override + public void emitRecordAttributes(RecordAttributes recordAttributes) {} + + @Override + public void collect(StreamRecord record) { + consumer.accept(record.getValue()); + } + + @Override + public void close() {} + } +} diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/SinkSavepointITCase.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/SinkSavepointITCase.java index 6b912d2e57fe..b1486deacb0c 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/SinkSavepointITCase.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/SinkSavepointITCase.java @@ -137,7 +137,7 @@ private JobClient runRecoverFromSavepointJob(String failingPath, String savepoin .parallelism(1) .allowRestart() .setConf(conf) - .setConf(StateBackendOptions.STATE_BACKEND, "filesystem") + .setConf(StateBackendOptions.STATE_BACKEND, "hashmap") .setConf( CheckpointingOptions.CHECKPOINTS_DIRECTORY, "file://" + path + "/checkpoint") diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/StoreCompactOperatorTest.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/StoreCompactOperatorTest.java index 3f2daedffd48..3740033e025e 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/StoreCompactOperatorTest.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/StoreCompactOperatorTest.java @@ -48,17 +48,18 @@ public void testCompactExactlyOnce(boolean streamingMode) throws Exception { CompactRememberStoreWrite compactRememberStoreWrite = new CompactRememberStoreWrite(streamingMode); - StoreCompactOperator operator = - new StoreCompactOperator( + StoreCompactOperator.Factory operatorFactory = + new StoreCompactOperator.Factory( getTableDefault(), (table, commitUser, state, ioManager, memoryPool, metricGroup) -> compactRememberStoreWrite, - "10086"); + "10086", + !streamingMode); TypeSerializer serializer = new CommittableTypeInfo().createSerializer(new ExecutionConfig()); OneInputStreamOperatorTestHarness harness = - new OneInputStreamOperatorTestHarness<>(operator); + new OneInputStreamOperatorTestHarness<>(operatorFactory); harness.setup(serializer); harness.initializeEmptyState(); harness.open(); @@ -69,7 +70,7 @@ public void testCompactExactlyOnce(boolean streamingMode) throws Exception { harness.processElement(new StreamRecord<>(data(1))); harness.processElement(new StreamRecord<>(data(2))); - operator.prepareCommit(true, 1); + ((StoreCompactOperator) harness.getOneInputOperator()).prepareCommit(true, 1); Assertions.assertThat(compactRememberStoreWrite.compactTime).isEqualTo(3); } diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/StoreMultiCommitterTest.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/StoreMultiCommitterTest.java index 10e432f3c8c2..752679fb5903 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/StoreMultiCommitterTest.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/StoreMultiCommitterTest.java @@ -645,11 +645,10 @@ public void testCommitMetrics() throws Exception { private OneInputStreamOperatorTestHarness createRecoverableTestHarness() throws Exception { - CommitterOperator operator = - new CommitterOperator<>( + CommitterOperatorFactory operator = + new CommitterOperatorFactory<>( true, false, - true, initialCommitUser, context -> new StoreMultiCommitter(catalogLoader, context), new RestoreAndFailCommittableStateManager<>( @@ -659,11 +658,10 @@ public void testCommitMetrics() throws Exception { private OneInputStreamOperatorTestHarness createLossyTestHarness() throws Exception { - CommitterOperator operator = - new CommitterOperator<>( + CommitterOperatorFactory operator = + new CommitterOperatorFactory<>( true, false, - true, initialCommitUser, context -> new StoreMultiCommitter(catalogLoader, context), new CommittableStateManager() { @@ -682,12 +680,13 @@ public void snapshotState( private OneInputStreamOperatorTestHarness createTestHarness( - CommitterOperator operator) + CommitterOperatorFactory + operatorFactory) throws Exception { TypeSerializer serializer = new MultiTableCommittableTypeInfo().createSerializer(new ExecutionConfig()); OneInputStreamOperatorTestHarness harness = - new OneInputStreamOperatorTestHarness<>(operator, serializer); + new OneInputStreamOperatorTestHarness<>(operatorFactory, serializer); harness.setup(serializer); return harness; } diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/WrappedManifestCommittableSerializerTest.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/WrappedManifestCommittableSerializerTest.java index 298f3155ba34..b0aa76f157ac 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/WrappedManifestCommittableSerializerTest.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/WrappedManifestCommittableSerializerTest.java @@ -98,7 +98,7 @@ public static void addFileCommittables( if (!committable.logOffsets().containsKey(bucket)) { int offset = ID.incrementAndGet(); - committable.addLogOffset(bucket, offset); + committable.addLogOffset(bucket, offset, false); assertThat(committable.logOffsets().get(bucket)).isEqualTo(offset); } } diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/WriterChainingStrategyTest.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/WriterChainingStrategyTest.java new file mode 100644 index 000000000000..24fb529b59ea --- /dev/null +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/WriterChainingStrategyTest.java @@ -0,0 +1,203 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.flink.sink; + +import org.apache.flink.api.dag.Transformation; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.runtime.jobgraph.JobVertex; +import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; +import org.apache.flink.table.api.CompiledPlan; +import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; +import org.apache.flink.table.api.internal.CompiledPlanUtils; +import org.apache.flink.util.TimeUtils; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Tests for {@link org.apache.flink.streaming.api.operators.ChainingStrategy} of writer operators. + */ +public class WriterChainingStrategyTest { + private static final String TABLE_NAME = "paimon_table"; + + @TempDir java.nio.file.Path tempDir; + + private StreamTableEnvironment tEnv; + + @BeforeEach + public void beforeEach() { + Configuration config = new Configuration(); + config.setString( + "execution.checkpointing.interval", + TimeUtils.formatWithHighestUnit(Duration.ofMillis(500))); + StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(config); + tEnv = StreamTableEnvironment.create(env); + + String catalog = "PAIMON"; + Map options = new HashMap<>(); + options.put("type", "paimon"); + options.put("warehouse", tempDir.toString()); + tEnv.executeSql( + String.format( + "CREATE CATALOG %s WITH ( %s )", + catalog, + options.entrySet().stream() + .map(e -> String.format("'%s'='%s'", e.getKey(), e.getValue())) + .collect(Collectors.joining(",")))); + tEnv.useCatalog(catalog); + } + + @Test + public void testAppendTable() throws Exception { + tEnv.executeSql( + String.format( + "CREATE TABLE %s (id INT, data STRING, dt STRING) " + + "WITH ('bucket' = '1', 'bucket-key'='id', 'write-only' = 'true')", + TABLE_NAME)) + .await(); + + verifyChaining(false, true); + } + + @Test + public void testAppendTableWithUnawareBucket() throws Exception { + tEnv.executeSql( + String.format( + "CREATE TABLE %s (id INT, data STRING, dt STRING) " + + "WITH ('bucket' = '-1', 'write-only' = 'true')", + TABLE_NAME)) + .await(); + + verifyChaining(true, true); + } + + @Test + public void testPrimaryKeyTable() throws Exception { + tEnv.executeSql( + String.format( + "CREATE TABLE %s (id INT, data STRING, dt STRING, PRIMARY KEY (id) NOT ENFORCED) " + + "WITH ('bucket' = '1', 'bucket-key'='id', 'write-only' = 'true')", + TABLE_NAME)) + .await(); + + verifyChaining(false, true); + } + + @Test + public void testPrimaryKeyTableWithDynamicBucket() throws Exception { + tEnv.executeSql( + String.format( + "CREATE TABLE %s (id INT, data STRING, dt STRING, PRIMARY KEY (id) NOT ENFORCED) " + + "WITH ('bucket' = '-1', 'write-only' = 'true')", + TABLE_NAME)) + .await(); + + verifyChaining(false, true); + } + + @Test + public void testPrimaryKeyTableWithMultipleWriter() throws Exception { + tEnv.executeSql( + String.format( + "CREATE TABLE %s (id INT, data STRING, dt STRING, PRIMARY KEY (id) NOT ENFORCED) " + + "WITH ('bucket' = '1', 'bucket-key'='id', 'write-only' = 'true', 'sink.parallelism' = '2')", + TABLE_NAME)) + .await(); + + verifyChaining(false, false); + } + + @Test + public void testPrimaryKeyTableWithCrossPartitionUpdate() throws Exception { + tEnv.executeSql( + String.format( + "CREATE TABLE %s (id INT, data STRING, dt STRING, PRIMARY KEY (id) NOT ENFORCED) " + + "PARTITIONED BY ( dt ) WITH ('bucket' = '-1', 'write-only' = 'true')", + TABLE_NAME)) + .await(); + + List vertices = verifyChaining(false, true); + JobVertex vertex = findVertex(vertices, "INDEX_BOOTSTRAP"); + assertThat(vertex.toString()).contains("Source"); + } + + @Test + public void testPrimaryKeyTableWithLocalMerge() throws Exception { + tEnv.executeSql( + String.format( + "CREATE TABLE %s (id INT, data STRING, dt STRING, PRIMARY KEY (id) NOT ENFORCED) " + + "WITH ('bucket' = '-1', 'write-only' = 'true', 'local-merge-buffer-size' = '1MB')", + TABLE_NAME)) + .await(); + + List vertices = verifyChaining(false, true); + JobVertex vertex = findVertex(vertices, "local merge"); + assertThat(vertex.toString()).contains("Source"); + } + + private List verifyChaining( + boolean isWriterChainedWithUpstream, boolean isWriterChainedWithDownStream) { + CompiledPlan plan = + tEnv.compilePlanSql( + String.format( + "INSERT INTO %s VALUES (1, 'AAA', ''), (2, 'BBB', '')", + TABLE_NAME)); + List> transformations = CompiledPlanUtils.toTransformations(tEnv, plan); + StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); + transformations.forEach(env::addOperator); + + List vertices = new ArrayList<>(); + env.getStreamGraph().getJobGraph().getVertices().forEach(vertices::add); + JobVertex vertex = findVertex(vertices, "Writer(write-only)"); + + if (isWriterChainedWithUpstream) { + assertThat(vertex.toString()).contains("Source"); + } else { + assertThat(vertex.toString()).doesNotContain("Source"); + } + + if (isWriterChainedWithDownStream) { + assertThat(vertex.toString()).contains("Committer"); + } else { + assertThat(vertex.toString()).doesNotContain("Committer"); + } + + return vertices; + } + + private JobVertex findVertex(List vertices, String key) { + for (JobVertex vertex : vertices) { + if (vertex.toString().contains(key)) { + return vertex; + } + } + throw new IllegalStateException( + String.format( + "Cannot find vertex with keyword %s among job vertices %s", key, vertices)); + } +} diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/WriterOperatorTest.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/WriterOperatorTest.java index 3a8c1557122f..83af15745078 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/WriterOperatorTest.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/WriterOperatorTest.java @@ -115,9 +115,10 @@ public void testAppendOnlyTableMetrics() throws Exception { private void testMetricsImpl(FileStoreTable fileStoreTable) throws Exception { String tableName = tablePath.getName(); - RowDataStoreWriteOperator operator = getStoreSinkWriteOperator(fileStoreTable); + RowDataStoreWriteOperator.Factory operatorFactory = + getStoreSinkWriteOperatorFactory(fileStoreTable); OneInputStreamOperatorTestHarness harness = - createHarness(operator); + createHarness(operatorFactory); TypeSerializer serializer = new CommittableTypeInfo().createSerializer(new ExecutionConfig()); @@ -133,7 +134,7 @@ private void testMetricsImpl(FileStoreTable fileStoreTable) throws Exception { harness.snapshot(1, 2); harness.notifyOfCompletedCheckpoint(1); - OperatorMetricGroup metricGroup = operator.getMetricGroup(); + OperatorMetricGroup metricGroup = harness.getOneInputOperator().getMetricGroup(); MetricGroup writerBufferMetricGroup = metricGroup .addGroup("paimon") @@ -173,9 +174,10 @@ public void testAsyncLookupWithFailure() throws Exception { rowType, Arrays.asList("pt", "k"), Collections.singletonList("k"), options); // we don't wait for compaction because this is async lookup test - RowDataStoreWriteOperator operator = getAsyncLookupWriteOperator(fileStoreTable, false); + RowDataStoreWriteOperator.Factory operatorFactory = + getAsyncLookupWriteOperatorFactory(fileStoreTable, false); OneInputStreamOperatorTestHarness harness = - createHarness(operator); + createHarness(operatorFactory); TableCommitImpl commit = fileStoreTable.newCommit(commitUser); @@ -205,8 +207,8 @@ public void testAsyncLookupWithFailure() throws Exception { harness.close(); // re-create operator from state, this time wait for compaction to check result - operator = getAsyncLookupWriteOperator(fileStoreTable, true); - harness = createHarness(operator); + operatorFactory = getAsyncLookupWriteOperatorFactory(fileStoreTable, true); + harness = createHarness(operatorFactory); harness.setup(serializer); harness.initializeState(state); harness.open(); @@ -263,9 +265,10 @@ private void testChangelog(boolean insertOnly) throws Exception { FileStoreTable fileStoreTable = createFileStoreTable( rowType, Arrays.asList("pt", "k"), Collections.singletonList("k"), options); - RowDataStoreWriteOperator operator = getStoreSinkWriteOperator(fileStoreTable); + RowDataStoreWriteOperator.Factory operatorFactory = + getStoreSinkWriteOperatorFactory(fileStoreTable); OneInputStreamOperatorTestHarness harness = - createHarness(operator); + createHarness(operatorFactory); TableCommitImpl commit = fileStoreTable.newCommit(commitUser); @@ -277,7 +280,7 @@ private void testChangelog(boolean insertOnly) throws Exception { if (insertOnly) { Field field = TableWriteOperator.class.getDeclaredField("write"); field.setAccessible(true); - StoreSinkWrite write = (StoreSinkWrite) field.get(operator); + StoreSinkWrite write = (StoreSinkWrite) field.get(harness.getOneInputOperator()); write.withInsertOnly(true); } @@ -339,17 +342,17 @@ public void testNumWritersMetric() throws Exception { options); TableCommitImpl commit = fileStoreTable.newCommit(commitUser); - RowDataStoreWriteOperator rowDataStoreWriteOperator = - getStoreSinkWriteOperator(fileStoreTable); + RowDataStoreWriteOperator.Factory operatorFactory = + getStoreSinkWriteOperatorFactory(fileStoreTable); OneInputStreamOperatorTestHarness harness = - createHarness(rowDataStoreWriteOperator); + createHarness(operatorFactory); TypeSerializer serializer = new CommittableTypeInfo().createSerializer(new ExecutionConfig()); harness.setup(serializer); harness.open(); - OperatorMetricGroup metricGroup = rowDataStoreWriteOperator.getMetricGroup(); + OperatorMetricGroup metricGroup = harness.getOneInputOperator().getMetricGroup(); MetricGroup writerBufferMetricGroup = metricGroup .addGroup("paimon") @@ -408,8 +411,9 @@ public void testNumWritersMetric() throws Exception { // Test utils // ------------------------------------------------------------------------ - private RowDataStoreWriteOperator getStoreSinkWriteOperator(FileStoreTable fileStoreTable) { - return new RowDataStoreWriteOperator( + private RowDataStoreWriteOperator.Factory getStoreSinkWriteOperatorFactory( + FileStoreTable fileStoreTable) { + return new RowDataStoreWriteOperator.Factory( fileStoreTable, null, (table, commitUser, state, ioManager, memoryPool, metricGroup) -> @@ -426,9 +430,9 @@ private RowDataStoreWriteOperator getStoreSinkWriteOperator(FileStoreTable fileS commitUser); } - private RowDataStoreWriteOperator getAsyncLookupWriteOperator( + private RowDataStoreWriteOperator.Factory getAsyncLookupWriteOperatorFactory( FileStoreTable fileStoreTable, boolean waitCompaction) { - return new RowDataStoreWriteOperator( + return new RowDataStoreWriteOperator.Factory( fileStoreTable, null, (table, commitUser, state, ioManager, memoryPool, metricGroup) -> @@ -471,10 +475,11 @@ private FileStoreTable createFileStoreTable( } private OneInputStreamOperatorTestHarness createHarness( - RowDataStoreWriteOperator operator) throws Exception { + RowDataStoreWriteOperator.Factory operatorFactory) throws Exception { InternalTypeInfo internalRowInternalTypeInfo = new InternalTypeInfo<>(new InternalRowTypeSerializer(RowType.builder().build())); return new OneInputStreamOperatorTestHarness<>( - operator, internalRowInternalTypeInfo.createSerializer(new ExecutionConfig())); + operatorFactory, + internalRowInternalTypeInfo.createSerializer(new ExecutionConfig())); } } diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/partition/AddDonePartitionActionTest.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/partition/AddDonePartitionActionTest.java index 5338b3886001..fca5dcf0ed69 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/partition/AddDonePartitionActionTest.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/partition/AddDonePartitionActionTest.java @@ -26,6 +26,7 @@ import java.util.HashSet; import java.util.LinkedHashMap; +import java.util.Map; import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; @@ -61,6 +62,16 @@ public void markDone(LinkedHashMap partitionSpec) throw new UnsupportedOperationException(); } + @Override + public void alterPartition( + LinkedHashMap partitionSpec, + Map parameters, + long modifyTime, + boolean ignoreIfNotExist) + throws Exception { + throw new UnsupportedOperationException(); + } + @Override public void close() throws Exception { closed.set(true); diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/partition/HmsReporterTest.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/partition/HmsReporterTest.java new file mode 100644 index 000000000000..f245940da57d --- /dev/null +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/partition/HmsReporterTest.java @@ -0,0 +1,152 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.flink.sink.partition; + +import org.apache.paimon.data.BinaryRow; +import org.apache.paimon.data.BinaryString; +import org.apache.paimon.data.GenericRow; +import org.apache.paimon.fs.Path; +import org.apache.paimon.fs.local.LocalFileIO; +import org.apache.paimon.metastore.MetastoreClient; +import org.apache.paimon.schema.Schema; +import org.apache.paimon.schema.SchemaManager; +import org.apache.paimon.table.FileStoreTable; +import org.apache.paimon.table.FileStoreTableFactory; +import org.apache.paimon.table.sink.BatchTableCommit; +import org.apache.paimon.table.sink.BatchTableWrite; +import org.apache.paimon.table.sink.CommitMessage; +import org.apache.paimon.types.DataField; +import org.apache.paimon.types.DataTypes; +import org.apache.paimon.utils.PartitionPathUtils; + +import org.apache.paimon.shade.guava30.com.google.common.collect.ImmutableMap; +import org.apache.paimon.shade.guava30.com.google.common.collect.Lists; +import org.apache.paimon.shade.guava30.com.google.common.collect.Maps; + +import org.assertj.core.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; + +/** Test for {@link HmsReporter}. */ +public class HmsReporterTest { + + @TempDir java.nio.file.Path tempDir; + + @Test + public void testReportAction() throws Exception { + Path tablePath = new Path(tempDir.toString(), "table"); + SchemaManager schemaManager = new SchemaManager(LocalFileIO.create(), tablePath); + Schema schema = + new Schema( + Lists.newArrayList( + new DataField(0, "c1", DataTypes.STRING()), + new DataField(1, "c2", DataTypes.STRING()), + new DataField(2, "c3", DataTypes.STRING())), + Collections.singletonList("c1"), + Collections.emptyList(), + Maps.newHashMap(), + ""); + schemaManager.createTable(schema); + + FileStoreTable table = FileStoreTableFactory.create(LocalFileIO.create(), tablePath); + BatchTableWrite writer = table.newBatchWriteBuilder().newWrite(); + writer.write( + GenericRow.of( + BinaryString.fromString("a"), + BinaryString.fromString("a"), + BinaryString.fromString("a"))); + writer.write( + GenericRow.of( + BinaryString.fromString("b"), + BinaryString.fromString("a"), + BinaryString.fromString("a"))); + List messages = writer.prepareCommit(); + BatchTableCommit committer = table.newBatchWriteBuilder().newCommit(); + committer.commit(messages); + AtomicBoolean closed = new AtomicBoolean(false); + Map> partitionParams = Maps.newHashMap(); + + MetastoreClient client = + new MetastoreClient() { + @Override + public void addPartition(BinaryRow partition) throws Exception { + throw new UnsupportedOperationException(); + } + + @Override + public void addPartition(LinkedHashMap partitionSpec) + throws Exception { + throw new UnsupportedOperationException(); + } + + @Override + public void deletePartition(LinkedHashMap partitionSpec) + throws Exception { + throw new UnsupportedOperationException(); + } + + @Override + public void markDone(LinkedHashMap partitionSpec) + throws Exception { + throw new UnsupportedOperationException(); + } + + @Override + public void alterPartition( + LinkedHashMap partitionSpec, + Map parameters, + long modifyTime, + boolean ignoreIfNotExist) + throws Exception { + partitionParams.put( + PartitionPathUtils.generatePartitionPath(partitionSpec), + parameters); + } + + @Override + public void close() throws Exception { + closed.set(true); + } + }; + + HmsReporter action = new HmsReporter(table, client); + long time = 1729598544974L; + action.report("c1=a/", time); + Assertions.assertThat(partitionParams).containsKey("c1=a/"); + Assertions.assertThat(partitionParams.get("c1=a/")) + .isEqualTo( + ImmutableMap.of( + "numFiles", + "1", + "totalSize", + "591", + "numRows", + "1", + "transient_lastDdlTime", + String.valueOf(time / 1000))); + action.close(); + Assertions.assertThat(closed).isTrue(); + } +} diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/partition/PartitionMarkDoneTest.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/partition/PartitionMarkDoneTest.java index 6dc14c9d0342..f0f4596c61bb 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/partition/PartitionMarkDoneTest.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/partition/PartitionMarkDoneTest.java @@ -51,7 +51,7 @@ import static java.util.Collections.singletonList; import static org.apache.paimon.CoreOptions.DELETION_VECTORS_ENABLED; import static org.apache.paimon.CoreOptions.PARTITION_MARK_DONE_ACTION; -import static org.apache.paimon.flink.FlinkConnectorOptions.PARTITION_MARK_DONE_WHEN_END_INPUT; +import static org.apache.paimon.CoreOptions.PARTITION_MARK_DONE_WHEN_END_INPUT; import static org.assertj.core.api.Assertions.assertThat; class PartitionMarkDoneTest extends TableTestBase { @@ -86,7 +86,7 @@ private void innerTest(boolean deletionVectors) throws Exception { Path location = catalog.getTableLocation(identifier); Path successFile = new Path(location, "a=0/_SUCCESS"); PartitionMarkDone markDone = - PartitionMarkDone.create(false, false, new MockOperatorStateStore(), table); + PartitionMarkDone.create(false, false, new MockOperatorStateStore(), table).get(); notifyCommits(markDone, true); assertThat(table.fileIO().exists(successFile)).isEqualTo(deletionVectors); diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sorter/SortOperatorTest.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sorter/SortOperatorTest.java index 155e259e02bb..c74c1c2c17a4 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sorter/SortOperatorTest.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sorter/SortOperatorTest.java @@ -65,7 +65,8 @@ public void testSort() throws Exception { 128, CompressOptions.defaultOptions(), 1, - MemorySize.MAX_VALUE) {}; + MemorySize.MAX_VALUE, + true) {}; OneInputStreamOperatorTestHarness harness = createTestHarness(sortOperator); harness.open(); @@ -114,7 +115,8 @@ public void testCloseSortOperator() throws Exception { 128, CompressOptions.defaultOptions(), 1, - MemorySize.MAX_VALUE) {}; + MemorySize.MAX_VALUE, + true) {}; OneInputStreamOperatorTestHarness harness = createTestHarness(sortOperator); harness.open(); File[] files = harness.getEnvironment().getIOManager().getSpillingDirectories(); diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/FileStoreSourceSplitGeneratorTest.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/FileStoreSourceSplitGeneratorTest.java index ad30f6388c90..92c8dd94a14e 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/FileStoreSourceSplitGeneratorTest.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/FileStoreSourceSplitGeneratorTest.java @@ -115,7 +115,8 @@ private DataSplit dataSplit(int partition, int bucket, String... fileNames) { 0, // not used 0L, // not used null, // not used - FileSource.APPEND)); + FileSource.APPEND, + null)); } return DataSplit.builder() .withSnapshot(1) diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/FileStoreSourceSplitSerializerTest.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/FileStoreSourceSplitSerializerTest.java index f2c0732a2208..9c884cd5c33d 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/FileStoreSourceSplitSerializerTest.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/FileStoreSourceSplitSerializerTest.java @@ -88,7 +88,8 @@ public static DataFileMeta newFile(int level) { level, 0L, null, - FileSource.APPEND); + FileSource.APPEND, + null); } public static FileStoreSourceSplit newSourceSplit( diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/IteratorSourcesITCase.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/IteratorSourcesITCase.java index 8404d994fa9f..0c5d485af7bc 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/IteratorSourcesITCase.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/IteratorSourcesITCase.java @@ -18,10 +18,10 @@ package org.apache.paimon.flink.source; +import org.apache.commons.collections.IteratorUtils; import org.apache.flink.api.common.eventtime.WatermarkStrategy; import org.apache.flink.runtime.testutils.MiniClusterResourceConfiguration; import org.apache.flink.streaming.api.datastream.DataStream; -import org.apache.flink.streaming.api.datastream.DataStreamUtils; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.table.data.RowData; import org.apache.flink.test.util.MiniClusterWithClientResource; @@ -67,7 +67,7 @@ public void testParallelSourceExecution() throws Exception { "iterator source"); final List result = - DataStreamUtils.collectBoundedStream(stream, "Iterator Source Test"); + IteratorUtils.toList(stream.executeAndCollect("Iterator Source Test")); verifySequence(result, 1L, 1_000L); } diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/TestChangelogDataReadWrite.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/TestChangelogDataReadWrite.java index aab8666fe998..d2bb9eb98274 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/TestChangelogDataReadWrite.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/TestChangelogDataReadWrite.java @@ -69,7 +69,7 @@ public class TestChangelogDataReadWrite { private static final RowType KEY_TYPE = new RowType(singletonList(new DataField(0, "k", new BigIntType()))); private static final RowType VALUE_TYPE = - new RowType(singletonList(new DataField(0, "v", new BigIntType()))); + new RowType(singletonList(new DataField(1, "v", new BigIntType()))); private static final RowType PARTITION_TYPE = new RowType(singletonList(new DataField(0, "p", new IntType()))); private static final Comparator COMPARATOR = @@ -87,7 +87,7 @@ public List keyFields(TableSchema schema) { @Override public List valueFields(TableSchema schema) { return Collections.singletonList( - new DataField(0, "v", new org.apache.paimon.types.BigIntType(false))); + new DataField(1, "v", new org.apache.paimon.types.BigIntType(false))); } }; @@ -107,7 +107,11 @@ public TestChangelogDataReadWrite(String root) { "default", CoreOptions.FILE_FORMAT.defaultValue().toString(), CoreOptions.DATA_FILE_PREFIX.defaultValue(), - CoreOptions.CHANGELOG_FILE_PREFIX.defaultValue()); + CoreOptions.CHANGELOG_FILE_PREFIX.defaultValue(), + CoreOptions.PARTITION_GENERATE_LEGCY_NAME.defaultValue(), + CoreOptions.FILE_SUFFIX_INCLUDE_COMPRESSION.defaultValue(), + CoreOptions.FILE_COMPRESSION.defaultValue(), + null); this.snapshotManager = new SnapshotManager(LocalFileIO.create(), new Path(root)); this.commitUser = UUID.randomUUID().toString(); } diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/operator/OperatorSourceTest.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/operator/OperatorSourceTest.java index 61a03a29a21b..0cd969707cfa 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/operator/OperatorSourceTest.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/operator/OperatorSourceTest.java @@ -33,12 +33,17 @@ import org.apache.paimon.table.source.TableRead; import org.apache.paimon.types.DataTypes; +import org.apache.flink.api.common.eventtime.WatermarkStrategy; import org.apache.flink.metrics.MetricGroup; import org.apache.flink.runtime.checkpoint.OperatorSubtaskState; -import org.apache.flink.streaming.api.functions.source.SourceFunction; -import org.apache.flink.streaming.api.operators.StreamSource; +import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; +import org.apache.flink.streaming.api.operators.SourceOperator; import org.apache.flink.streaming.api.watermark.Watermark; +import org.apache.flink.streaming.runtime.io.PushingAsyncDataInput; +import org.apache.flink.streaming.runtime.streamrecord.LatencyMarker; +import org.apache.flink.streaming.runtime.streamrecord.RecordAttributes; import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; +import org.apache.flink.streaming.runtime.watermarkstatus.WatermarkStatus; import org.apache.flink.streaming.util.AbstractStreamOperatorTestHarness; import org.apache.flink.streaming.util.OneInputStreamOperatorTestHarness; import org.apache.flink.table.data.GenericRowData; @@ -46,6 +51,7 @@ import org.apache.flink.table.runtime.typeutils.InternalSerializers; import org.apache.flink.table.types.logical.IntType; import org.apache.flink.table.types.logical.RowType; +import org.apache.flink.util.CloseableIterator; import org.apache.flink.util.function.SupplierWithException; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -58,11 +64,13 @@ import java.util.List; import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; import static org.apache.paimon.CoreOptions.CONSUMER_ID; import static org.assertj.core.api.Assertions.assertThat; -/** Test for {@link MonitorFunction} and {@link ReadOperator}. */ +/** Test for {@link MonitorSource} and {@link ReadOperator}. */ public class OperatorSourceTest { @TempDir Path tempDir; @@ -114,28 +122,39 @@ private List> readSplit(Split split) throws IOException { } @Test - public void testMonitorFunction() throws Exception { + public void testMonitorSource() throws Exception { + StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); // 1. run first OperatorSubtaskState snapshot; { - MonitorFunction function = new MonitorFunction(table.newReadBuilder(), 10, false); - StreamSource src = new StreamSource<>(function); + MonitorSource source = new MonitorSource(table.newReadBuilder(), 10, false); + TestingSourceOperator operator = + (TestingSourceOperator) + TestingSourceOperator.createTestOperator( + source.createReader(null), + WatermarkStrategy.noWatermarks(), + false); AbstractStreamOperatorTestHarness testHarness = - new AbstractStreamOperatorTestHarness<>(src, 1, 1, 0); + new AbstractStreamOperatorTestHarness<>(operator, 1, 1, 0); testHarness.open(); - snapshot = testReadSplit(function, () -> testHarness.snapshot(0, 0), 1, 1, 1); + snapshot = testReadSplit(operator, () -> testHarness.snapshot(0, 0), 1, 1, 1); } // 2. restore from state { - MonitorFunction functionCopy1 = new MonitorFunction(table.newReadBuilder(), 10, false); - StreamSource srcCopy1 = new StreamSource<>(functionCopy1); + MonitorSource sourceCopy1 = new MonitorSource(table.newReadBuilder(), 10, false); + TestingSourceOperator operatorCopy1 = + (TestingSourceOperator) + TestingSourceOperator.createTestOperator( + sourceCopy1.createReader(null), + WatermarkStrategy.noWatermarks(), + false); AbstractStreamOperatorTestHarness testHarnessCopy1 = - new AbstractStreamOperatorTestHarness<>(srcCopy1, 1, 1, 0); + new AbstractStreamOperatorTestHarness<>(operatorCopy1, 1, 1, 0); testHarnessCopy1.initializeState(snapshot); testHarnessCopy1.open(); testReadSplit( - functionCopy1, + operatorCopy1, () -> { testHarnessCopy1.snapshot(1, 1); testHarnessCopy1.notifyOfCompletedCheckpoint(1); @@ -148,12 +167,17 @@ public void testMonitorFunction() throws Exception { // 3. restore from consumer id { - MonitorFunction functionCopy2 = new MonitorFunction(table.newReadBuilder(), 10, false); - StreamSource srcCopy2 = new StreamSource<>(functionCopy2); + MonitorSource sourceCopy2 = new MonitorSource(table.newReadBuilder(), 10, false); + TestingSourceOperator operatorCopy2 = + (TestingSourceOperator) + TestingSourceOperator.createTestOperator( + sourceCopy2.createReader(null), + WatermarkStrategy.noWatermarks(), + false); AbstractStreamOperatorTestHarness testHarnessCopy2 = - new AbstractStreamOperatorTestHarness<>(srcCopy2, 1, 1, 0); + new AbstractStreamOperatorTestHarness<>(operatorCopy2, 1, 1, 0); testHarnessCopy2.open(); - testReadSplit(functionCopy2, () -> null, 3, 3, 3); + testReadSplit(operatorCopy2, () -> null, 3, 3, 3); } } @@ -204,6 +228,14 @@ public void testReadOperatorMetricsRegisterAndUpdate() throws Exception { .getValue()) .isEqualTo(-1L); + Thread.sleep(300L); + assertThat( + (Long) + TestingMetricUtils.getGauge( + readerOperatorMetricGroup, "sourceIdleTime") + .getValue()) + .isGreaterThan(299L); + harness.processElement(new StreamRecord<>(splits.get(0))); assertThat( (Long) @@ -228,10 +260,18 @@ public void testReadOperatorMetricsRegisterAndUpdate() throws Exception { "currentEmitEventTimeLag") .getValue()) .isEqualTo(emitEventTimeLag); + + assertThat( + (Long) + TestingMetricUtils.getGauge( + readerOperatorMetricGroup, "sourceIdleTime") + .getValue()) + .isGreaterThan(99L) + .isLessThan(300L); } private T testReadSplit( - MonitorFunction function, + SourceOperator operator, SupplierWithException beforeClose, int a, int b, @@ -239,20 +279,36 @@ private T testReadSplit( throws Exception { Throwable[] error = new Throwable[1]; ArrayBlockingQueue queue = new ArrayBlockingQueue<>(10); + AtomicReference> iteratorRef = new AtomicReference<>(); - DummySourceContext sourceContext = - new DummySourceContext() { + PushingAsyncDataInput.DataOutput output = + new PushingAsyncDataInput.DataOutput() { @Override - public void collect(Split element) { - queue.add(element); + public void emitRecord(StreamRecord streamRecord) { + queue.add(streamRecord.getValue()); } + + @Override + public void emitWatermark(Watermark watermark) {} + + @Override + public void emitWatermarkStatus(WatermarkStatus watermarkStatus) {} + + @Override + public void emitLatencyMarker(LatencyMarker latencyMarker) {} + + @Override + public void emitRecordAttributes(RecordAttributes recordAttributes) {} }; + AtomicBoolean isRunning = new AtomicBoolean(true); Thread runner = new Thread( () -> { try { - function.run(sourceContext); + while (isRunning.get()) { + operator.emitNext(output); + } } catch (Throwable t) { t.printStackTrace(); error[0] = t; @@ -266,34 +322,15 @@ public void collect(Split element) { assertThat(readSplit(split)).containsExactlyInAnyOrder(Arrays.asList(a, b, c)); T t = beforeClose.get(); - function.cancel(); + CloseableIterator iterator = iteratorRef.get(); + if (iterator != null) { + iterator.close(); + } + isRunning.set(false); runner.join(); assertThat(error[0]).isNull(); return t; } - - private abstract static class DummySourceContext - implements SourceFunction.SourceContext { - - private final Object lock = new Object(); - - @Override - public void collectWithTimestamp(Split element, long timestamp) {} - - @Override - public void emitWatermark(Watermark mark) {} - - @Override - public void markAsTemporarilyIdle() {} - - @Override - public Object getCheckpointLock() { - return lock; - } - - @Override - public void close() {} - } } diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/operator/TestingSourceOperator.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/operator/TestingSourceOperator.java new file mode 100644 index 000000000000..77b44d5b0e5c --- /dev/null +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/operator/TestingSourceOperator.java @@ -0,0 +1,184 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.flink.source.operator; + +import org.apache.paimon.flink.source.SimpleSourceSplit; +import org.apache.paimon.flink.source.SimpleSourceSplitSerializer; + +import org.apache.flink.api.common.ExecutionConfig; +import org.apache.flink.api.common.eventtime.WatermarkStrategy; +import org.apache.flink.api.common.state.OperatorStateStore; +import org.apache.flink.api.connector.source.SourceReader; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.core.fs.CloseableRegistry; +import org.apache.flink.runtime.execution.Environment; +import org.apache.flink.runtime.metrics.groups.UnregisteredMetricGroups; +import org.apache.flink.runtime.operators.coordination.OperatorEvent; +import org.apache.flink.runtime.operators.coordination.OperatorEventGateway; +import org.apache.flink.runtime.operators.testutils.DummyEnvironment; +import org.apache.flink.runtime.operators.testutils.MockEnvironmentBuilder; +import org.apache.flink.runtime.state.AbstractStateBackend; +import org.apache.flink.runtime.state.OperatorStateBackendParametersImpl; +import org.apache.flink.runtime.state.StateInitializationContext; +import org.apache.flink.runtime.state.StateInitializationContextImpl; +import org.apache.flink.runtime.state.hashmap.HashMapStateBackend; +import org.apache.flink.streaming.api.operators.SourceOperator; +import org.apache.flink.streaming.api.operators.StreamOperatorParameters; +import org.apache.flink.streaming.api.operators.StreamingRuntimeContext; +import org.apache.flink.streaming.runtime.tasks.ProcessingTimeService; +import org.apache.flink.streaming.runtime.tasks.SourceOperatorStreamTask; +import org.apache.flink.streaming.runtime.tasks.TestProcessingTimeService; +import org.apache.flink.streaming.util.MockOutput; +import org.apache.flink.streaming.util.MockStreamConfig; +import org.apache.flink.streaming.util.MockStreamingRuntimeContext; + +import java.util.ArrayList; +import java.util.Collections; + +/** + * A SourceOperator extension to simplify test setup. + * + *

    This class is implemented in reference to {@link + * org.apache.flink.streaming.api.operators.source.TestingSourceOperator}. + * + *

    See Flink + * PR that introduced this class + */ +public class TestingSourceOperator extends SourceOperator { + + private static final long serialVersionUID = 1L; + + private final int subtaskIndex; + private final int parallelism; + + public TestingSourceOperator( + StreamOperatorParameters parameters, + SourceReader reader, + WatermarkStrategy watermarkStrategy, + ProcessingTimeService timeService, + boolean emitProgressiveWatermarks) { + + this( + parameters, + reader, + watermarkStrategy, + timeService, + new TestingOperatorEventGateway(), + 1, + 5, + emitProgressiveWatermarks); + } + + public TestingSourceOperator( + StreamOperatorParameters parameters, + SourceReader reader, + WatermarkStrategy watermarkStrategy, + ProcessingTimeService timeService, + OperatorEventGateway eventGateway, + int subtaskIndex, + int parallelism, + boolean emitProgressiveWatermarks) { + + super( + (context) -> reader, + eventGateway, + new SimpleSourceSplitSerializer(), + watermarkStrategy, + timeService, + new Configuration(), + "localhost", + emitProgressiveWatermarks, + () -> false); + + this.subtaskIndex = subtaskIndex; + this.parallelism = parallelism; + this.metrics = UnregisteredMetricGroups.createUnregisteredOperatorMetricGroup(); + initSourceMetricGroup(); + + // unchecked wrapping is okay to keep tests simpler + try { + initReader(); + } catch (Exception e) { + throw new RuntimeException(e); + } + + setup(parameters.getContainingTask(), parameters.getStreamConfig(), parameters.getOutput()); + } + + @Override + public StreamingRuntimeContext getRuntimeContext() { + return new MockStreamingRuntimeContext(false, parallelism, subtaskIndex); + } + + // this is overridden to avoid complex mock injection through the "containingTask" + @Override + public ExecutionConfig getExecutionConfig() { + ExecutionConfig cfg = new ExecutionConfig(); + cfg.setAutoWatermarkInterval(100); + return cfg; + } + + public static SourceOperator createTestOperator( + SourceReader reader, + WatermarkStrategy watermarkStrategy, + boolean emitProgressiveWatermarks) + throws Exception { + + AbstractStateBackend abstractStateBackend = new HashMapStateBackend(); + Environment env = new MockEnvironmentBuilder().build(); + CloseableRegistry cancelStreamRegistry = new CloseableRegistry(); + final OperatorStateStore operatorStateStore = + abstractStateBackend.createOperatorStateBackend( + new OperatorStateBackendParametersImpl( + env, + "test-operator", + Collections.emptyList(), + cancelStreamRegistry)); + + final StateInitializationContext stateContext = + new StateInitializationContextImpl(null, operatorStateStore, null, null, null); + + TestProcessingTimeService timeService = new TestProcessingTimeService(); + timeService.setCurrentTime(Integer.MAX_VALUE); // start somewhere that is not zero + + final SourceOperator sourceOperator = + new TestingSourceOperator<>( + new StreamOperatorParameters<>( + new SourceOperatorStreamTask(new DummyEnvironment()), + new MockStreamConfig(new Configuration(), 1), + new MockOutput<>(new ArrayList<>()), + null, + null, + null), + reader, + watermarkStrategy, + timeService, + emitProgressiveWatermarks); + sourceOperator.initializeState(stateContext); + sourceOperator.open(); + + return sourceOperator; + } + + private static class TestingOperatorEventGateway implements OperatorEventGateway { + @Override + public void sendEventToCoordinator(OperatorEvent event) {} + } +} diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/statistics/FileStoreTableStatisticsTestBase.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/statistics/FileStoreTableStatisticsTestBase.java index f8aadb8bdc3b..42a47ea1e298 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/statistics/FileStoreTableStatisticsTestBase.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/statistics/FileStoreTableStatisticsTestBase.java @@ -18,13 +18,18 @@ package org.apache.paimon.flink.source.statistics; +import org.apache.paimon.FileStore; +import org.apache.paimon.Snapshot; import org.apache.paimon.data.BinaryString; import org.apache.paimon.data.GenericRow; import org.apache.paimon.data.InternalRow; import org.apache.paimon.flink.source.DataTableSource; import org.apache.paimon.fs.Path; +import org.apache.paimon.operation.FileStoreCommit; import org.apache.paimon.predicate.PredicateBuilder; import org.apache.paimon.schema.Schema; +import org.apache.paimon.stats.ColStats; +import org.apache.paimon.stats.Statistics; import org.apache.paimon.table.FileStoreTable; import org.apache.paimon.table.sink.StreamTableCommit; import org.apache.paimon.table.sink.StreamTableWrite; @@ -33,6 +38,7 @@ import org.apache.paimon.types.VarCharType; import org.apache.flink.table.catalog.ObjectIdentifier; +import org.apache.flink.table.plan.stats.ColumnStats; import org.apache.flink.table.plan.stats.TableStats; import org.assertj.core.api.Assertions; import org.junit.jupiter.api.BeforeEach; @@ -40,6 +46,8 @@ import org.junit.jupiter.api.io.TempDir; import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; import java.util.UUID; /** Statistics tests for {@link FileStoreTable}. */ @@ -60,9 +68,66 @@ public void before() { @Test public void testTableScanStatistics() throws Exception { FileStoreTable table = writeData(); + Map> colStatsMap = new HashMap<>(); + colStatsMap.put("pt", ColStats.newColStats(0, 2L, 1, 2, 0L, null, null)); + colStatsMap.put("a", ColStats.newColStats(1, 9L, 10, 90, 0L, null, null)); + colStatsMap.put("b", ColStats.newColStats(2, 7L, 100L, 900L, 2L, null, null)); + colStatsMap.put( + "c", + ColStats.newColStats( + 3, + 7L, + BinaryString.fromString("S1"), + BinaryString.fromString("S8"), + 2L, + null, + null)); + + FileStore fileStore = table.store(); + FileStoreCommit fileStoreCommit = fileStore.newCommit(commitUser); + Snapshot latestSnapshot = fileStore.snapshotManager().latestSnapshot(); + Statistics colStats = + new Statistics( + latestSnapshot.id(), latestSnapshot.schemaId(), 9L, null, colStatsMap); + fileStoreCommit.commitStatistics(colStats, Long.MAX_VALUE); + fileStoreCommit.close(); DataTableSource scanSource = new DataTableSource(identifier, table, false, null, null); Assertions.assertThat(scanSource.reportStatistics().getRowCount()).isEqualTo(9L); - // TODO validate column statistics + Map expectedColStats = new HashMap<>(); + expectedColStats.put( + "pt", + ColumnStats.Builder.builder() + .setNdv(2L) + .setMin(1) + .setMax(2) + .setNullCount(0L) + .build()); + expectedColStats.put( + "a", + ColumnStats.Builder.builder() + .setNdv(9L) + .setMin(10) + .setMax(90) + .setNullCount(0L) + .build()); + expectedColStats.put( + "b", + ColumnStats.Builder.builder() + .setNdv(7L) + .setMin(100L) + .setMax(900L) + .setNullCount(2L) + .build()); + expectedColStats.put( + "c", + ColumnStats.Builder.builder() + .setNdv(7L) + .setMin(BinaryString.fromString("S1")) + .setMax(BinaryString.fromString("S8")) + .setNullCount(2L) + .build()); + Assertions.assertThat(scanSource.reportStatistics().getColumnStats()) + .isEqualTo(expectedColStats); } @Test @@ -88,9 +153,67 @@ public void testTableFilterPartitionStatistics() throws Exception { null, null, null, - false); + null); Assertions.assertThat(partitionFilterSource.reportStatistics().getRowCount()).isEqualTo(5L); - // TODO validate column statistics + Map> colStatsMap = new HashMap<>(); + colStatsMap.put("pt", ColStats.newColStats(0, 1L, 1, 2, 0L, null, null)); + colStatsMap.put("a", ColStats.newColStats(1, 5L, 10, 90, 0L, null, null)); + colStatsMap.put("b", ColStats.newColStats(2, 3L, 100L, 900L, 2L, null, null)); + colStatsMap.put( + "c", + ColStats.newColStats( + 3, + 3L, + BinaryString.fromString("S1"), + BinaryString.fromString("S7"), + 2L, + null, + null)); + + FileStore fileStore = table.store(); + FileStoreCommit fileStoreCommit = fileStore.newCommit(commitUser); + Snapshot latestSnapshot = fileStore.snapshotManager().latestSnapshot(); + Statistics colStats = + new Statistics( + latestSnapshot.id(), latestSnapshot.schemaId(), 9L, null, colStatsMap); + fileStoreCommit.commitStatistics(colStats, Long.MAX_VALUE); + fileStoreCommit.close(); + + Map expectedColStats = new HashMap<>(); + expectedColStats.put( + "pt", + ColumnStats.Builder.builder() + .setNdv(1L) + .setMin(1) + .setMax(2) + .setNullCount(0L) + .build()); + expectedColStats.put( + "a", + ColumnStats.Builder.builder() + .setNdv(5L) + .setMin(10) + .setMax(90) + .setNullCount(0L) + .build()); + expectedColStats.put( + "b", + ColumnStats.Builder.builder() + .setNdv(3L) + .setMin(100L) + .setMax(900L) + .setNullCount(2L) + .build()); + expectedColStats.put( + "c", + ColumnStats.Builder.builder() + .setNdv(3L) + .setMin(BinaryString.fromString("S1")) + .setMax(BinaryString.fromString("S7")) + .setNullCount(2L) + .build()); + Assertions.assertThat(partitionFilterSource.reportStatistics().getColumnStats()) + .isEqualTo(expectedColStats); } @Test @@ -109,9 +232,67 @@ public void testTableFilterKeyStatistics() throws Exception { null, null, null, - false); + null); Assertions.assertThat(keyFilterSource.reportStatistics().getRowCount()).isEqualTo(2L); - // TODO validate column statistics + Map> colStatsMap = new HashMap<>(); + colStatsMap.put("pt", ColStats.newColStats(0, 1L, 2, 2, 0L, null, null)); + colStatsMap.put("a", ColStats.newColStats(1, 1L, 50, 50, 0L, null, null)); + colStatsMap.put("b", ColStats.newColStats(2, 1L, null, null, 1L, null, null)); + colStatsMap.put( + "c", + ColStats.newColStats( + 3, + 1L, + BinaryString.fromString("S5"), + BinaryString.fromString("S5"), + 0L, + null, + null)); + + FileStore fileStore = table.store(); + FileStoreCommit fileStoreCommit = fileStore.newCommit(commitUser); + Snapshot latestSnapshot = fileStore.snapshotManager().latestSnapshot(); + Statistics colStats = + new Statistics( + latestSnapshot.id(), latestSnapshot.schemaId(), 9L, null, colStatsMap); + fileStoreCommit.commitStatistics(colStats, Long.MAX_VALUE); + fileStoreCommit.close(); + + Map expectedColStats = new HashMap<>(); + expectedColStats.put( + "pt", + ColumnStats.Builder.builder() + .setNdv(1L) + .setMin(2) + .setMax(2) + .setNullCount(0L) + .build()); + expectedColStats.put( + "a", + ColumnStats.Builder.builder() + .setNdv(1L) + .setMin(50) + .setMax(50) + .setNullCount(0L) + .build()); + expectedColStats.put( + "b", + ColumnStats.Builder.builder() + .setNdv(1L) + .setMin(null) + .setMax(null) + .setNullCount(1L) + .build()); + expectedColStats.put( + "c", + ColumnStats.Builder.builder() + .setNdv(1L) + .setMin(BinaryString.fromString("S5")) + .setMax(BinaryString.fromString("S5")) + .setNullCount(0L) + .build()); + Assertions.assertThat(keyFilterSource.reportStatistics().getColumnStats()) + .isEqualTo(expectedColStats); } @Test @@ -130,9 +311,67 @@ public void testTableFilterValueStatistics() throws Exception { null, null, null, - false); + null); Assertions.assertThat(keyFilterSource.reportStatistics().getRowCount()).isEqualTo(4L); - // TODO validate column statistics + Map> colStatsMap = new HashMap<>(); + colStatsMap.put("pt", ColStats.newColStats(0, 4L, 2, 2, 0L, null, null)); + colStatsMap.put("a", ColStats.newColStats(1, 4L, 50, 50, 0L, null, null)); + colStatsMap.put("b", ColStats.newColStats(2, 4L, null, null, 1L, null, null)); + colStatsMap.put( + "c", + ColStats.newColStats( + 3, + 4L, + BinaryString.fromString("S5"), + BinaryString.fromString("S8"), + 0L, + null, + null)); + + FileStore fileStore = table.store(); + FileStoreCommit fileStoreCommit = fileStore.newCommit(commitUser); + Snapshot latestSnapshot = fileStore.snapshotManager().latestSnapshot(); + Statistics colStats = + new Statistics( + latestSnapshot.id(), latestSnapshot.schemaId(), 9L, null, colStatsMap); + fileStoreCommit.commitStatistics(colStats, Long.MAX_VALUE); + fileStoreCommit.close(); + + Map expectedColStats = new HashMap<>(); + expectedColStats.put( + "pt", + ColumnStats.Builder.builder() + .setNdv(4L) + .setMin(2) + .setMax(2) + .setNullCount(0L) + .build()); + expectedColStats.put( + "a", + ColumnStats.Builder.builder() + .setNdv(4L) + .setMin(50) + .setMax(50) + .setNullCount(0L) + .build()); + expectedColStats.put( + "b", + ColumnStats.Builder.builder() + .setNdv(4L) + .setMin(null) + .setMax(null) + .setNullCount(1L) + .build()); + expectedColStats.put( + "c", + ColumnStats.Builder.builder() + .setNdv(4L) + .setMin(BinaryString.fromString("S5")) + .setMax(BinaryString.fromString("S8")) + .setNullCount(0L) + .build()); + Assertions.assertThat(keyFilterSource.reportStatistics().getColumnStats()) + .isEqualTo(expectedColStats); } protected FileStoreTable writeData() throws Exception { diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/statistics/PrimaryKeyTableStatisticsTest.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/statistics/PrimaryKeyTableStatisticsTest.java index f5d4121672b0..ea47df2d9d72 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/statistics/PrimaryKeyTableStatisticsTest.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/statistics/PrimaryKeyTableStatisticsTest.java @@ -52,7 +52,7 @@ public void testTableFilterValueStatistics() throws Exception { null, null, null, - false); + null); Assertions.assertThat(keyFilterSource.reportStatistics().getRowCount()).isEqualTo(9L); // TODO validate column statistics } diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/util/AbstractTestBase.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/util/AbstractTestBase.java index ce0017eb1874..ee838ed68255 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/util/AbstractTestBase.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/util/AbstractTestBase.java @@ -19,6 +19,7 @@ package org.apache.paimon.flink.util; import org.apache.paimon.utils.FileIOUtils; +import org.apache.paimon.utils.TimeUtils; import org.apache.flink.api.common.RuntimeExecutionMode; import org.apache.flink.api.dag.Transformation; @@ -29,7 +30,6 @@ import org.apache.flink.runtime.client.JobStatusMessage; import org.apache.flink.runtime.testutils.MiniClusterResourceConfiguration; import org.apache.flink.streaming.api.CheckpointingMode; -import org.apache.flink.streaming.api.environment.ExecutionCheckpointingOptions; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.table.api.EnvironmentSettings; import org.apache.flink.table.api.TableEnvironment; @@ -164,6 +164,11 @@ public TableEnvironmentBuilder setConf(ConfigOption option, T value) { return this; } + public TableEnvironmentBuilder setString(String key, String value) { + conf.setString(key, value); + return this; + } + public TableEnvironmentBuilder setConf(Configuration conf) { this.conf.addAll(conf); return this; @@ -182,9 +187,10 @@ public TableEnvironment build() { if (checkpointIntervalMs != null) { tEnv.getConfig() .getConfiguration() - .set( - ExecutionCheckpointingOptions.CHECKPOINTING_INTERVAL, - Duration.ofMillis(checkpointIntervalMs)); + .setString( + "execution.checkpointing.interval", + TimeUtils.formatWithHighestUnit( + Duration.ofMillis(checkpointIntervalMs))); } } else { tEnv = diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/util/MiniClusterWithClientExtension.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/util/MiniClusterWithClientExtension.java index cfc23a0a44d8..39939f78670b 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/util/MiniClusterWithClientExtension.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/util/MiniClusterWithClientExtension.java @@ -29,7 +29,6 @@ import org.apache.flink.runtime.testutils.MiniClusterResourceConfiguration; import org.apache.flink.streaming.util.TestStreamEnvironment; import org.apache.flink.test.junit5.InjectClusterClient; -import org.apache.flink.test.util.TestEnvironment; import org.junit.jupiter.api.extension.AfterAllCallback; import org.junit.jupiter.api.extension.AfterEachCallback; import org.junit.jupiter.api.extension.BeforeAllCallback; @@ -167,17 +166,12 @@ private void registerEnv(InternalMiniClusterExtension internalMiniClusterExtensi .getOptional(CoreOptions.DEFAULT_PARALLELISM) .orElse(internalMiniClusterExtension.getNumberSlots()); - TestEnvironment executionEnvironment = - new TestEnvironment( - internalMiniClusterExtension.getMiniCluster(), defaultParallelism, false); - executionEnvironment.setAsContext(); TestStreamEnvironment.setAsContext( internalMiniClusterExtension.getMiniCluster(), defaultParallelism); } private void unregisterEnv(InternalMiniClusterExtension internalMiniClusterExtension) { TestStreamEnvironment.unsetAsContext(); - TestEnvironment.unsetAsContext(); } private MiniClusterClient createMiniClusterClient( diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/util/ReadWriteTableTestUtil.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/util/ReadWriteTableTestUtil.java index 86b0014eb39c..0eac2ed2936e 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/util/ReadWriteTableTestUtil.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/util/ReadWriteTableTestUtil.java @@ -23,8 +23,8 @@ import org.apache.paimon.utils.BlockingIterator; import org.apache.flink.api.common.RuntimeExecutionMode; -import org.apache.flink.api.common.restartstrategy.RestartStrategies; -import org.apache.flink.api.common.time.Time; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.configuration.RestartStrategyOptions; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.table.api.EnvironmentSettings; import org.apache.flink.table.api.TableEnvironment; @@ -36,6 +36,7 @@ import javax.annotation.Nullable; import java.nio.file.Paths; +import java.time.Duration; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -52,7 +53,7 @@ /** Test util for {@link ReadWriteTableITCase}. */ public class ReadWriteTableTestUtil { - private static final Time TIME_OUT = Time.seconds(10); + private static final Duration TIME_OUT = Duration.ofSeconds(10); public static final int DEFAULT_PARALLELISM = 2; @@ -75,12 +76,11 @@ public static void init(String warehouse) { } public static void init(String warehouse, int parallelism) { - StreamExecutionEnvironment sExeEnv = buildStreamEnv(parallelism); - sExeEnv.getConfig().setRestartStrategy(RestartStrategies.noRestart()); + // Using `none` to avoid compatibility issues with Flink 1.18-. + StreamExecutionEnvironment sExeEnv = buildStreamEnv(parallelism, "none"); sEnv = StreamTableEnvironment.create(sExeEnv); - bExeEnv = buildBatchEnv(parallelism); - bExeEnv.getConfig().setRestartStrategy(RestartStrategies.noRestart()); + bExeEnv = buildBatchEnv(parallelism, "none"); bEnv = StreamTableEnvironment.create(bExeEnv, EnvironmentSettings.inBatchMode()); ReadWriteTableTestUtil.warehouse = warehouse; @@ -95,16 +95,24 @@ public static void init(String warehouse, int parallelism) { bEnv.useCatalog(catalog); } - public static StreamExecutionEnvironment buildStreamEnv(int parallelism) { - final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); + public static StreamExecutionEnvironment buildStreamEnv( + int parallelism, String restartStrategy) { + Configuration configuration = new Configuration(); + configuration.set(RestartStrategyOptions.RESTART_STRATEGY, restartStrategy); + final StreamExecutionEnvironment env = + StreamExecutionEnvironment.getExecutionEnvironment(configuration); env.setRuntimeMode(RuntimeExecutionMode.STREAMING); env.enableCheckpointing(100); env.setParallelism(parallelism); return env; } - public static StreamExecutionEnvironment buildBatchEnv(int parallelism) { - final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); + public static StreamExecutionEnvironment buildBatchEnv( + int parallelism, String restartStrategy) { + Configuration configuration = new Configuration(); + configuration.set(RestartStrategyOptions.RESTART_STRATEGY, restartStrategy); + final StreamExecutionEnvironment env = + StreamExecutionEnvironment.getExecutionEnvironment(configuration); env.setRuntimeMode(RuntimeExecutionMode.BATCH); env.setParallelism(parallelism); return env; @@ -270,7 +278,7 @@ public static void testBatchRead(String query, List expected) throws Except try (BlockingIterator iterator = BlockingIterator.of(resultItr)) { if (!expected.isEmpty()) { List result = - iterator.collect(expected.size(), TIME_OUT.getSize(), TIME_OUT.getUnit()); + iterator.collect(expected.size(), TIME_OUT.getSeconds(), TimeUnit.SECONDS); assertThat(toInsertOnlyRows(result)) .containsExactlyInAnyOrderElementsOf(toInsertOnlyRows(expected)); } diff --git a/paimon-flink/paimon-flink-common/src/test/resources/META-INF/services/org.apache.paimon.factories.Factory b/paimon-flink/paimon-flink-common/src/test/resources/META-INF/services/org.apache.paimon.factories.Factory index fcb6fe982943..3c05b5fba3ec 100644 --- a/paimon-flink/paimon-flink-common/src/test/resources/META-INF/services/org.apache.paimon.factories.Factory +++ b/paimon-flink/paimon-flink-common/src/test/resources/META-INF/services/org.apache.paimon.factories.Factory @@ -15,8 +15,5 @@ org.apache.paimon.flink.FlinkCatalogTest$TestingLogSoreRegisterFactory -# Lineage meta factory -org.apache.paimon.flink.FlinkLineageITCase$TestingMemoryLineageMetaFactory - # Catalog lock factory org.apache.paimon.flink.FileSystemCatalogITCase$FileSystemCatalogDummyLockFactory \ No newline at end of file diff --git a/paimon-format/pom.xml b/paimon-format/pom.xml index e96fddcd8480..5804a704b0cf 100644 --- a/paimon-format/pom.xml +++ b/paimon-format/pom.xml @@ -32,13 +32,10 @@ under the License. Paimon : Format - 1.13.1 - 1.9.2 2.5 1.6 3.12.0 2.8.1 - 3.19.6 diff --git a/paimon-format/src/main/java/org/apache/orc/impl/RecordReaderImpl.java b/paimon-format/src/main/java/org/apache/orc/impl/RecordReaderImpl.java new file mode 100644 index 000000000000..93aa0719caea --- /dev/null +++ b/paimon-format/src/main/java/org/apache/orc/impl/RecordReaderImpl.java @@ -0,0 +1,1891 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.orc.impl; + +import org.apache.paimon.fileindex.FileIndexResult; +import org.apache.paimon.fileindex.bitmap.BitmapIndexResult; +import org.apache.paimon.utils.RoaringBitmap32; + +import org.apache.commons.lang3.ArrayUtils; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.common.type.HiveDecimal; +import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; +import org.apache.hadoop.hive.ql.io.sarg.PredicateLeaf; +import org.apache.hadoop.hive.ql.io.sarg.SearchArgument; +import org.apache.hadoop.hive.ql.util.TimestampUtils; +import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; +import org.apache.hadoop.io.Text; +import org.apache.orc.BooleanColumnStatistics; +import org.apache.orc.CollectionColumnStatistics; +import org.apache.orc.ColumnStatistics; +import org.apache.orc.CompressionCodec; +import org.apache.orc.DataReader; +import org.apache.orc.DateColumnStatistics; +import org.apache.orc.DecimalColumnStatistics; +import org.apache.orc.DoubleColumnStatistics; +import org.apache.orc.IntegerColumnStatistics; +import org.apache.orc.OrcConf; +import org.apache.orc.OrcFile; +import org.apache.orc.OrcFilterContext; +import org.apache.orc.OrcProto; +import org.apache.orc.Reader; +import org.apache.orc.RecordReader; +import org.apache.orc.StringColumnStatistics; +import org.apache.orc.StripeInformation; +import org.apache.orc.TimestampColumnStatistics; +import org.apache.orc.TypeDescription; +import org.apache.orc.filter.BatchFilter; +import org.apache.orc.impl.filter.FilterFactory; +import org.apache.orc.impl.reader.ReaderEncryption; +import org.apache.orc.impl.reader.StripePlanner; +import org.apache.orc.impl.reader.tree.BatchReader; +import org.apache.orc.impl.reader.tree.TypeReader; +import org.apache.orc.util.BloomFilter; +import org.apache.orc.util.BloomFilterIO; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.annotation.Nullable; + +import java.io.IOException; +import java.math.BigDecimal; +import java.sql.Timestamp; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.LocalTime; +import java.time.ZoneOffset; +import java.time.chrono.ChronoLocalDate; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Date; +import java.util.List; +import java.util.SortedSet; +import java.util.TimeZone; +import java.util.TreeSet; +import java.util.function.Consumer; + +/* This file is based on source code from the ORC Project (http://orc.apache.org/), licensed by the Apache + * Software Foundation (ASF) under the Apache License, Version 2.0. See the NOTICE file distributed with this work for + * additional information regarding copyright ownership. */ + +/** An orc RecordReaderImpl. */ +public class RecordReaderImpl implements RecordReader { + static final Logger LOG = LoggerFactory.getLogger(RecordReaderImpl.class); + private static final boolean isLogDebugEnabled = LOG.isDebugEnabled(); + // as public for use with test cases + public static final OrcProto.ColumnStatistics EMPTY_COLUMN_STATISTICS = + OrcProto.ColumnStatistics.newBuilder() + .setNumberOfValues(0) + .setHasNull(false) + .setBytesOnDisk(0) + .build(); + protected final Path path; + private final long firstRow; + private final List stripes = new ArrayList<>(); + private OrcProto.StripeFooter stripeFooter; + private final long totalRowCount; + protected final TypeDescription schema; + // the file included columns indexed by the file's column ids. + private final boolean[] fileIncluded; + private final long rowIndexStride; + private long rowInStripe = 0; + // position of the follow reader within the stripe + private long followRowInStripe = 0; + private int currentStripe = -1; + private long rowBaseInStripe = 0; + private long rowCountInStripe = 0; + private final BatchReader reader; + private final OrcIndex indexes; + // identifies the columns requiring row indexes + private final boolean[] rowIndexColsToRead; + private final SargApplier sargApp; + // an array about which row groups aren't skipped + private boolean[] includedRowGroups = null; + private final DataReader dataReader; + private final int maxDiskRangeChunkLimit; + private final StripePlanner planner; + // identifies the type of read, ALL(read everything), LEADERS(read only the filter columns) + private final TypeReader.ReadPhase startReadPhase; + // identifies that follow columns bytes must be read + private boolean needsFollowColumnsRead; + private final boolean noSelectedVector; + // identifies whether the file has bad bloom filters that we should not use. + private final boolean skipBloomFilters; + @Nullable private final FileIndexResult fileIndexResult; + static final String[] BAD_CPP_BLOOM_FILTER_VERSIONS = { + "1.6.0", "1.6.1", "1.6.2", "1.6.3", "1.6.4", "1.6.5", "1.6.6", "1.6.7", "1.6.8", "1.6.9", + "1.6.10", "1.6.11", "1.7.0" + }; + + /** + * Given a list of column names, find the given column and return the index. + * + * @param evolution the mapping from reader to file schema + * @param columnName the fully qualified column name to look for + * @return the file column number or -1 if the column wasn't found in the file schema + * @throws IllegalArgumentException if the column was not found in the reader schema + */ + static int findColumns(SchemaEvolution evolution, String columnName) { + TypeDescription fileColumn = findColumnType(evolution, columnName); + return fileColumn == null ? -1 : fileColumn.getId(); + } + + static TypeDescription findColumnType(SchemaEvolution evolution, String columnName) { + try { + TypeDescription readerColumn = + evolution + .getReaderBaseSchema() + .findSubtype(columnName, evolution.isSchemaEvolutionCaseAware); + return evolution.getFileType(readerColumn); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException( + "Filter could not find column with name: " + + columnName + + " on " + + evolution.getReaderBaseSchema(), + e); + } + } + + /** + * Given a column name such as 'a.b.c', this method returns the column 'a.b.c' if present in the + * file. In case 'a.b.c' is not found in file then it tries to look for 'a.b', then 'a'. If none + * are present then it shall return null. + * + * @param evolution the mapping from reader to file schema + * @param columnName the fully qualified column name to look for + * @return the file column type or null in case none of the branch columns are present in the + * file + * @throws IllegalArgumentException if the column was not found in the reader schema + */ + static TypeDescription findMostCommonColumn(SchemaEvolution evolution, String columnName) { + try { + TypeDescription readerColumn = + evolution + .getReaderBaseSchema() + .findSubtype(columnName, evolution.isSchemaEvolutionCaseAware); + TypeDescription fileColumn; + do { + fileColumn = evolution.getFileType(readerColumn); + if (fileColumn == null) { + readerColumn = readerColumn.getParent(); + } else { + return fileColumn; + } + } while (readerColumn != null); + return null; + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException( + "Filter could not find column with name: " + + columnName + + " on " + + evolution.getReaderBaseSchema(), + e); + } + } + + /** + * Find the mapping from predicate leaves to columns. + * + * @param sargLeaves the search argument that we need to map + * @param evolution the mapping from reader to file schema + * @return an array mapping the sarg leaves to concrete column numbers in the file + */ + public static int[] mapSargColumnsToOrcInternalColIdx( + List sargLeaves, SchemaEvolution evolution) { + int[] result = new int[sargLeaves.size()]; + for (int i = 0; i < sargLeaves.size(); ++i) { + int colNum = -1; + try { + String colName = sargLeaves.get(i).getColumnName(); + colNum = findColumns(evolution, colName); + } catch (IllegalArgumentException e) { + LOG.debug("{}", e.getMessage()); + } + result[i] = colNum; + } + return result; + } + + public RecordReaderImpl( + ReaderImpl fileReader, Reader.Options options, FileIndexResult fileIndexResult) + throws IOException { + this.fileIndexResult = fileIndexResult; + OrcFile.WriterVersion writerVersion = fileReader.getWriterVersion(); + SchemaEvolution evolution; + if (options.getSchema() == null) { + LOG.info("Reader schema not provided -- using file schema " + fileReader.getSchema()); + evolution = new SchemaEvolution(fileReader.getSchema(), null, options); + } else { + + // Now that we are creating a record reader for a file, validate that + // the schema to read is compatible with the file schema. + // + evolution = new SchemaEvolution(fileReader.getSchema(), options.getSchema(), options); + if (LOG.isDebugEnabled() && evolution.hasConversion()) { + LOG.debug( + "ORC file " + + fileReader.path.toString() + + " has data type conversion --\n" + + "reader schema: " + + options.getSchema().toString() + + "\n" + + "file schema: " + + fileReader.getSchema()); + } + } + this.noSelectedVector = !options.useSelected(); + LOG.debug("noSelectedVector={}", this.noSelectedVector); + this.schema = evolution.getReaderSchema(); + this.path = fileReader.path; + this.rowIndexStride = fileReader.rowIndexStride; + boolean ignoreNonUtf8BloomFilter = + OrcConf.IGNORE_NON_UTF8_BLOOM_FILTERS.getBoolean(fileReader.conf); + ReaderEncryption encryption = fileReader.getEncryption(); + this.fileIncluded = evolution.getFileIncluded(); + SearchArgument sarg = options.getSearchArgument(); + boolean[] rowIndexCols = new boolean[evolution.getFileIncluded().length]; + if (sarg != null && rowIndexStride > 0) { + sargApp = + new SargApplier( + sarg, + rowIndexStride, + evolution, + writerVersion, + fileReader.useUTCTimestamp, + fileReader.writerUsedProlepticGregorian(), + fileReader.options.getConvertToProlepticGregorian()); + sargApp.setRowIndexCols(rowIndexCols); + } else { + sargApp = null; + } + + long rows = 0; + long skippedRows = 0; + long offset = options.getOffset(); + long maxOffset = options.getMaxOffset(); + for (StripeInformation stripe : fileReader.getStripes()) { + long stripeStart = stripe.getOffset(); + if (offset > stripeStart) { + skippedRows += stripe.getNumberOfRows(); + } else if (stripeStart < maxOffset) { + this.stripes.add(stripe); + rows += stripe.getNumberOfRows(); + } + } + this.maxDiskRangeChunkLimit = + OrcConf.ORC_MAX_DISK_RANGE_CHUNK_LIMIT.getInt(fileReader.conf); + Boolean zeroCopy = options.getUseZeroCopy(); + if (zeroCopy == null) { + zeroCopy = OrcConf.USE_ZEROCOPY.getBoolean(fileReader.conf); + } + if (options.getDataReader() != null) { + this.dataReader = options.getDataReader().clone(); + } else { + InStream.StreamOptions unencryptedOptions = + InStream.options() + .withCodec(OrcCodecPool.getCodec(fileReader.getCompressionKind())) + .withBufferSize(fileReader.getCompressionSize()); + DataReaderProperties.Builder builder = + DataReaderProperties.builder() + .withCompression(unencryptedOptions) + .withFileSystemSupplier(fileReader.getFileSystemSupplier()) + .withPath(fileReader.path) + .withMaxDiskRangeChunkLimit(maxDiskRangeChunkLimit) + .withZeroCopy(zeroCopy) + .withMinSeekSize(options.minSeekSize()) + .withMinSeekSizeTolerance(options.minSeekSizeTolerance()); + FSDataInputStream file = fileReader.takeFile(); + if (file != null) { + builder.withFile(file); + } + this.dataReader = RecordReaderUtils.createDefaultDataReader(builder.build()); + } + firstRow = skippedRows; + totalRowCount = rows; + Boolean skipCorrupt = options.getSkipCorruptRecords(); + if (skipCorrupt == null) { + skipCorrupt = OrcConf.SKIP_CORRUPT_DATA.getBoolean(fileReader.conf); + } + + String[] filterCols = null; + Consumer filterCallBack = null; + String filePath = + options.allowPluginFilters() + ? fileReader.getFileSystem().makeQualified(fileReader.path).toString() + : null; + BatchFilter filter = + FilterFactory.createBatchFilter( + options, + evolution.getReaderBaseSchema(), + evolution.isSchemaEvolutionCaseAware(), + fileReader.getFileVersion(), + false, + filePath, + fileReader.conf); + if (filter != null) { + // If a filter is determined then use this + filterCallBack = filter; + filterCols = filter.getColumnNames(); + } + + // Map columnNames to ColumnIds + SortedSet filterColIds = new TreeSet<>(); + if (filterCols != null) { + for (String colName : filterCols) { + TypeDescription expandCol = findColumnType(evolution, colName); + // If the column is not present in the file then this can be ignored from read. + if (expandCol == null || expandCol.getId() == -1) { + // Add -1 to filter columns so that the NullTreeReader is invoked during the + // LEADERS phase + filterColIds.add(-1); + // Determine the common parent and include these + expandCol = findMostCommonColumn(evolution, colName); + } + while (expandCol != null && expandCol.getId() != -1) { + // classify the column and the parent branch as LEAD + filterColIds.add(expandCol.getId()); + rowIndexCols[expandCol.getId()] = true; + expandCol = expandCol.getParent(); + } + } + this.startReadPhase = TypeReader.ReadPhase.LEADERS; + LOG.debug( + "Using startReadPhase: {} with filter columns: {}", + startReadPhase, + filterColIds); + } else { + this.startReadPhase = TypeReader.ReadPhase.ALL; + } + + this.rowIndexColsToRead = ArrayUtils.contains(rowIndexCols, true) ? rowIndexCols : null; + TreeReaderFactory.ReaderContext readerContext = + new TreeReaderFactory.ReaderContext() + .setSchemaEvolution(evolution) + .setFilterCallback(filterColIds, filterCallBack) + .skipCorrupt(skipCorrupt) + .fileFormat(fileReader.getFileVersion()) + .useUTCTimestamp(fileReader.useUTCTimestamp) + .setProlepticGregorian( + fileReader.writerUsedProlepticGregorian(), + fileReader.options.getConvertToProlepticGregorian()) + .setEncryption(encryption); + reader = TreeReaderFactory.createRootReader(evolution.getReaderSchema(), readerContext); + skipBloomFilters = hasBadBloomFilters(fileReader.getFileTail().getFooter()); + + int columns = evolution.getFileSchema().getMaximumId() + 1; + indexes = + new OrcIndex( + new OrcProto.RowIndex[columns], + new OrcProto.Stream.Kind[columns], + new OrcProto.BloomFilterIndex[columns]); + + planner = + new StripePlanner( + evolution.getFileSchema(), + encryption, + dataReader, + writerVersion, + ignoreNonUtf8BloomFilter, + maxDiskRangeChunkLimit, + filterColIds); + + try { + advanceToNextRow(reader, 0L, true); + } catch (Exception e) { + // Try to close since this happens in constructor. + close(); + long stripeId = stripes.size() == 0 ? 0 : stripes.get(0).getStripeId(); + throw new IOException( + String.format("Problem opening stripe %d footer in %s.", stripeId, path), e); + } + } + + /** + * Check if the file has inconsistent bloom filters. We will skip using them in the following + * reads. + * + * @return true if it has. + */ + private boolean hasBadBloomFilters(OrcProto.Footer footer) { + // Only C++ writer in old releases could have bad bloom filters. + if (footer.getWriter() != 1) { + return false; + } + // 'softwareVersion' is added in 1.5.13, 1.6.11, and 1.7.0. + // 1.6.x releases before 1.6.11 won't have it. On the other side, the C++ writer + // supports writing bloom filters since 1.6.0. So files written by the C++ writer + // and with 'softwareVersion' unset would have bad bloom filters. + if (!footer.hasSoftwareVersion()) { + return true; + } + String fullVersion = footer.getSoftwareVersion(); + String version = fullVersion; + // Deal with snapshot versions, e.g. 1.6.12-SNAPSHOT. + if (fullVersion.contains("-")) { + version = fullVersion.substring(0, fullVersion.indexOf('-')); + } + for (String v : BAD_CPP_BLOOM_FILTER_VERSIONS) { + if (v.equals(version)) { + return true; + } + } + return false; + } + + /** An orc PositionProvider impl. */ + public static final class PositionProviderImpl implements PositionProvider { + private final OrcProto.RowIndexEntry entry; + private int index; + + public PositionProviderImpl(OrcProto.RowIndexEntry entry) { + this(entry, 0); + } + + public PositionProviderImpl(OrcProto.RowIndexEntry entry, int startPos) { + this.entry = entry; + this.index = startPos; + } + + @Override + public long getNext() { + return entry.getPositions(index++); + } + } + + /** An orc PositionProvider impl. */ + public static final class ZeroPositionProvider implements PositionProvider { + @Override + public long getNext() { + return 0; + } + } + + public OrcProto.StripeFooter readStripeFooter(StripeInformation stripe) throws IOException { + return dataReader.readStripeFooter(stripe); + } + + enum Location { + BEFORE, + MIN, + MIDDLE, + MAX, + AFTER + } + + static class ValueRange { + final Comparable lower; + final Comparable upper; + final boolean onlyLowerBound; + final boolean onlyUpperBound; + final boolean hasNulls; + final boolean hasValue; + final boolean comparable; + + ValueRange( + PredicateLeaf predicate, + T lower, + T upper, + boolean hasNulls, + boolean onlyLowerBound, + boolean onlyUpperBound, + boolean hasValue, + boolean comparable) { + PredicateLeaf.Type type = predicate.getType(); + this.lower = getBaseObjectForComparison(type, lower); + this.upper = getBaseObjectForComparison(type, upper); + this.hasNulls = hasNulls; + this.onlyLowerBound = onlyLowerBound; + this.onlyUpperBound = onlyUpperBound; + this.hasValue = hasValue; + this.comparable = comparable; + } + + ValueRange( + PredicateLeaf predicate, + T lower, + T upper, + boolean hasNulls, + boolean onlyLowerBound, + boolean onlyUpperBound) { + this( + predicate, + lower, + upper, + hasNulls, + onlyLowerBound, + onlyUpperBound, + lower != null, + lower != null); + } + + ValueRange(PredicateLeaf predicate, T lower, T upper, boolean hasNulls) { + this(predicate, lower, upper, hasNulls, false, false); + } + + /** + * A value range where the data is either missing or all null. + * + * @param predicate the predicate to test + * @param hasNulls whether there are nulls + */ + ValueRange(PredicateLeaf predicate, boolean hasNulls) { + this(predicate, null, null, hasNulls, false, false); + } + + boolean hasValues() { + return hasValue; + } + + /** + * Whether min or max is provided for comparison. + * + * @return is it comparable + */ + boolean isComparable() { + return hasValue && comparable; + } + + /** + * value range is invalid if the column statistics are non-existent. + * + * @see ColumnStatisticsImpl#isStatsExists() this method is similar to isStatsExists + * @return value range is valid or not + */ + boolean isValid() { + return hasValue || hasNulls; + } + + /** + * Given a point and min and max, determine if the point is before, at the min, in the + * middle, at the max, or after the range. + * + * @param point the point to test + * @return the location of the point + */ + Location compare(Comparable point) { + int minCompare = point.compareTo(lower); + if (minCompare < 0) { + return Location.BEFORE; + } else if (minCompare == 0) { + return onlyLowerBound ? Location.BEFORE : Location.MIN; + } + int maxCompare = point.compareTo(upper); + if (maxCompare > 0) { + return Location.AFTER; + } else if (maxCompare == 0) { + return onlyUpperBound ? Location.AFTER : Location.MAX; + } + return Location.MIDDLE; + } + + /** + * Is this range a single point? + * + * @return true if min == max + */ + boolean isSingleton() { + return lower != null && !onlyLowerBound && !onlyUpperBound && lower.equals(upper); + } + + /** + * Add the null option to the truth value, if the range includes nulls. + * + * @param value the original truth value + * @return the truth value extended with null if appropriate + */ + SearchArgument.TruthValue addNull(SearchArgument.TruthValue value) { + if (hasNulls) { + switch (value) { + case YES: + return SearchArgument.TruthValue.YES_NULL; + case NO: + return SearchArgument.TruthValue.NO_NULL; + case YES_NO: + return SearchArgument.TruthValue.YES_NO_NULL; + default: + return value; + } + } else { + return value; + } + } + } + + /** + * Get the maximum value out of an index entry. Includes option to specify if timestamp column + * stats values should be in UTC. + * + * @param index the index entry + * @param predicate the kind of predicate + * @param useUTCTimestamp use UTC for timestamps + * @return the object for the maximum value or null if there isn't one + */ + static ValueRange getValueRange( + ColumnStatistics index, PredicateLeaf predicate, boolean useUTCTimestamp) { + if (index.getNumberOfValues() == 0) { + return new ValueRange<>(predicate, index.hasNull()); + } else if (index instanceof IntegerColumnStatistics) { + IntegerColumnStatistics stats = (IntegerColumnStatistics) index; + Long min = stats.getMinimum(); + Long max = stats.getMaximum(); + return new ValueRange<>(predicate, min, max, stats.hasNull()); + } else if (index instanceof CollectionColumnStatistics) { + CollectionColumnStatistics stats = (CollectionColumnStatistics) index; + Long min = stats.getMinimumChildren(); + Long max = stats.getMaximumChildren(); + return new ValueRange<>(predicate, min, max, stats.hasNull()); + } else if (index instanceof DoubleColumnStatistics) { + DoubleColumnStatistics stats = (DoubleColumnStatistics) index; + Double min = stats.getMinimum(); + Double max = stats.getMaximum(); + return new ValueRange<>(predicate, min, max, stats.hasNull()); + } else if (index instanceof StringColumnStatistics) { + StringColumnStatistics stats = (StringColumnStatistics) index; + return new ValueRange<>( + predicate, + stats.getLowerBound(), + stats.getUpperBound(), + stats.hasNull(), + stats.getMinimum() == null, + stats.getMaximum() == null); + } else if (index instanceof DateColumnStatistics) { + DateColumnStatistics stats = (DateColumnStatistics) index; + ChronoLocalDate min = stats.getMinimumLocalDate(); + ChronoLocalDate max = stats.getMaximumLocalDate(); + return new ValueRange<>(predicate, min, max, stats.hasNull()); + } else if (index instanceof DecimalColumnStatistics) { + DecimalColumnStatistics stats = (DecimalColumnStatistics) index; + HiveDecimal min = stats.getMinimum(); + HiveDecimal max = stats.getMaximum(); + return new ValueRange<>(predicate, min, max, stats.hasNull()); + } else if (index instanceof TimestampColumnStatistics) { + TimestampColumnStatistics stats = (TimestampColumnStatistics) index; + Timestamp min = useUTCTimestamp ? stats.getMinimumUTC() : stats.getMinimum(); + Timestamp max = useUTCTimestamp ? stats.getMaximumUTC() : stats.getMaximum(); + return new ValueRange<>(predicate, min, max, stats.hasNull()); + } else if (index instanceof BooleanColumnStatistics) { + BooleanColumnStatistics stats = (BooleanColumnStatistics) index; + Boolean min = stats.getFalseCount() == 0; + Boolean max = stats.getTrueCount() != 0; + return new ValueRange<>(predicate, min, max, stats.hasNull()); + } else { + return new ValueRange( + predicate, null, null, index.hasNull(), false, false, true, false); + } + } + + /** + * Evaluate a predicate with respect to the statistics from the column that is referenced in the + * predicate. + * + * @param statsProto the statistics for the column mentioned in the predicate + * @param predicate the leaf predicate we need to evaluation + * @param bloomFilter the bloom filter + * @param writerVersion the version of software that wrote the file + * @param type what is the kind of this column + * @return the set of truth values that may be returned for the given predicate. + */ + static SearchArgument.TruthValue evaluatePredicateProto( + OrcProto.ColumnStatistics statsProto, + PredicateLeaf predicate, + OrcProto.Stream.Kind kind, + OrcProto.ColumnEncoding encoding, + OrcProto.BloomFilter bloomFilter, + OrcFile.WriterVersion writerVersion, + TypeDescription type) { + return evaluatePredicateProto( + statsProto, + predicate, + kind, + encoding, + bloomFilter, + writerVersion, + type, + true, + false); + } + + /** + * Evaluate a predicate with respect to the statistics from the column that is referenced in the + * predicate. Includes option to specify if timestamp column stats values should be in UTC and + * if the file writer used proleptic Gregorian calendar. + * + * @param statsProto the statistics for the column mentioned in the predicate + * @param predicate the leaf predicate we need to evaluation + * @param bloomFilter the bloom filter + * @param writerVersion the version of software that wrote the file + * @param type what is the kind of this column + * @param writerUsedProlepticGregorian file written using the proleptic Gregorian calendar + * @param useUTCTimestamp + * @return the set of truth values that may be returned for the given predicate. + */ + static SearchArgument.TruthValue evaluatePredicateProto( + OrcProto.ColumnStatistics statsProto, + PredicateLeaf predicate, + OrcProto.Stream.Kind kind, + OrcProto.ColumnEncoding encoding, + OrcProto.BloomFilter bloomFilter, + OrcFile.WriterVersion writerVersion, + TypeDescription type, + boolean writerUsedProlepticGregorian, + boolean useUTCTimestamp) { + ColumnStatistics cs = + ColumnStatisticsImpl.deserialize( + null, statsProto, writerUsedProlepticGregorian, true); + ValueRange range = getValueRange(cs, predicate, useUTCTimestamp); + + // files written before ORC-135 stores timestamp wrt to local timezone causing issues with + // PPD. + // disable PPD for timestamp for all old files + TypeDescription.Category category = type.getCategory(); + if (category == TypeDescription.Category.TIMESTAMP) { + if (!writerVersion.includes(OrcFile.WriterVersion.ORC_135)) { + LOG.debug( + "Not using predication pushdown on {} because it doesn't " + + "include ORC-135. Writer version: {}", + predicate.getColumnName(), + writerVersion); + return range.addNull(SearchArgument.TruthValue.YES_NO); + } + if (predicate.getType() != PredicateLeaf.Type.TIMESTAMP + && predicate.getType() != PredicateLeaf.Type.DATE + && predicate.getType() != PredicateLeaf.Type.STRING) { + return range.addNull(SearchArgument.TruthValue.YES_NO); + } + } else if (writerVersion == OrcFile.WriterVersion.ORC_135 + && category == TypeDescription.Category.DECIMAL + && type.getPrecision() <= TypeDescription.MAX_DECIMAL64_PRECISION) { + // ORC 1.5.0 to 1.5.5, which use WriterVersion.ORC_135, have broken + // min and max values for decimal64. See ORC-517. + LOG.debug( + "Not using predicate push down on {}, because the file doesn't" + + " include ORC-517. Writer version: {}", + predicate.getColumnName(), + writerVersion); + return SearchArgument.TruthValue.YES_NO_NULL; + } else if ((category == TypeDescription.Category.DOUBLE + || category == TypeDescription.Category.FLOAT) + && cs instanceof DoubleColumnStatistics) { + DoubleColumnStatistics dstas = (DoubleColumnStatistics) cs; + if (Double.isNaN(dstas.getSum())) { + LOG.debug( + "Not using predication pushdown on {} because stats contain NaN values", + predicate.getColumnName()); + return dstas.hasNull() + ? SearchArgument.TruthValue.YES_NO_NULL + : SearchArgument.TruthValue.YES_NO; + } + } + return evaluatePredicateRange( + predicate, + range, + BloomFilterIO.deserialize( + kind, encoding, writerVersion, type.getCategory(), bloomFilter), + useUTCTimestamp); + } + + /** + * Evaluate a predicate with respect to the statistics from the column that is referenced in the + * predicate. + * + * @param stats the statistics for the column mentioned in the predicate + * @param predicate the leaf predicate we need to evaluation + * @return the set of truth values that may be returned for the given predicate. + */ + public static SearchArgument.TruthValue evaluatePredicate( + ColumnStatistics stats, PredicateLeaf predicate, BloomFilter bloomFilter) { + return evaluatePredicate(stats, predicate, bloomFilter, false); + } + + /** + * Evaluate a predicate with respect to the statistics from the column that is referenced in the + * predicate. Includes option to specify if timestamp column stats values should be in UTC. + * + * @param stats the statistics for the column mentioned in the predicate + * @param predicate the leaf predicate we need to evaluation + * @param bloomFilter + * @param useUTCTimestamp + * @return the set of truth values that may be returned for the given predicate. + */ + public static SearchArgument.TruthValue evaluatePredicate( + ColumnStatistics stats, + PredicateLeaf predicate, + BloomFilter bloomFilter, + boolean useUTCTimestamp) { + ValueRange range = getValueRange(stats, predicate, useUTCTimestamp); + + return evaluatePredicateRange(predicate, range, bloomFilter, useUTCTimestamp); + } + + static SearchArgument.TruthValue evaluatePredicateRange( + PredicateLeaf predicate, + ValueRange range, + BloomFilter bloomFilter, + boolean useUTCTimestamp) { + if (!range.isValid()) { + return SearchArgument.TruthValue.YES_NO_NULL; + } + + // if we didn't have any values, everything must have been null + if (!range.hasValues()) { + if (predicate.getOperator() == PredicateLeaf.Operator.IS_NULL) { + return SearchArgument.TruthValue.YES; + } else { + return SearchArgument.TruthValue.NULL; + } + } else if (!range.isComparable()) { + return range.hasNulls + ? SearchArgument.TruthValue.YES_NO_NULL + : SearchArgument.TruthValue.YES_NO; + } + + SearchArgument.TruthValue result; + Comparable baseObj = (Comparable) predicate.getLiteral(); + // Predicate object and stats objects are converted to the type of the predicate object. + Comparable predObj = getBaseObjectForComparison(predicate.getType(), baseObj); + + result = evaluatePredicateMinMax(predicate, predObj, range); + if (shouldEvaluateBloomFilter(predicate, result, bloomFilter)) { + return evaluatePredicateBloomFilter( + predicate, predObj, bloomFilter, range.hasNulls, useUTCTimestamp); + } else { + return result; + } + } + + private static boolean shouldEvaluateBloomFilter( + PredicateLeaf predicate, SearchArgument.TruthValue result, BloomFilter bloomFilter) { + // evaluate bloom filter only when + // 1) Bloom filter is available + // 2) Min/Max evaluation yield YES or MAYBE + // 3) Predicate is EQUALS or IN list + return bloomFilter != null + && result != SearchArgument.TruthValue.NO_NULL + && result != SearchArgument.TruthValue.NO + && (predicate.getOperator().equals(PredicateLeaf.Operator.EQUALS) + || predicate.getOperator().equals(PredicateLeaf.Operator.NULL_SAFE_EQUALS) + || predicate.getOperator().equals(PredicateLeaf.Operator.IN)); + } + + private static SearchArgument.TruthValue evaluatePredicateMinMax( + PredicateLeaf predicate, Comparable predObj, ValueRange range) { + Location loc; + + switch (predicate.getOperator()) { + case NULL_SAFE_EQUALS: + loc = range.compare(predObj); + if (loc == Location.BEFORE || loc == Location.AFTER) { + return SearchArgument.TruthValue.NO; + } else { + return SearchArgument.TruthValue.YES_NO; + } + case EQUALS: + loc = range.compare(predObj); + if (range.isSingleton() && loc == Location.MIN) { + return range.addNull(SearchArgument.TruthValue.YES); + } else if (loc == Location.BEFORE || loc == Location.AFTER) { + return range.addNull(SearchArgument.TruthValue.NO); + } else { + return range.addNull(SearchArgument.TruthValue.YES_NO); + } + case LESS_THAN: + loc = range.compare(predObj); + if (loc == Location.AFTER) { + return range.addNull(SearchArgument.TruthValue.YES); + } else if (loc == Location.BEFORE || loc == Location.MIN) { + return range.addNull(SearchArgument.TruthValue.NO); + } else { + return range.addNull(SearchArgument.TruthValue.YES_NO); + } + case LESS_THAN_EQUALS: + loc = range.compare(predObj); + if (loc == Location.AFTER + || loc == Location.MAX + || (loc == Location.MIN && range.isSingleton())) { + return range.addNull(SearchArgument.TruthValue.YES); + } else if (loc == Location.BEFORE) { + return range.addNull(SearchArgument.TruthValue.NO); + } else { + return range.addNull(SearchArgument.TruthValue.YES_NO); + } + case IN: + if (range.isSingleton()) { + // for a single value, look through to see if that value is in the + // set + for (Object arg : predicate.getLiteralList()) { + predObj = getBaseObjectForComparison(predicate.getType(), (Comparable) arg); + if (range.compare(predObj) == Location.MIN) { + return range.addNull(SearchArgument.TruthValue.YES); + } + } + return range.addNull(SearchArgument.TruthValue.NO); + } else { + // are all of the values outside of the range? + for (Object arg : predicate.getLiteralList()) { + predObj = getBaseObjectForComparison(predicate.getType(), (Comparable) arg); + loc = range.compare(predObj); + if (loc == Location.MIN || loc == Location.MIDDLE || loc == Location.MAX) { + return range.addNull(SearchArgument.TruthValue.YES_NO); + } + } + return range.addNull(SearchArgument.TruthValue.NO); + } + case BETWEEN: + List args = predicate.getLiteralList(); + if (args == null || args.isEmpty()) { + return range.addNull(SearchArgument.TruthValue.YES_NO); + } + Comparable predObj1 = + getBaseObjectForComparison(predicate.getType(), (Comparable) args.get(0)); + + loc = range.compare(predObj1); + if (loc == Location.BEFORE || loc == Location.MIN) { + Comparable predObj2 = + getBaseObjectForComparison( + predicate.getType(), (Comparable) args.get(1)); + Location loc2 = range.compare(predObj2); + if (loc2 == Location.AFTER || loc2 == Location.MAX) { + return range.addNull(SearchArgument.TruthValue.YES); + } else if (loc2 == Location.BEFORE) { + return range.addNull(SearchArgument.TruthValue.NO); + } else { + return range.addNull(SearchArgument.TruthValue.YES_NO); + } + } else if (loc == Location.AFTER) { + return range.addNull(SearchArgument.TruthValue.NO); + } else { + return range.addNull(SearchArgument.TruthValue.YES_NO); + } + case IS_NULL: + // min = null condition above handles the all-nulls YES case + return range.hasNulls + ? SearchArgument.TruthValue.YES_NO + : SearchArgument.TruthValue.NO; + default: + return range.addNull(SearchArgument.TruthValue.YES_NO); + } + } + + private static SearchArgument.TruthValue evaluatePredicateBloomFilter( + PredicateLeaf predicate, + final Object predObj, + BloomFilter bloomFilter, + boolean hasNull, + boolean useUTCTimestamp) { + switch (predicate.getOperator()) { + case NULL_SAFE_EQUALS: + // null safe equals does not return *_NULL variant. So set hasNull to false + return checkInBloomFilter(bloomFilter, predObj, false, useUTCTimestamp); + case EQUALS: + return checkInBloomFilter(bloomFilter, predObj, hasNull, useUTCTimestamp); + case IN: + for (Object arg : predicate.getLiteralList()) { + // if atleast one value in IN list exist in bloom filter, qualify the row + // group/stripe + Object predObjItem = + getBaseObjectForComparison(predicate.getType(), (Comparable) arg); + SearchArgument.TruthValue result = + checkInBloomFilter(bloomFilter, predObjItem, hasNull, useUTCTimestamp); + if (result == SearchArgument.TruthValue.YES_NO_NULL + || result == SearchArgument.TruthValue.YES_NO) { + return result; + } + } + return hasNull ? SearchArgument.TruthValue.NO_NULL : SearchArgument.TruthValue.NO; + default: + return hasNull + ? SearchArgument.TruthValue.YES_NO_NULL + : SearchArgument.TruthValue.YES_NO; + } + } + + private static SearchArgument.TruthValue checkInBloomFilter( + BloomFilter bf, Object predObj, boolean hasNull, boolean useUTCTimestamp) { + SearchArgument.TruthValue result = + hasNull ? SearchArgument.TruthValue.NO_NULL : SearchArgument.TruthValue.NO; + + if (predObj instanceof Long) { + if (bf.testLong((Long) predObj)) { + result = SearchArgument.TruthValue.YES_NO_NULL; + } + } else if (predObj instanceof Double) { + if (bf.testDouble((Double) predObj)) { + result = SearchArgument.TruthValue.YES_NO_NULL; + } + } else if (predObj instanceof String + || predObj instanceof Text + || predObj instanceof HiveDecimalWritable + || predObj instanceof BigDecimal) { + if (bf.testString(predObj.toString())) { + result = SearchArgument.TruthValue.YES_NO_NULL; + } + } else if (predObj instanceof Timestamp) { + if (useUTCTimestamp) { + if (bf.testLong(((Timestamp) predObj).getTime())) { + result = SearchArgument.TruthValue.YES_NO_NULL; + } + } else { + if (bf.testLong( + SerializationUtils.convertToUtc( + TimeZone.getDefault(), ((Timestamp) predObj).getTime()))) { + result = SearchArgument.TruthValue.YES_NO_NULL; + } + } + } else if (predObj instanceof ChronoLocalDate) { + if (bf.testLong(((ChronoLocalDate) predObj).toEpochDay())) { + result = SearchArgument.TruthValue.YES_NO_NULL; + } + } else { + // if the predicate object is null and if hasNull says there are no nulls then return NO + if (predObj == null && !hasNull) { + result = SearchArgument.TruthValue.NO; + } else { + result = SearchArgument.TruthValue.YES_NO_NULL; + } + } + + if (result == SearchArgument.TruthValue.YES_NO_NULL && !hasNull) { + result = SearchArgument.TruthValue.YES_NO; + } + + LOG.debug("Bloom filter evaluation: {}", result); + + return result; + } + + /** An exception for when we can't cast things appropriately. */ + static class SargCastException extends IllegalArgumentException { + + SargCastException(String string) { + super(string); + } + } + + private static Comparable getBaseObjectForComparison(PredicateLeaf.Type type, Comparable obj) { + if (obj == null) { + return null; + } + switch (type) { + case BOOLEAN: + if (obj instanceof Boolean) { + return obj; + } else { + // will only be true if the string conversion yields "true", all other values + // are + // considered false + return Boolean.valueOf(obj.toString()); + } + case DATE: + if (obj instanceof ChronoLocalDate) { + return obj; + } else if (obj instanceof java.sql.Date) { + return ((java.sql.Date) obj).toLocalDate(); + } else if (obj instanceof Date) { + return LocalDateTime.ofInstant(((Date) obj).toInstant(), ZoneOffset.UTC) + .toLocalDate(); + } else if (obj instanceof String) { + return LocalDate.parse((String) obj); + } else if (obj instanceof Timestamp) { + return ((Timestamp) obj).toLocalDateTime().toLocalDate(); + } + // always string, but prevent the comparison to numbers (are they + // days/seconds/milliseconds?) + break; + case DECIMAL: + if (obj instanceof Boolean) { + return new HiveDecimalWritable( + (Boolean) obj ? HiveDecimal.ONE : HiveDecimal.ZERO); + } else if (obj instanceof Integer) { + return new HiveDecimalWritable((Integer) obj); + } else if (obj instanceof Long) { + return new HiveDecimalWritable(((Long) obj)); + } else if (obj instanceof Float || obj instanceof Double || obj instanceof String) { + return new HiveDecimalWritable(obj.toString()); + } else if (obj instanceof BigDecimal) { + return new HiveDecimalWritable(HiveDecimal.create((BigDecimal) obj)); + } else if (obj instanceof HiveDecimal) { + return new HiveDecimalWritable((HiveDecimal) obj); + } else if (obj instanceof HiveDecimalWritable) { + return obj; + } else if (obj instanceof Timestamp) { + return new HiveDecimalWritable( + Double.toString(TimestampUtils.getDouble((Timestamp) obj))); + } + break; + case FLOAT: + if (obj instanceof Number) { + // widening conversion + return ((Number) obj).doubleValue(); + } else if (obj instanceof HiveDecimal) { + return ((HiveDecimal) obj).doubleValue(); + } else if (obj instanceof String) { + return Double.valueOf(obj.toString()); + } else if (obj instanceof Timestamp) { + return TimestampUtils.getDouble((Timestamp) obj); + } + break; + case LONG: + if (obj instanceof Number) { + // widening conversion + return ((Number) obj).longValue(); + } else if (obj instanceof HiveDecimal) { + return ((HiveDecimal) obj).longValue(); + } else if (obj instanceof String) { + return Long.valueOf(obj.toString()); + } + break; + case STRING: + if (obj instanceof ChronoLocalDate) { + ChronoLocalDate date = (ChronoLocalDate) obj; + return date.format( + DateTimeFormatter.ISO_LOCAL_DATE.withChronology(date.getChronology())); + } + return (obj.toString()); + case TIMESTAMP: + if (obj instanceof Timestamp) { + return obj; + } else if (obj instanceof Integer) { + return new Timestamp(((Number) obj).longValue()); + } else if (obj instanceof Float) { + return TimestampUtils.doubleToTimestamp(((Float) obj).doubleValue()); + } else if (obj instanceof Double) { + return TimestampUtils.doubleToTimestamp((Double) obj); + } else if (obj instanceof HiveDecimal) { + return TimestampUtils.decimalToTimestamp((HiveDecimal) obj); + } else if (obj instanceof HiveDecimalWritable) { + return TimestampUtils.decimalToTimestamp( + ((HiveDecimalWritable) obj).getHiveDecimal()); + } else if (obj instanceof Date) { + return new Timestamp(((Date) obj).getTime()); + } else if (obj instanceof ChronoLocalDate) { + return new Timestamp( + ((ChronoLocalDate) obj) + .atTime(LocalTime.MIDNIGHT) + .toInstant(ZoneOffset.UTC) + .getEpochSecond() + * 1000L); + } + // float/double conversion to timestamp is interpreted as seconds whereas integer + // conversion + // to timestamp is interpreted as milliseconds by default. The integer to timestamp + // casting + // is also config driven. The filter operator changes its promotion based on config: + // "int.timestamp.conversion.in.seconds". Disable PPD for integer cases. + break; + default: + break; + } + + throw new SargCastException( + String.format( + "ORC SARGS could not convert from %s to %s", + obj.getClass().getSimpleName(), type)); + } + + /** search argument applier. */ + public static class SargApplier { + public static final boolean[] READ_ALL_RGS = null; + public static final boolean[] READ_NO_RGS = new boolean[0]; + + private final OrcFile.WriterVersion writerVersion; + private final SearchArgument sarg; + private final List sargLeaves; + private final int[] filterColumns; + private final long rowIndexStride; + // same as the above array, but indices are set to true + private final SchemaEvolution evolution; + private final long[] exceptionCount; + private final boolean useUTCTimestamp; + private final boolean writerUsedProlepticGregorian; + private final boolean convertToProlepticGregorian; + + /** + * @deprecated Use the constructor having full parameters. This exists for backward + * compatibility. + */ + public SargApplier( + SearchArgument sarg, + long rowIndexStride, + SchemaEvolution evolution, + OrcFile.WriterVersion writerVersion, + boolean useUTCTimestamp) { + this(sarg, rowIndexStride, evolution, writerVersion, useUTCTimestamp, false, false); + } + + public SargApplier( + SearchArgument sarg, + long rowIndexStride, + SchemaEvolution evolution, + OrcFile.WriterVersion writerVersion, + boolean useUTCTimestamp, + boolean writerUsedProlepticGregorian, + boolean convertToProlepticGregorian) { + this.writerVersion = writerVersion; + this.sarg = sarg; + sargLeaves = sarg.getLeaves(); + this.writerUsedProlepticGregorian = writerUsedProlepticGregorian; + this.convertToProlepticGregorian = convertToProlepticGregorian; + filterColumns = mapSargColumnsToOrcInternalColIdx(sargLeaves, evolution); + this.rowIndexStride = rowIndexStride; + this.evolution = evolution; + exceptionCount = new long[sargLeaves.size()]; + this.useUTCTimestamp = useUTCTimestamp; + } + + public void setRowIndexCols(boolean[] rowIndexCols) { + // included will not be null, row options will fill the array with + // trues if null + for (int i : filterColumns) { + // filter columns may have -1 as index which could be partition + // column in SARG. + if (i > 0) { + rowIndexCols[i] = true; + } + } + } + + /** + * Pick the row groups that we need to load from the current stripe. + * + * @return an array with a boolean for each row group or null if all of the row groups must + * be read. + * @throws IOException + */ + public boolean[] pickRowGroups( + StripeInformation stripe, + OrcProto.RowIndex[] indexes, + OrcProto.Stream.Kind[] bloomFilterKinds, + List encodings, + OrcProto.BloomFilterIndex[] bloomFilterIndices, + boolean returnNone, + long rowBaseInStripe, + FileIndexResult fileIndexResult) + throws IOException { + long rowsInStripe = stripe.getNumberOfRows(); + int groupsInStripe = (int) ((rowsInStripe + rowIndexStride - 1) / rowIndexStride); + boolean[] result = new boolean[groupsInStripe]; // TODO: avoid alloc? + SearchArgument.TruthValue[] leafValues = + new SearchArgument.TruthValue[sargLeaves.size()]; + boolean hasSelected = false; + boolean hasSkipped = false; + SearchArgument.TruthValue[] exceptionAnswer = + new SearchArgument.TruthValue[leafValues.length]; + RoaringBitmap32 bitmap = null; + if (fileIndexResult instanceof BitmapIndexResult) { + bitmap = ((BitmapIndexResult) fileIndexResult).get(); + } + for (int rowGroup = 0; rowGroup < result.length; ++rowGroup) { + for (int pred = 0; pred < leafValues.length; ++pred) { + int columnIx = filterColumns[pred]; + if (columnIx == -1) { + // the column is a virtual column + leafValues[pred] = SearchArgument.TruthValue.YES_NO_NULL; + } else if (exceptionAnswer[pred] != null) { + leafValues[pred] = exceptionAnswer[pred]; + } else { + if (indexes[columnIx] == null) { + LOG.warn("Index is not populated for " + columnIx); + return READ_ALL_RGS; + } + OrcProto.RowIndexEntry entry = indexes[columnIx].getEntry(rowGroup); + if (entry == null) { + throw new AssertionError( + "RG is not populated for " + columnIx + " rg " + rowGroup); + } + OrcProto.ColumnStatistics stats = EMPTY_COLUMN_STATISTICS; + if (entry.hasStatistics()) { + stats = entry.getStatistics(); + } + OrcProto.BloomFilter bf = null; + OrcProto.Stream.Kind bfk = null; + if (bloomFilterIndices != null && bloomFilterIndices[columnIx] != null) { + bfk = bloomFilterKinds[columnIx]; + bf = bloomFilterIndices[columnIx].getBloomFilter(rowGroup); + } + if (evolution != null && evolution.isPPDSafeConversion(columnIx)) { + PredicateLeaf predicate = sargLeaves.get(pred); + try { + leafValues[pred] = + evaluatePredicateProto( + stats, + predicate, + bfk, + encodings.get(columnIx), + bf, + writerVersion, + evolution.getFileSchema().findSubtype(columnIx), + writerUsedProlepticGregorian, + useUTCTimestamp); + } catch (Exception e) { + exceptionCount[pred] += 1; + if (e instanceof SargCastException) { + LOG.info( + "Skipping ORC PPD - " + + e.getMessage() + + " on " + + predicate); + } else { + final String reason = + e.getClass().getSimpleName() + + " when evaluating predicate." + + " Skipping ORC PPD." + + " Stats: " + + stats + + " Predicate: " + + predicate; + LOG.warn(reason, e); + } + boolean hasNoNull = stats.hasHasNull() && !stats.getHasNull(); + if (predicate + .getOperator() + .equals(PredicateLeaf.Operator.NULL_SAFE_EQUALS) + || hasNoNull) { + exceptionAnswer[pred] = SearchArgument.TruthValue.YES_NO; + } else { + exceptionAnswer[pred] = SearchArgument.TruthValue.YES_NO_NULL; + } + leafValues[pred] = exceptionAnswer[pred]; + } + } else { + leafValues[pred] = SearchArgument.TruthValue.YES_NO_NULL; + } + if (LOG.isTraceEnabled()) { + LOG.trace("Stats = " + stats); + LOG.trace( + "Setting " + sargLeaves.get(pred) + " to " + leafValues[pred]); + } + } + } + result[rowGroup] = sarg.evaluate(leafValues).isNeeded(); + if (bitmap != null) { + long firstRow = rowBaseInStripe + rowIndexStride * rowGroup; + long lastRow = Math.min(firstRow + rowIndexStride, firstRow + rowsInStripe); + result[rowGroup] &= bitmap.rangeCardinality(firstRow, lastRow) > 0; + } + hasSelected = hasSelected || result[rowGroup]; + hasSkipped = hasSkipped || (!result[rowGroup]); + if (LOG.isDebugEnabled()) { + LOG.debug( + "Row group " + + (rowIndexStride * rowGroup) + + " to " + + (rowIndexStride * (rowGroup + 1) - 1) + + " is " + + (result[rowGroup] ? "" : "not ") + + "included."); + } + } + + return hasSkipped + ? ((hasSelected || !returnNone) ? result : READ_NO_RGS) + : READ_ALL_RGS; + } + + /** + * Get the count of exceptions for testing. + * + * @return + */ + long[] getExceptionCount() { + return exceptionCount; + } + } + + /** + * Pick the row groups that we need to load from the current stripe. + * + * @return an array with a boolean for each row group or null if all of the row groups must be + * read. + * @throws IOException + */ + protected boolean[] pickRowGroups() throws IOException { + // Read the Row Indicies if required + if (rowIndexColsToRead != null) { + readCurrentStripeRowIndex(); + } + + // In the absence of SArg all rows groups should be included + if (sargApp == null) { + return null; + } + return sargApp.pickRowGroups( + stripes.get(currentStripe), + indexes.getRowGroupIndex(), + skipBloomFilters ? null : indexes.getBloomFilterKinds(), + stripeFooter.getColumnsList(), + skipBloomFilters ? null : indexes.getBloomFilterIndex(), + false, + rowBaseInStripe, + fileIndexResult); + } + + private void clearStreams() { + planner.clearStreams(); + } + + /** + * Read the current stripe into memory. + * + * @throws IOException + */ + private void readStripe() throws IOException { + StripeInformation stripe = beginReadStripe(); + planner.parseStripe(stripe, fileIncluded); + includedRowGroups = pickRowGroups(); + + // move forward to the first unskipped row + if (includedRowGroups != null) { + while (rowInStripe < rowCountInStripe + && !includedRowGroups[(int) (rowInStripe / rowIndexStride)]) { + rowInStripe = Math.min(rowCountInStripe, rowInStripe + rowIndexStride); + } + } + + // if we haven't skipped the whole stripe, read the data + if (rowInStripe < rowCountInStripe) { + planner.readData(indexes, includedRowGroups, false, startReadPhase); + reader.startStripe(planner, startReadPhase); + needsFollowColumnsRead = true; + // if we skipped the first row group, move the pointers forward + if (rowInStripe != 0) { + seekToRowEntry(reader, (int) (rowInStripe / rowIndexStride), startReadPhase); + } + } + } + + private StripeInformation beginReadStripe() throws IOException { + StripeInformation stripe = stripes.get(currentStripe); + stripeFooter = readStripeFooter(stripe); + clearStreams(); + // setup the position in the stripe + rowCountInStripe = stripe.getNumberOfRows(); + rowInStripe = 0; + followRowInStripe = 0; + rowBaseInStripe = 0; + for (int i = 0; i < currentStripe; ++i) { + rowBaseInStripe += stripes.get(i).getNumberOfRows(); + } + // reset all of the indexes + OrcProto.RowIndex[] rowIndex = indexes.getRowGroupIndex(); + for (int i = 0; i < rowIndex.length; ++i) { + rowIndex[i] = null; + } + return stripe; + } + + /** + * Read the next stripe until we find a row that we don't skip. + * + * @throws IOException + */ + private void advanceStripe() throws IOException { + rowInStripe = rowCountInStripe; + while (rowInStripe >= rowCountInStripe && currentStripe < stripes.size() - 1) { + currentStripe += 1; + readStripe(); + } + } + + /** + * Determine the RowGroup based on the supplied row id. + * + * @param rowIdx Row for which the row group is being determined + * @return Id of the RowGroup that the row belongs to + */ + private int computeRGIdx(long rowIdx) { + return rowIndexStride == 0 ? 0 : (int) (rowIdx / rowIndexStride); + } + + /** + * Skip over rows that we aren't selecting, so that the next row is one that we will read. + * + * @param nextRow the row we want to go to + * @throws IOException + */ + private boolean advanceToNextRow(BatchReader reader, long nextRow, boolean canAdvanceStripe) + throws IOException { + long nextRowInStripe = nextRow - rowBaseInStripe; + // check for row skipping + if (rowIndexStride != 0 + && includedRowGroups != null + && nextRowInStripe < rowCountInStripe) { + int rowGroup = computeRGIdx(nextRowInStripe); + if (!includedRowGroups[rowGroup]) { + while (rowGroup < includedRowGroups.length && !includedRowGroups[rowGroup]) { + rowGroup += 1; + } + if (rowGroup >= includedRowGroups.length) { + if (canAdvanceStripe) { + advanceStripe(); + } + return canAdvanceStripe; + } + nextRowInStripe = Math.min(rowCountInStripe, rowGroup * rowIndexStride); + } + } + if (nextRowInStripe >= rowCountInStripe) { + if (canAdvanceStripe) { + advanceStripe(); + } + return canAdvanceStripe; + } + if (nextRowInStripe != rowInStripe) { + if (rowIndexStride != 0) { + int rowGroup = (int) (nextRowInStripe / rowIndexStride); + seekToRowEntry(reader, rowGroup, startReadPhase); + reader.skipRows(nextRowInStripe - rowGroup * rowIndexStride, startReadPhase); + } else { + reader.skipRows(nextRowInStripe - rowInStripe, startReadPhase); + } + rowInStripe = nextRowInStripe; + } + return true; + } + + @Override + public boolean nextBatch(VectorizedRowBatch batch) throws IOException { + try { + int batchSize; + // do...while is required to handle the case where the filter eliminates all rows in the + // batch, we never return an empty batch unless the file is exhausted + do { + if (rowInStripe >= rowCountInStripe) { + currentStripe += 1; + if (currentStripe >= stripes.size()) { + batch.size = 0; + return false; + } + // Read stripe in Memory + readStripe(); + followRowInStripe = rowInStripe; + } + + batchSize = computeBatchSize(batch.getMaxSize()); + reader.setVectorColumnCount(batch.getDataColumnCount()); + reader.nextBatch(batch, batchSize, startReadPhase); + if (startReadPhase == TypeReader.ReadPhase.LEADERS && batch.size > 0) { + // At least 1 row has been selected and as a result we read the follow columns + // into the + // row batch + reader.nextBatch( + batch, batchSize, prepareFollowReaders(rowInStripe, followRowInStripe)); + followRowInStripe = rowInStripe + batchSize; + } + rowInStripe += batchSize; + advanceToNextRow(reader, rowInStripe + rowBaseInStripe, true); + // batch.size can be modified by filter so only batchSize can tell if we actually + // read rows + } while (batchSize != 0 && batch.size == 0); + + if (noSelectedVector) { + // In case selected vector is not supported we leave the size to be read size. In + // this case + // the non filter columns might be read selectively, however the filter after the + // reader + // should eliminate rows that don't match predicate conditions + batch.size = batchSize; + batch.selectedInUse = false; + } + + return batchSize != 0; + } catch (IOException e) { + // Rethrow exception with file name in log message + throw new IOException("Error reading file: " + path, e); + } + } + + /** + * This method prepares the non-filter column readers for next batch. This involves the + * following 1. Determine position 2. Perform IO if required 3. Position the non-filter readers + * + *

    This method is repositioning the non-filter columns and as such this method shall never + * have to deal with navigating the stripe forward or skipping row groups, all of this should + * have already taken place based on the filter columns. + * + * @param toFollowRow The rowIdx identifies the required row position within the stripe for + * follow read + * @param fromFollowRow Indicates the current position of the follow read, exclusive + * @return the read phase for reading non-filter columns, this shall be FOLLOWERS_AND_PARENTS in + * case of a seek otherwise will be FOLLOWERS + */ + private TypeReader.ReadPhase prepareFollowReaders(long toFollowRow, long fromFollowRow) + throws IOException { + // 1. Determine the required row group and skip rows needed from the RG start + int needRG = computeRGIdx(toFollowRow); + // The current row is not yet read so we -1 to compute the previously read row group + int readRG = computeRGIdx(fromFollowRow - 1); + long skipRows; + if (needRG == readRG && toFollowRow >= fromFollowRow) { + // In case we are skipping forward within the same row group, we compute skip rows from + // the + // current position + skipRows = toFollowRow - fromFollowRow; + } else { + // In all other cases including seeking backwards, we compute the skip rows from the + // start of + // the required row group + skipRows = toFollowRow - (needRG * rowIndexStride); + } + + // 2. Plan the row group idx for the non-filter columns if this has not already taken place + if (needsFollowColumnsRead) { + needsFollowColumnsRead = false; + planner.readFollowData(indexes, includedRowGroups, needRG, false); + reader.startStripe(planner, TypeReader.ReadPhase.FOLLOWERS); + } + + // 3. Position the non-filter readers to the required RG and skipRows + TypeReader.ReadPhase result = TypeReader.ReadPhase.FOLLOWERS; + if (needRG != readRG || toFollowRow < fromFollowRow) { + // When having to change a row group or in case of back navigation, seek both the filter + // parents and non-filter. This will re-position the parents present vector. This is + // needed + // to determine the number of non-null values to skip on the non-filter columns. + seekToRowEntry(reader, needRG, TypeReader.ReadPhase.FOLLOWERS_AND_PARENTS); + // skip rows on both the filter parents and non-filter as both have been positioned in + // the + // previous step + reader.skipRows(skipRows, TypeReader.ReadPhase.FOLLOWERS_AND_PARENTS); + result = TypeReader.ReadPhase.FOLLOWERS_AND_PARENTS; + } else if (skipRows > 0) { + // in case we are only skipping within the row group, position the filter parents back + // to the + // position of the follow. This is required to determine the non-null values to skip on + // the + // non-filter columns. + seekToRowEntry(reader, readRG, TypeReader.ReadPhase.LEADER_PARENTS); + reader.skipRows( + fromFollowRow - (readRG * rowIndexStride), TypeReader.ReadPhase.LEADER_PARENTS); + // Move both the filter parents and non-filter forward, this will compute the correct + // non-null skips on follow children + reader.skipRows(skipRows, TypeReader.ReadPhase.FOLLOWERS_AND_PARENTS); + result = TypeReader.ReadPhase.FOLLOWERS_AND_PARENTS; + } + // Identifies the read level that should be performed for the read + // FOLLOWERS_WITH_PARENTS indicates repositioning identifying both non-filter and filter + // parents + // FOLLOWERS indicates read only of the non-filter level without the parents, which is used + // during + // contiguous read. During a contiguous read no skips are needed and the non-null + // information of + // the parent is available in the column vector for use during non-filter read + return result; + } + + private int computeBatchSize(long targetBatchSize) { + final int batchSize; + // In case of PPD, batch size should be aware of row group boundaries. If only a subset of + // row + // groups are selected then marker position is set to the end of range (subset of row groups + // within strip). Batch size computed out of marker position makes sure that batch size is + // aware of row group boundary and will not cause overflow when reading rows + // illustration of this case is here https://issues.apache.org/jira/browse/HIVE-6287 + if (rowIndexStride != 0 + && (includedRowGroups != null || startReadPhase != TypeReader.ReadPhase.ALL) + && rowInStripe < rowCountInStripe) { + int startRowGroup = (int) (rowInStripe / rowIndexStride); + if (includedRowGroups != null && !includedRowGroups[startRowGroup]) { + while (startRowGroup < includedRowGroups.length + && !includedRowGroups[startRowGroup]) { + startRowGroup += 1; + } + } + + int endRowGroup = startRowGroup; + // We force row group boundaries when dealing with filters. We adjust the end row group + // to + // be the next row group even if more than one are possible selections. + if (includedRowGroups != null && startReadPhase == TypeReader.ReadPhase.ALL) { + while (endRowGroup < includedRowGroups.length && includedRowGroups[endRowGroup]) { + endRowGroup += 1; + } + } else { + endRowGroup += 1; + } + + final long markerPosition = Math.min((endRowGroup * rowIndexStride), rowCountInStripe); + batchSize = (int) Math.min(targetBatchSize, (markerPosition - rowInStripe)); + + if (isLogDebugEnabled && batchSize < targetBatchSize) { + LOG.debug("markerPosition: " + markerPosition + " batchSize: " + batchSize); + } + } else { + batchSize = (int) Math.min(targetBatchSize, (rowCountInStripe - rowInStripe)); + } + return batchSize; + } + + @Override + public void close() throws IOException { + clearStreams(); + dataReader.close(); + } + + @Override + public long getRowNumber() { + return rowInStripe + rowBaseInStripe + firstRow; + } + + /** + * Return the fraction of rows that have been read from the selected. section of the file + * + * @return fraction between 0.0 and 1.0 of rows consumed + */ + @Override + public float getProgress() { + return ((float) rowBaseInStripe + rowInStripe) / totalRowCount; + } + + private int findStripe(long rowNumber) { + for (int i = 0; i < stripes.size(); i++) { + StripeInformation stripe = stripes.get(i); + if (stripe.getNumberOfRows() > rowNumber) { + return i; + } + rowNumber -= stripe.getNumberOfRows(); + } + throw new IllegalArgumentException("Seek after the end of reader range"); + } + + private void readCurrentStripeRowIndex() throws IOException { + planner.readRowIndex(rowIndexColsToRead, indexes); + } + + public OrcIndex readRowIndex(int stripeIndex, boolean[] included, boolean[] readCols) + throws IOException { + // Use the cached objects if the read request matches the cached request + if (stripeIndex == currentStripe + && (readCols == null || Arrays.equals(readCols, rowIndexColsToRead))) { + if (rowIndexColsToRead != null) { + return indexes; + } else { + return planner.readRowIndex(readCols, indexes); + } + } else { + StripePlanner copy = new StripePlanner(planner); + if (included == null) { + included = new boolean[schema.getMaximumId() + 1]; + Arrays.fill(included, true); + } + copy.parseStripe(stripes.get(stripeIndex), included); + return copy.readRowIndex(readCols, null); + } + } + + private void seekToRowEntry(BatchReader reader, int rowEntry, TypeReader.ReadPhase readPhase) + throws IOException { + OrcProto.RowIndex[] rowIndices = indexes.getRowGroupIndex(); + PositionProvider[] index = new PositionProvider[rowIndices.length]; + for (int i = 0; i < index.length; ++i) { + if (rowIndices[i] != null) { + OrcProto.RowIndexEntry entry = rowIndices[i].getEntry(rowEntry); + // This is effectively a test for pre-ORC-569 files. + if (rowEntry == 0 && entry.getPositionsCount() == 0) { + index[i] = new ZeroPositionProvider(); + } else { + index[i] = new PositionProviderImpl(entry); + } + } + } + reader.seek(index, readPhase); + } + + @Override + public void seekToRow(long rowNumber) throws IOException { + if (rowNumber < 0) { + throw new IllegalArgumentException("Seek to a negative row number " + rowNumber); + } else if (rowNumber < firstRow) { + throw new IllegalArgumentException("Seek before reader range " + rowNumber); + } + // convert to our internal form (rows from the beginning of slice) + rowNumber -= firstRow; + + // move to the right stripe + int rightStripe = findStripe(rowNumber); + if (rightStripe != currentStripe) { + currentStripe = rightStripe; + readStripe(); + } + if (rowIndexColsToRead == null) { + // Read the row indexes only if they were not already read as part of readStripe() + readCurrentStripeRowIndex(); + } + + // if we aren't to the right row yet, advance in the stripe. + advanceToNextRow(reader, rowNumber, true); + } + + private static final String TRANSLATED_SARG_SEPARATOR = "_"; + + public static String encodeTranslatedSargColumn(int rootColumn, Integer indexInSourceTable) { + return rootColumn + + TRANSLATED_SARG_SEPARATOR + + ((indexInSourceTable == null) ? -1 : indexInSourceTable); + } + + public static int[] mapTranslatedSargColumns( + List types, List sargLeaves) { + int[] result = new int[sargLeaves.size()]; + OrcProto.Type lastRoot = null; // Root will be the same for everyone as of now. + String lastRootStr = null; + for (int i = 0; i < result.length; ++i) { + String[] rootAndIndex = + sargLeaves.get(i).getColumnName().split(TRANSLATED_SARG_SEPARATOR); + assert rootAndIndex.length == 2; + String rootStr = rootAndIndex[0], indexStr = rootAndIndex[1]; + int index = Integer.parseInt(indexStr); + // First, check if the column even maps to anything. + if (index == -1) { + result[i] = -1; + continue; + } + assert index >= 0; + // Then, find the root type if needed. + if (!rootStr.equals(lastRootStr)) { + lastRoot = types.get(Integer.parseInt(rootStr)); + lastRootStr = rootStr; + } + // Subtypes of the root types correspond, in order, to the columns in the table schema + // (disregarding schema evolution that doesn't presently work). Get the index for the + // corresponding subtype. + result[i] = lastRoot.getSubtypes(index); + } + return result; + } + + public CompressionCodec getCompressionCodec() { + return dataReader.getCompressionOptions().getCodec(); + } + + public int getMaxDiskRangeChunkLimit() { + return maxDiskRangeChunkLimit; + } + + /** + * Get sargApplier for testing. + * + * @return sargApplier in record reader. + */ + SargApplier getSargApp() { + return sargApp; + } +} diff --git a/paimon-format/src/main/java/org/apache/paimon/format/avro/AvroBulkFormat.java b/paimon-format/src/main/java/org/apache/paimon/format/avro/AvroBulkFormat.java index 7f3e275183cf..a06ca9948c44 100644 --- a/paimon-format/src/main/java/org/apache/paimon/format/avro/AvroBulkFormat.java +++ b/paimon-format/src/main/java/org/apache/paimon/format/avro/AvroBulkFormat.java @@ -22,7 +22,7 @@ import org.apache.paimon.format.FormatReaderFactory; import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.Path; -import org.apache.paimon.reader.RecordReader; +import org.apache.paimon.reader.FileRecordReader; import org.apache.paimon.types.RowType; import org.apache.paimon.utils.IOUtils; import org.apache.paimon.utils.IteratorResultIterator; @@ -49,12 +49,12 @@ public AvroBulkFormat(RowType projectedRowType) { } @Override - public RecordReader createReader(FormatReaderFactory.Context context) + public FileRecordReader createReader(FormatReaderFactory.Context context) throws IOException { return new AvroReader(context.fileIO(), context.filePath(), context.fileSize()); } - private class AvroReader implements RecordReader { + private class AvroReader implements FileRecordReader { private final FileIO fileIO; private final DataFileReader reader; @@ -90,7 +90,7 @@ private DataFileReader createReaderFromPath(Path path, long fileSiz @Nullable @Override - public RecordIterator readBatch() throws IOException { + public IteratorResultIterator readBatch() throws IOException { Object ticket; try { ticket = pool.pollEntry(); diff --git a/paimon-format/src/main/java/org/apache/paimon/format/avro/AvroFileFormat.java b/paimon-format/src/main/java/org/apache/paimon/format/avro/AvroFileFormat.java index 63a51c0a13a9..fcce9ae50530 100644 --- a/paimon-format/src/main/java/org/apache/paimon/format/avro/AvroFileFormat.java +++ b/paimon-format/src/main/java/org/apache/paimon/format/avro/AvroFileFormat.java @@ -105,7 +105,7 @@ private CodecFactory createCodecFactory(String compression) { if (compression.equalsIgnoreCase("zstd")) { return CodecFactory.zstandardCodec(zstdLevel); } - return CodecFactory.fromString(options.get(AVRO_OUTPUT_CODEC)); + return CodecFactory.fromString(compression); } /** A {@link FormatWriterFactory} to write {@link InternalRow}. */ diff --git a/paimon-format/src/main/java/org/apache/paimon/format/avro/AvroRowDatumWriter.java b/paimon-format/src/main/java/org/apache/paimon/format/avro/AvroRowDatumWriter.java index c2bd81d0038f..d30245162572 100644 --- a/paimon-format/src/main/java/org/apache/paimon/format/avro/AvroRowDatumWriter.java +++ b/paimon-format/src/main/java/org/apache/paimon/format/avro/AvroRowDatumWriter.java @@ -56,6 +56,15 @@ public void write(InternalRow datum, Encoder out) throws IOException { // top Row is a UNION type out.writeIndex(1); } - this.writer.writeRow(datum, out); + try { + this.writer.writeRow(datum, out); + } catch (NullPointerException npe) { + throw new RuntimeException( + "Caught NullPointerException, the possible reason is you have set following options together:\n" + + " 1. file.format = avro;\n" + + " 2. merge-function = aggregation/partial-update;\n" + + " 3. some fields are not null.", + npe); + } } } diff --git a/paimon-format/src/main/java/org/apache/paimon/format/orc/OrcFileFormat.java b/paimon-format/src/main/java/org/apache/paimon/format/orc/OrcFileFormat.java index 302b72f87644..c3521c6f1a37 100644 --- a/paimon-format/src/main/java/org/apache/paimon/format/orc/OrcFileFormat.java +++ b/paimon-format/src/main/java/org/apache/paimon/format/orc/OrcFileFormat.java @@ -28,7 +28,6 @@ import org.apache.paimon.format.orc.filter.OrcFilters; import org.apache.paimon.format.orc.filter.OrcPredicateFunctionVisitor; import org.apache.paimon.format.orc.filter.OrcSimpleStatsExtractor; -import org.apache.paimon.format.orc.reader.OrcSplitReaderUtil; import org.apache.paimon.format.orc.writer.RowDataVectorizer; import org.apache.paimon.format.orc.writer.Vectorizer; import org.apache.paimon.options.MemorySize; @@ -56,6 +55,7 @@ import java.util.Properties; import java.util.stream.Collectors; +import static org.apache.paimon.CoreOptions.DELETION_VECTORS_ENABLED; import static org.apache.paimon.types.DataTypeChecks.getFieldTypes; /** Orc {@link FileFormat}. */ @@ -69,6 +69,7 @@ public class OrcFileFormat extends FileFormat { private final org.apache.hadoop.conf.Configuration writerConf; private final int readBatchSize; private final int writeBatchSize; + private final boolean deletionVectorsEnabled; public OrcFileFormat(FormatContext formatContext) { super(IDENTIFIER); @@ -79,6 +80,7 @@ public OrcFileFormat(FormatContext formatContext) { this.orcProperties.forEach((k, v) -> writerConf.set(k.toString(), v.toString())); this.readBatchSize = formatContext.readBatchSize(); this.writeBatchSize = formatContext.writeBatchSize(); + this.deletionVectorsEnabled = formatContext.options().get(DELETION_VECTORS_ENABLED); } @VisibleForTesting @@ -101,7 +103,6 @@ public Optional createStatsExtractor( public FormatReaderFactory createReaderFactory( RowType projectedRowType, @Nullable List filters) { List orcPredicates = new ArrayList<>(); - if (filters != null) { for (Predicate pred : filters) { Optional orcPred = @@ -114,13 +115,14 @@ public FormatReaderFactory createReaderFactory( readerConf, (RowType) refineDataType(projectedRowType), orcPredicates, - readBatchSize); + readBatchSize, + deletionVectorsEnabled); } @Override public void validateDataFields(RowType rowType) { DataType refinedType = refineDataType(rowType); - OrcSplitReaderUtil.toOrcType(refinedType); + OrcTypeUtil.convertToOrcSchema((RowType) refinedType); } /** @@ -138,9 +140,8 @@ public FormatWriterFactory createWriterFactory(RowType type) { DataType refinedType = refineDataType(type); DataType[] orcTypes = getFieldTypes(refinedType).toArray(new DataType[0]); - TypeDescription typeDescription = OrcSplitReaderUtil.toOrcType(refinedType); - Vectorizer vectorizer = - new RowDataVectorizer(typeDescription.toString(), orcTypes); + TypeDescription typeDescription = OrcTypeUtil.convertToOrcSchema((RowType) refinedType); + Vectorizer vectorizer = new RowDataVectorizer(typeDescription, orcTypes); return new OrcWriterFactory(vectorizer, orcProperties, writerConf, writeBatchSize); } diff --git a/paimon-format/src/main/java/org/apache/paimon/format/orc/OrcReaderFactory.java b/paimon-format/src/main/java/org/apache/paimon/format/orc/OrcReaderFactory.java index 5093a5010773..db17357bfd70 100644 --- a/paimon-format/src/main/java/org/apache/paimon/format/orc/OrcReaderFactory.java +++ b/paimon-format/src/main/java/org/apache/paimon/format/orc/OrcReaderFactory.java @@ -23,12 +23,15 @@ import org.apache.paimon.data.columnar.ColumnarRow; import org.apache.paimon.data.columnar.ColumnarRowIterator; import org.apache.paimon.data.columnar.VectorizedColumnBatch; +import org.apache.paimon.fileindex.FileIndexResult; +import org.apache.paimon.fileindex.bitmap.BitmapIndexResult; import org.apache.paimon.format.FormatReaderFactory; import org.apache.paimon.format.OrcFormatReaderContext; import org.apache.paimon.format.fs.HadoopReadOnlyFileSystem; import org.apache.paimon.format.orc.filter.OrcFilters; import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.Path; +import org.apache.paimon.reader.FileRecordReader; import org.apache.paimon.reader.RecordReader.RecordIterator; import org.apache.paimon.types.DataType; import org.apache.paimon.types.RowType; @@ -45,28 +48,27 @@ import org.apache.orc.RecordReader; import org.apache.orc.StripeInformation; import org.apache.orc.TypeDescription; +import org.apache.orc.impl.ReaderImpl; +import org.apache.orc.impl.RecordReaderImpl; import javax.annotation.Nullable; import java.io.IOException; import java.util.List; +import static org.apache.paimon.format.orc.OrcTypeUtil.convertToOrcSchema; import static org.apache.paimon.format.orc.reader.AbstractOrcColumnVector.createPaimonVector; -import static org.apache.paimon.format.orc.reader.OrcSplitReaderUtil.toOrcType; import static org.apache.paimon.utils.Preconditions.checkNotNull; /** An ORC reader that produces a stream of {@link ColumnarRow} records. */ public class OrcReaderFactory implements FormatReaderFactory { protected final Configuration hadoopConfig; - protected final TypeDescription schema; - - private final RowType tableType; - + protected final RowType tableType; protected final List conjunctPredicates; - protected final int batchSize; + protected final boolean deletionVectorsEnabled; /** * @param hadoopConfig the hadoop config for orc reader. @@ -77,12 +79,14 @@ public OrcReaderFactory( final org.apache.hadoop.conf.Configuration hadoopConfig, final RowType readType, final List conjunctPredicates, - final int batchSize) { + final int batchSize, + final boolean deletionVectorsEnabled) { this.hadoopConfig = checkNotNull(hadoopConfig); - this.schema = toOrcType(readType); + this.schema = convertToOrcSchema(readType); this.tableType = readType; this.conjunctPredicates = checkNotNull(conjunctPredicates); this.batchSize = batchSize; + this.deletionVectorsEnabled = deletionVectorsEnabled; } // ------------------------------------------------------------------------ @@ -104,7 +108,9 @@ public OrcVectorizedReader createReader(FormatReaderFactory.Context context) context.fileIO(), context.filePath(), 0, - context.fileSize()); + context.fileSize(), + context.fileIndex(), + deletionVectorsEnabled); return new OrcVectorizedReader(orcReader, poolOfBatches); } @@ -123,7 +129,9 @@ public OrcReaderBatch createReaderBatch( for (int i = 0; i < vectors.length; i++) { String name = tableFieldNames.get(i); DataType type = tableFieldTypes.get(i); - vectors[i] = createPaimonVector(orcBatch.cols[tableFieldNames.indexOf(name)], type); + vectors[i] = + createPaimonVector( + orcBatch.cols[tableFieldNames.indexOf(name)], orcBatch, type); } return new OrcReaderBatch(filePath, orcBatch, new VectorizedColumnBatch(vectors), recycler); } @@ -178,7 +186,7 @@ public VectorizedRowBatch orcVectorizedRowBatch() { return orcVectorizedRowBatch; } - private RecordIterator convertAndGetIterator( + private ColumnarRowIterator convertAndGetIterator( VectorizedRowBatch orcBatch, long rowNumber) { // no copying from the ORC column vectors to the Paimon columns vectors necessary, // because they point to the same data arrays internally design @@ -203,8 +211,7 @@ private RecordIterator convertAndGetIterator( * batch is addressed by the starting row number of the batch, plus the number of records to be * skipped before. */ - private static final class OrcVectorizedReader - implements org.apache.paimon.reader.RecordReader { + private static final class OrcVectorizedReader implements FileRecordReader { private final RecordReader orcReader; private final Pool pool; @@ -216,7 +223,7 @@ private OrcVectorizedReader(final RecordReader orcReader, final Pool readBatch() throws IOException { + public ColumnarRowIterator readBatch() throws IOException { final OrcReaderBatch batch = getCachedEntry(); final VectorizedRowBatch orcVectorBatch = batch.orcVectorizedRowBatch(); @@ -251,9 +258,11 @@ private static RecordReader createRecordReader( FileIO fileIO, org.apache.paimon.fs.Path path, long splitStart, - long splitLength) + long splitLength, + @Nullable FileIndexResult fileIndexResult, + boolean deletionVectorsEnabled) throws IOException { - org.apache.orc.Reader orcReader = createReader(conf, fileIO, path); + org.apache.orc.Reader orcReader = createReader(conf, fileIO, path, fileIndexResult); try { // get offset and length for the stripes that start in the split Pair offsetAndLength = @@ -268,7 +277,14 @@ private static RecordReader createRecordReader( .skipCorruptRecords(OrcConf.SKIP_CORRUPT_DATA.getBoolean(conf)) .tolerateMissingSchema( OrcConf.TOLERATE_MISSING_SCHEMA.getBoolean(conf)); - + if (!conjunctPredicates.isEmpty() + && !deletionVectorsEnabled + && !(fileIndexResult instanceof BitmapIndexResult)) { + // row group filter push down will make row number change incorrect + // so deletion vectors mode and bitmap index cannot work with row group push down + options.useSelected(OrcConf.READER_USE_SELECTED.getBoolean(conf)); + options.allowSARGToFilter(OrcConf.ALLOW_SARG_TO_FILTER.getBoolean(conf)); + } // configure filters if (!conjunctPredicates.isEmpty()) { SearchArgument.Builder b = SearchArgumentFactory.newBuilder(); @@ -328,7 +344,8 @@ private static Pair getOffsetAndLengthForSplit( public static org.apache.orc.Reader createReader( org.apache.hadoop.conf.Configuration conf, FileIO fileIO, - org.apache.paimon.fs.Path path) + org.apache.paimon.fs.Path path, + @Nullable FileIndexResult fileIndexResult) throws IOException { // open ORC file and create reader org.apache.hadoop.fs.Path hPath = new org.apache.hadoop.fs.Path(path.toUri()); @@ -338,6 +355,11 @@ public static org.apache.orc.Reader createReader( // configure filesystem from Paimon FileIO readerOptions.filesystem(new HadoopReadOnlyFileSystem(fileIO)); - return OrcFile.createReader(hPath, readerOptions); + return new ReaderImpl(hPath, readerOptions) { + @Override + public RecordReader rows(Options options) throws IOException { + return new RecordReaderImpl(this, options, fileIndexResult); + } + }; } } diff --git a/paimon-format/src/main/java/org/apache/paimon/format/orc/OrcTypeUtil.java b/paimon-format/src/main/java/org/apache/paimon/format/orc/OrcTypeUtil.java new file mode 100644 index 000000000000..f7d3d626d44f --- /dev/null +++ b/paimon-format/src/main/java/org/apache/paimon/format/orc/OrcTypeUtil.java @@ -0,0 +1,147 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.format.orc; + +import org.apache.paimon.annotation.VisibleForTesting; +import org.apache.paimon.table.SpecialFields; +import org.apache.paimon.types.ArrayType; +import org.apache.paimon.types.CharType; +import org.apache.paimon.types.DataField; +import org.apache.paimon.types.DataType; +import org.apache.paimon.types.DataTypes; +import org.apache.paimon.types.DecimalType; +import org.apache.paimon.types.MapType; +import org.apache.paimon.types.RowType; +import org.apache.paimon.types.VarCharType; + +import org.apache.orc.TypeDescription; + +/** Util for orc types. */ +public class OrcTypeUtil { + + public static final String PAIMON_ORC_FIELD_ID_KEY = "paimon.id"; + + public static TypeDescription convertToOrcSchema(RowType rowType) { + TypeDescription struct = TypeDescription.createStruct(); + for (DataField dataField : rowType.getFields()) { + TypeDescription child = convertToOrcType(dataField.type(), dataField.id(), 0); + struct.addField(dataField.name(), child); + } + return struct; + } + + @VisibleForTesting + static TypeDescription convertToOrcType(DataType type, int fieldId, int depth) { + type = type.copy(true); + switch (type.getTypeRoot()) { + case CHAR: + return TypeDescription.createChar() + .withMaxLength(((CharType) type).getLength()) + .setAttribute(PAIMON_ORC_FIELD_ID_KEY, String.valueOf(fieldId)); + case VARCHAR: + int len = ((VarCharType) type).getLength(); + if (len == VarCharType.MAX_LENGTH) { + return TypeDescription.createString() + .setAttribute(PAIMON_ORC_FIELD_ID_KEY, String.valueOf(fieldId)); + } else { + return TypeDescription.createVarchar() + .withMaxLength(len) + .setAttribute(PAIMON_ORC_FIELD_ID_KEY, String.valueOf(fieldId)); + } + case BOOLEAN: + return TypeDescription.createBoolean() + .setAttribute(PAIMON_ORC_FIELD_ID_KEY, String.valueOf(fieldId)); + case VARBINARY: + if (type.equals(DataTypes.BYTES())) { + return TypeDescription.createBinary() + .setAttribute(PAIMON_ORC_FIELD_ID_KEY, String.valueOf(fieldId)); + } else { + throw new UnsupportedOperationException( + "Not support other binary type: " + type); + } + case DECIMAL: + DecimalType decimalType = (DecimalType) type; + return TypeDescription.createDecimal() + .withScale(decimalType.getScale()) + .withPrecision(decimalType.getPrecision()) + .setAttribute(PAIMON_ORC_FIELD_ID_KEY, String.valueOf(fieldId)); + case TINYINT: + return TypeDescription.createByte() + .setAttribute(PAIMON_ORC_FIELD_ID_KEY, String.valueOf(fieldId)); + case SMALLINT: + return TypeDescription.createShort() + .setAttribute(PAIMON_ORC_FIELD_ID_KEY, String.valueOf(fieldId)); + case INTEGER: + case TIME_WITHOUT_TIME_ZONE: + return TypeDescription.createInt() + .setAttribute(PAIMON_ORC_FIELD_ID_KEY, String.valueOf(fieldId)); + case BIGINT: + return TypeDescription.createLong() + .setAttribute(PAIMON_ORC_FIELD_ID_KEY, String.valueOf(fieldId)); + case FLOAT: + return TypeDescription.createFloat() + .setAttribute(PAIMON_ORC_FIELD_ID_KEY, String.valueOf(fieldId)); + case DOUBLE: + return TypeDescription.createDouble() + .setAttribute(PAIMON_ORC_FIELD_ID_KEY, String.valueOf(fieldId)); + case DATE: + return TypeDescription.createDate() + .setAttribute(PAIMON_ORC_FIELD_ID_KEY, String.valueOf(fieldId)); + case TIMESTAMP_WITHOUT_TIME_ZONE: + return TypeDescription.createTimestamp() + .setAttribute(PAIMON_ORC_FIELD_ID_KEY, String.valueOf(fieldId)); + case TIMESTAMP_WITH_LOCAL_TIME_ZONE: + return TypeDescription.createTimestampInstant() + .setAttribute(PAIMON_ORC_FIELD_ID_KEY, String.valueOf(fieldId)); + case ARRAY: + ArrayType arrayType = (ArrayType) type; + + String elementFieldId = + String.valueOf(SpecialFields.getArrayElementFieldId(fieldId, depth + 1)); + TypeDescription elementOrcType = + convertToOrcType(arrayType.getElementType(), fieldId, depth + 1) + .setAttribute(PAIMON_ORC_FIELD_ID_KEY, elementFieldId); + + return TypeDescription.createList(elementOrcType) + .setAttribute(PAIMON_ORC_FIELD_ID_KEY, String.valueOf(fieldId)); + case MAP: + MapType mapType = (MapType) type; + + String mapKeyFieldId = + String.valueOf(SpecialFields.getMapKeyFieldId(fieldId, depth + 1)); + TypeDescription mapKeyOrcType = + convertToOrcType(mapType.getKeyType(), fieldId, depth + 1) + .setAttribute(PAIMON_ORC_FIELD_ID_KEY, mapKeyFieldId); + + String mapValueFieldId = + String.valueOf(SpecialFields.getMapValueFieldId(fieldId, depth + 1)); + TypeDescription mapValueOrcType = + convertToOrcType(mapType.getValueType(), fieldId, depth + 1) + .setAttribute(PAIMON_ORC_FIELD_ID_KEY, mapValueFieldId); + + return TypeDescription.createMap(mapKeyOrcType, mapValueOrcType) + .setAttribute(PAIMON_ORC_FIELD_ID_KEY, String.valueOf(fieldId)); + case ROW: + return convertToOrcSchema((RowType) type) + .setAttribute(PAIMON_ORC_FIELD_ID_KEY, String.valueOf(fieldId)); + default: + throw new UnsupportedOperationException("Unsupported type: " + type); + } + } +} diff --git a/paimon-format/src/main/java/org/apache/paimon/format/orc/filter/OrcSimpleStatsExtractor.java b/paimon-format/src/main/java/org/apache/paimon/format/orc/filter/OrcSimpleStatsExtractor.java index dc6fd69dd0ed..e3c741cb8e36 100644 --- a/paimon-format/src/main/java/org/apache/paimon/format/orc/filter/OrcSimpleStatsExtractor.java +++ b/paimon-format/src/main/java/org/apache/paimon/format/orc/filter/OrcSimpleStatsExtractor.java @@ -74,7 +74,8 @@ public SimpleColStats[] extract(FileIO fileIO, Path path) throws IOException { @Override public Pair extractWithFileInfo(FileIO fileIO, Path path) throws IOException { - try (Reader reader = OrcReaderFactory.createReader(new Configuration(), fileIO, path)) { + try (Reader reader = + OrcReaderFactory.createReader(new Configuration(), fileIO, path, null)) { long rowCount = reader.getNumberOfRows(); ColumnStatistics[] columnStatistics = reader.getStatistics(); TypeDescription schema = reader.getSchema(); diff --git a/paimon-format/src/main/java/org/apache/paimon/format/orc/reader/AbstractOrcColumnVector.java b/paimon-format/src/main/java/org/apache/paimon/format/orc/reader/AbstractOrcColumnVector.java index 21154c4967b7..0557a72230cc 100644 --- a/paimon-format/src/main/java/org/apache/paimon/format/orc/reader/AbstractOrcColumnVector.java +++ b/paimon-format/src/main/java/org/apache/paimon/format/orc/reader/AbstractOrcColumnVector.java @@ -33,6 +33,7 @@ import org.apache.hadoop.hive.ql.exec.vector.MapColumnVector; import org.apache.hadoop.hive.ql.exec.vector.StructColumnVector; import org.apache.hadoop.hive.ql.exec.vector.TimestampColumnVector; +import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; /** This column vector is used to adapt hive's ColumnVector to Paimon's ColumnVector. */ public abstract class AbstractOrcColumnVector @@ -40,37 +41,49 @@ public abstract class AbstractOrcColumnVector private final ColumnVector vector; - AbstractOrcColumnVector(ColumnVector vector) { + private final VectorizedRowBatch orcBatch; + + AbstractOrcColumnVector(ColumnVector vector, VectorizedRowBatch orcBatch) { this.vector = vector; + this.orcBatch = orcBatch; + } + + protected int rowMapper(int r) { + if (vector.isRepeating) { + return 0; + } + return this.orcBatch.selectedInUse ? this.orcBatch.getSelected()[r] : r; } @Override public boolean isNullAt(int i) { - return !vector.noNulls && vector.isNull[vector.isRepeating ? 0 : i]; + return !vector.noNulls && vector.isNull[rowMapper(i)]; } public static org.apache.paimon.data.columnar.ColumnVector createPaimonVector( - ColumnVector vector, DataType dataType) { + ColumnVector vector, VectorizedRowBatch orcBatch, DataType dataType) { if (vector instanceof LongColumnVector) { if (dataType.getTypeRoot() == DataTypeRoot.TIMESTAMP_WITHOUT_TIME_ZONE) { - return new OrcLegacyTimestampColumnVector((LongColumnVector) vector); + return new OrcLegacyTimestampColumnVector((LongColumnVector) vector, orcBatch); } else { - return new OrcLongColumnVector((LongColumnVector) vector); + return new OrcLongColumnVector((LongColumnVector) vector, orcBatch); } } else if (vector instanceof DoubleColumnVector) { - return new OrcDoubleColumnVector((DoubleColumnVector) vector); + return new OrcDoubleColumnVector((DoubleColumnVector) vector, orcBatch); } else if (vector instanceof BytesColumnVector) { - return new OrcBytesColumnVector((BytesColumnVector) vector); + return new OrcBytesColumnVector((BytesColumnVector) vector, orcBatch); } else if (vector instanceof DecimalColumnVector) { - return new OrcDecimalColumnVector((DecimalColumnVector) vector); + return new OrcDecimalColumnVector((DecimalColumnVector) vector, orcBatch); } else if (vector instanceof TimestampColumnVector) { - return new OrcTimestampColumnVector(vector); + return new OrcTimestampColumnVector(vector, orcBatch); } else if (vector instanceof ListColumnVector) { - return new OrcArrayColumnVector((ListColumnVector) vector, (ArrayType) dataType); + return new OrcArrayColumnVector( + (ListColumnVector) vector, orcBatch, (ArrayType) dataType); } else if (vector instanceof StructColumnVector) { - return new OrcRowColumnVector((StructColumnVector) vector, (RowType) dataType); + return new OrcRowColumnVector( + (StructColumnVector) vector, orcBatch, (RowType) dataType); } else if (vector instanceof MapColumnVector) { - return new OrcMapColumnVector((MapColumnVector) vector, (MapType) dataType); + return new OrcMapColumnVector((MapColumnVector) vector, orcBatch, (MapType) dataType); } else { throw new UnsupportedOperationException( "Unsupported vector: " + vector.getClass().getName()); diff --git a/paimon-format/src/main/java/org/apache/paimon/format/orc/reader/OrcArrayColumnVector.java b/paimon-format/src/main/java/org/apache/paimon/format/orc/reader/OrcArrayColumnVector.java index ed16a0b51084..25a1935f3e4b 100644 --- a/paimon-format/src/main/java/org/apache/paimon/format/orc/reader/OrcArrayColumnVector.java +++ b/paimon-format/src/main/java/org/apache/paimon/format/orc/reader/OrcArrayColumnVector.java @@ -24,6 +24,7 @@ import org.apache.paimon.types.ArrayType; import org.apache.hadoop.hive.ql.exec.vector.ListColumnVector; +import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; /** This column vector is used to adapt hive's ListColumnVector to Paimon's ArrayColumnVector. */ public class OrcArrayColumnVector extends AbstractOrcColumnVector @@ -32,14 +33,16 @@ public class OrcArrayColumnVector extends AbstractOrcColumnVector private final ListColumnVector hiveVector; private final ColumnVector paimonVector; - public OrcArrayColumnVector(ListColumnVector hiveVector, ArrayType type) { - super(hiveVector); + public OrcArrayColumnVector( + ListColumnVector hiveVector, VectorizedRowBatch orcBatch, ArrayType type) { + super(hiveVector, orcBatch); this.hiveVector = hiveVector; - this.paimonVector = createPaimonVector(hiveVector.child, type.getElementType()); + this.paimonVector = createPaimonVector(hiveVector.child, orcBatch, type.getElementType()); } @Override public InternalArray getArray(int i) { + i = rowMapper(i); long offset = hiveVector.offsets[i]; long length = hiveVector.lengths[i]; return new ColumnarArray(paimonVector, (int) offset, (int) length); diff --git a/paimon-format/src/main/java/org/apache/paimon/format/orc/reader/OrcBytesColumnVector.java b/paimon-format/src/main/java/org/apache/paimon/format/orc/reader/OrcBytesColumnVector.java index d48bad886a47..7f812bb5628b 100644 --- a/paimon-format/src/main/java/org/apache/paimon/format/orc/reader/OrcBytesColumnVector.java +++ b/paimon-format/src/main/java/org/apache/paimon/format/orc/reader/OrcBytesColumnVector.java @@ -19,6 +19,7 @@ package org.apache.paimon.format.orc.reader; import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector; +import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; /** This column vector is used to adapt hive's BytesColumnVector to Paimon's BytesColumnVector. */ public class OrcBytesColumnVector extends AbstractOrcColumnVector @@ -26,14 +27,14 @@ public class OrcBytesColumnVector extends AbstractOrcColumnVector private final BytesColumnVector vector; - public OrcBytesColumnVector(BytesColumnVector vector) { - super(vector); + public OrcBytesColumnVector(BytesColumnVector vector, VectorizedRowBatch orcBatch) { + super(vector, orcBatch); this.vector = vector; } @Override public Bytes getBytes(int i) { - int rowId = vector.isRepeating ? 0 : i; + int rowId = rowMapper(i); byte[][] data = vector.vector; int[] start = vector.start; int[] length = vector.length; diff --git a/paimon-format/src/main/java/org/apache/paimon/format/orc/reader/OrcDecimalColumnVector.java b/paimon-format/src/main/java/org/apache/paimon/format/orc/reader/OrcDecimalColumnVector.java index 9ea4d763a5d8..382c19f45be1 100644 --- a/paimon-format/src/main/java/org/apache/paimon/format/orc/reader/OrcDecimalColumnVector.java +++ b/paimon-format/src/main/java/org/apache/paimon/format/orc/reader/OrcDecimalColumnVector.java @@ -21,6 +21,7 @@ import org.apache.paimon.data.Decimal; import org.apache.hadoop.hive.ql.exec.vector.DecimalColumnVector; +import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; import java.math.BigDecimal; @@ -32,15 +33,15 @@ public class OrcDecimalColumnVector extends AbstractOrcColumnVector private final DecimalColumnVector vector; - public OrcDecimalColumnVector(DecimalColumnVector vector) { - super(vector); + public OrcDecimalColumnVector(DecimalColumnVector vector, VectorizedRowBatch orcBatch) { + super(vector, orcBatch); this.vector = vector; } @Override public Decimal getDecimal(int i, int precision, int scale) { - BigDecimal data = - vector.vector[vector.isRepeating ? 0 : i].getHiveDecimal().bigDecimalValue(); + i = rowMapper(i); + BigDecimal data = vector.vector[i].getHiveDecimal().bigDecimalValue(); return Decimal.fromBigDecimal(data, precision, scale); } } diff --git a/paimon-format/src/main/java/org/apache/paimon/format/orc/reader/OrcDoubleColumnVector.java b/paimon-format/src/main/java/org/apache/paimon/format/orc/reader/OrcDoubleColumnVector.java index 0c0b0cc51d38..f26dac6de9da 100644 --- a/paimon-format/src/main/java/org/apache/paimon/format/orc/reader/OrcDoubleColumnVector.java +++ b/paimon-format/src/main/java/org/apache/paimon/format/orc/reader/OrcDoubleColumnVector.java @@ -19,6 +19,7 @@ package org.apache.paimon.format.orc.reader; import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector; +import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; /** * This column vector is used to adapt hive's DoubleColumnVector to Paimon's float and double @@ -30,18 +31,20 @@ public class OrcDoubleColumnVector extends AbstractOrcColumnVector private final DoubleColumnVector vector; - public OrcDoubleColumnVector(DoubleColumnVector vector) { - super(vector); + public OrcDoubleColumnVector(DoubleColumnVector vector, VectorizedRowBatch orcBatch) { + super(vector, orcBatch); this.vector = vector; } @Override public double getDouble(int i) { - return vector.vector[vector.isRepeating ? 0 : i]; + i = rowMapper(i); + return vector.vector[i]; } @Override public float getFloat(int i) { - return (float) vector.vector[vector.isRepeating ? 0 : i]; + i = rowMapper(i); + return (float) vector.vector[i]; } } diff --git a/paimon-format/src/main/java/org/apache/paimon/format/orc/reader/OrcLegacyTimestampColumnVector.java b/paimon-format/src/main/java/org/apache/paimon/format/orc/reader/OrcLegacyTimestampColumnVector.java index 18227ecf3dd2..5107e722edb4 100644 --- a/paimon-format/src/main/java/org/apache/paimon/format/orc/reader/OrcLegacyTimestampColumnVector.java +++ b/paimon-format/src/main/java/org/apache/paimon/format/orc/reader/OrcLegacyTimestampColumnVector.java @@ -22,6 +22,7 @@ import org.apache.hadoop.hive.ql.exec.vector.ColumnVector; import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; +import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; import java.time.LocalDateTime; @@ -34,15 +35,15 @@ public class OrcLegacyTimestampColumnVector extends AbstractOrcColumnVector private final LongColumnVector hiveVector; - OrcLegacyTimestampColumnVector(LongColumnVector vector) { - super(vector); + OrcLegacyTimestampColumnVector(LongColumnVector vector, VectorizedRowBatch orcBatch) { + super(vector, orcBatch); this.hiveVector = vector; } @Override public Timestamp getTimestamp(int i, int precision) { - int index = hiveVector.isRepeating ? 0 : i; - java.sql.Timestamp timestamp = toTimestamp(hiveVector.vector[index]); + i = rowMapper(i); + java.sql.Timestamp timestamp = toTimestamp(hiveVector.vector[i]); return Timestamp.fromSQLTimestamp(timestamp); } diff --git a/paimon-format/src/main/java/org/apache/paimon/format/orc/reader/OrcLongColumnVector.java b/paimon-format/src/main/java/org/apache/paimon/format/orc/reader/OrcLongColumnVector.java index e7dfe0e6134e..c289b74c58b2 100644 --- a/paimon-format/src/main/java/org/apache/paimon/format/orc/reader/OrcLongColumnVector.java +++ b/paimon-format/src/main/java/org/apache/paimon/format/orc/reader/OrcLongColumnVector.java @@ -19,6 +19,7 @@ package org.apache.paimon.format.orc.reader; import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; +import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; /** * This column vector is used to adapt hive's LongColumnVector to Paimon's boolean, byte, short, int @@ -33,33 +34,38 @@ public class OrcLongColumnVector extends AbstractOrcColumnVector private final LongColumnVector vector; - public OrcLongColumnVector(LongColumnVector vector) { - super(vector); + public OrcLongColumnVector(LongColumnVector vector, VectorizedRowBatch orcBatch) { + super(vector, orcBatch); this.vector = vector; } @Override public long getLong(int i) { - return vector.vector[vector.isRepeating ? 0 : i]; + i = rowMapper(i); + return vector.vector[i]; } @Override public boolean getBoolean(int i) { - return vector.vector[vector.isRepeating ? 0 : i] == 1; + i = rowMapper(i); + return vector.vector[i] == 1; } @Override public byte getByte(int i) { - return (byte) vector.vector[vector.isRepeating ? 0 : i]; + i = rowMapper(i); + return (byte) vector.vector[i]; } @Override public int getInt(int i) { - return (int) vector.vector[vector.isRepeating ? 0 : i]; + i = rowMapper(i); + return (int) vector.vector[i]; } @Override public short getShort(int i) { - return (short) vector.vector[vector.isRepeating ? 0 : i]; + i = rowMapper(i); + return (short) vector.vector[i]; } } diff --git a/paimon-format/src/main/java/org/apache/paimon/format/orc/reader/OrcMapColumnVector.java b/paimon-format/src/main/java/org/apache/paimon/format/orc/reader/OrcMapColumnVector.java index 66a1af6dccf4..c7245275fdd2 100644 --- a/paimon-format/src/main/java/org/apache/paimon/format/orc/reader/OrcMapColumnVector.java +++ b/paimon-format/src/main/java/org/apache/paimon/format/orc/reader/OrcMapColumnVector.java @@ -24,6 +24,7 @@ import org.apache.paimon.types.MapType; import org.apache.hadoop.hive.ql.exec.vector.MapColumnVector; +import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; /** This column vector is used to adapt hive's MapColumnVector to Paimon's MapColumnVector. */ public class OrcMapColumnVector extends AbstractOrcColumnVector @@ -33,15 +34,18 @@ public class OrcMapColumnVector extends AbstractOrcColumnVector private final ColumnVector keyPaimonVector; private final ColumnVector valuePaimonVector; - public OrcMapColumnVector(MapColumnVector hiveVector, MapType type) { - super(hiveVector); + public OrcMapColumnVector( + MapColumnVector hiveVector, VectorizedRowBatch orcBatch, MapType type) { + super(hiveVector, orcBatch); this.hiveVector = hiveVector; - this.keyPaimonVector = createPaimonVector(hiveVector.keys, type.getKeyType()); - this.valuePaimonVector = createPaimonVector(hiveVector.values, type.getValueType()); + this.keyPaimonVector = createPaimonVector(hiveVector.keys, orcBatch, type.getKeyType()); + this.valuePaimonVector = + createPaimonVector(hiveVector.values, orcBatch, type.getValueType()); } @Override public InternalMap getMap(int i) { + i = rowMapper(i); long offset = hiveVector.offsets[i]; long length = hiveVector.lengths[i]; return new ColumnarMap(keyPaimonVector, valuePaimonVector, (int) offset, (int) length); diff --git a/paimon-format/src/main/java/org/apache/paimon/format/orc/reader/OrcRowColumnVector.java b/paimon-format/src/main/java/org/apache/paimon/format/orc/reader/OrcRowColumnVector.java index caa22467f9c3..6c73c9fdbe0d 100644 --- a/paimon-format/src/main/java/org/apache/paimon/format/orc/reader/OrcRowColumnVector.java +++ b/paimon-format/src/main/java/org/apache/paimon/format/orc/reader/OrcRowColumnVector.java @@ -24,6 +24,7 @@ import org.apache.paimon.types.RowType; import org.apache.hadoop.hive.ql.exec.vector.StructColumnVector; +import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; /** This column vector is used to adapt hive's StructColumnVector to Flink's RowColumnVector. */ public class OrcRowColumnVector extends AbstractOrcColumnVector @@ -31,18 +32,21 @@ public class OrcRowColumnVector extends AbstractOrcColumnVector private final VectorizedColumnBatch batch; - public OrcRowColumnVector(StructColumnVector hiveVector, RowType type) { - super(hiveVector); + public OrcRowColumnVector( + StructColumnVector hiveVector, VectorizedRowBatch orcBatch, RowType type) { + super(hiveVector, orcBatch); int len = hiveVector.fields.length; ColumnVector[] paimonVectors = new ColumnVector[len]; for (int i = 0; i < len; i++) { - paimonVectors[i] = createPaimonVector(hiveVector.fields[i], type.getTypeAt(i)); + paimonVectors[i] = + createPaimonVector(hiveVector.fields[i], orcBatch, type.getTypeAt(i)); } this.batch = new VectorizedColumnBatch(paimonVectors); } @Override public ColumnarRow getRow(int i) { + i = rowMapper(i); return new ColumnarRow(batch, i); } diff --git a/paimon-format/src/main/java/org/apache/paimon/format/orc/reader/OrcSplitReaderUtil.java b/paimon-format/src/main/java/org/apache/paimon/format/orc/reader/OrcSplitReaderUtil.java deleted file mode 100644 index 882f1c753991..000000000000 --- a/paimon-format/src/main/java/org/apache/paimon/format/orc/reader/OrcSplitReaderUtil.java +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.paimon.format.orc.reader; - -import org.apache.paimon.types.ArrayType; -import org.apache.paimon.types.CharType; -import org.apache.paimon.types.DataType; -import org.apache.paimon.types.DataTypes; -import org.apache.paimon.types.DecimalType; -import org.apache.paimon.types.MapType; -import org.apache.paimon.types.RowType; -import org.apache.paimon.types.VarCharType; - -import org.apache.orc.TypeDescription; - -/** Util for orc types. */ -public class OrcSplitReaderUtil { - - public static TypeDescription toOrcType(DataType type) { - type = type.copy(true); - switch (type.getTypeRoot()) { - case CHAR: - return TypeDescription.createChar().withMaxLength(((CharType) type).getLength()); - case VARCHAR: - int len = ((VarCharType) type).getLength(); - if (len == VarCharType.MAX_LENGTH) { - return TypeDescription.createString(); - } else { - return TypeDescription.createVarchar().withMaxLength(len); - } - case BOOLEAN: - return TypeDescription.createBoolean(); - case VARBINARY: - if (type.equals(DataTypes.BYTES())) { - return TypeDescription.createBinary(); - } else { - throw new UnsupportedOperationException( - "Not support other binary type: " + type); - } - case DECIMAL: - DecimalType decimalType = (DecimalType) type; - return TypeDescription.createDecimal() - .withScale(decimalType.getScale()) - .withPrecision(decimalType.getPrecision()); - case TINYINT: - return TypeDescription.createByte(); - case SMALLINT: - return TypeDescription.createShort(); - case INTEGER: - case TIME_WITHOUT_TIME_ZONE: - return TypeDescription.createInt(); - case BIGINT: - return TypeDescription.createLong(); - case FLOAT: - return TypeDescription.createFloat(); - case DOUBLE: - return TypeDescription.createDouble(); - case DATE: - return TypeDescription.createDate(); - case TIMESTAMP_WITHOUT_TIME_ZONE: - return TypeDescription.createTimestamp(); - case TIMESTAMP_WITH_LOCAL_TIME_ZONE: - return TypeDescription.createTimestampInstant(); - case ARRAY: - ArrayType arrayType = (ArrayType) type; - return TypeDescription.createList(toOrcType(arrayType.getElementType())); - case MAP: - MapType mapType = (MapType) type; - return TypeDescription.createMap( - toOrcType(mapType.getKeyType()), toOrcType(mapType.getValueType())); - case ROW: - RowType rowType = (RowType) type; - TypeDescription struct = TypeDescription.createStruct(); - for (int i = 0; i < rowType.getFieldCount(); i++) { - struct.addField( - rowType.getFieldNames().get(i), toOrcType(rowType.getTypeAt(i))); - } - return struct; - default: - throw new UnsupportedOperationException("Unsupported type: " + type); - } - } -} diff --git a/paimon-format/src/main/java/org/apache/paimon/format/orc/reader/OrcTimestampColumnVector.java b/paimon-format/src/main/java/org/apache/paimon/format/orc/reader/OrcTimestampColumnVector.java index dd8ac08f2f57..a6e71d6016f2 100644 --- a/paimon-format/src/main/java/org/apache/paimon/format/orc/reader/OrcTimestampColumnVector.java +++ b/paimon-format/src/main/java/org/apache/paimon/format/orc/reader/OrcTimestampColumnVector.java @@ -23,6 +23,7 @@ import org.apache.hadoop.hive.ql.exec.vector.ColumnVector; import org.apache.hadoop.hive.ql.exec.vector.TimestampColumnVector; +import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; /** * This column vector is used to adapt hive's TimestampColumnVector to Paimon's @@ -33,14 +34,14 @@ public class OrcTimestampColumnVector extends AbstractOrcColumnVector private final TimestampColumnVector vector; - public OrcTimestampColumnVector(ColumnVector vector) { - super(vector); + public OrcTimestampColumnVector(ColumnVector vector, VectorizedRowBatch orcBatch) { + super(vector, orcBatch); this.vector = (TimestampColumnVector) vector; } @Override public Timestamp getTimestamp(int i, int precision) { - int index = vector.isRepeating ? 0 : i; - return DateTimeUtils.toInternal(vector.time[index], vector.nanos[index] % 1_000_000); + i = rowMapper(i); + return DateTimeUtils.toInternal(vector.time[i], vector.nanos[i] % 1_000_000); } } diff --git a/paimon-format/src/main/java/org/apache/paimon/format/orc/writer/RowDataVectorizer.java b/paimon-format/src/main/java/org/apache/paimon/format/orc/writer/RowDataVectorizer.java index 21443cdf9463..46c936a0263e 100644 --- a/paimon-format/src/main/java/org/apache/paimon/format/orc/writer/RowDataVectorizer.java +++ b/paimon-format/src/main/java/org/apache/paimon/format/orc/writer/RowDataVectorizer.java @@ -23,6 +23,7 @@ import org.apache.hadoop.hive.ql.exec.vector.ColumnVector; import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; +import org.apache.orc.TypeDescription; import java.util.Arrays; import java.util.List; @@ -35,7 +36,7 @@ public class RowDataVectorizer extends Vectorizer { private final List fieldWriters; - public RowDataVectorizer(String schema, DataType[] fieldTypes) { + public RowDataVectorizer(TypeDescription schema, DataType[] fieldTypes) { super(schema); this.fieldWriters = Arrays.stream(fieldTypes) diff --git a/paimon-format/src/main/java/org/apache/paimon/format/orc/writer/Vectorizer.java b/paimon-format/src/main/java/org/apache/paimon/format/orc/writer/Vectorizer.java index 0f0e6bba74a8..2add46531a61 100644 --- a/paimon-format/src/main/java/org/apache/paimon/format/orc/writer/Vectorizer.java +++ b/paimon-format/src/main/java/org/apache/paimon/format/orc/writer/Vectorizer.java @@ -39,9 +39,9 @@ public abstract class Vectorizer implements Serializable { private final TypeDescription schema; - public Vectorizer(final String schema) { + public Vectorizer(final TypeDescription schema) { checkNotNull(schema); - this.schema = TypeDescription.fromString(schema); + this.schema = schema; } /** diff --git a/paimon-format/src/main/java/org/apache/paimon/format/parquet/ColumnConfigParser.java b/paimon-format/src/main/java/org/apache/paimon/format/parquet/ColumnConfigParser.java new file mode 100644 index 000000000000..a4e33807f6ee --- /dev/null +++ b/paimon-format/src/main/java/org/apache/paimon/format/parquet/ColumnConfigParser.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.format.parquet; + +import org.apache.hadoop.conf.Configuration; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.function.BiConsumer; +import java.util.function.Function; + +/** + * Parses the specified key-values in the format of root.key#column.path from a {@link + * Configuration} object. + * + *

    NOTE: The file was copied from Apache parquet project. + */ +public class ColumnConfigParser { + + private static class ConfigHelper { + private final String prefix; + private final Function function; + private final BiConsumer consumer; + + public ConfigHelper( + String prefix, Function function, BiConsumer consumer) { + this.prefix = prefix; + this.function = function; + this.consumer = consumer; + } + + public void processKey(String key) { + if (key.startsWith(prefix)) { + String columnPath = key.substring(prefix.length()); + T value = function.apply(key); + consumer.accept(columnPath, value); + } + } + } + + private final List> helpers = new ArrayList<>(); + + public ColumnConfigParser withColumnConfig( + String rootKey, Function function, BiConsumer consumer) { + helpers.add(new ConfigHelper(rootKey + '#', function, consumer)); + return this; + } + + public void parseConfig(Configuration conf) { + for (Map.Entry entry : conf) { + for (ConfigHelper helper : helpers) { + // We retrieve the value from function instead of parsing from the string here to + // use the exact + // implementations + // in Configuration + helper.processKey(entry.getKey()); + } + } + } +} diff --git a/paimon-format/src/main/java/org/apache/paimon/format/parquet/ParquetReaderFactory.java b/paimon-format/src/main/java/org/apache/paimon/format/parquet/ParquetReaderFactory.java index 2feebb321222..6f8cab2202d6 100644 --- a/paimon-format/src/main/java/org/apache/paimon/format/parquet/ParquetReaderFactory.java +++ b/paimon-format/src/main/java/org/apache/paimon/format/parquet/ParquetReaderFactory.java @@ -23,16 +23,17 @@ import org.apache.paimon.data.columnar.ColumnarRow; import org.apache.paimon.data.columnar.ColumnarRowIterator; import org.apache.paimon.data.columnar.VectorizedColumnBatch; +import org.apache.paimon.data.columnar.heap.ElementCountable; import org.apache.paimon.data.columnar.writable.WritableColumnVector; import org.apache.paimon.format.FormatReaderFactory; import org.apache.paimon.format.parquet.reader.ColumnReader; import org.apache.paimon.format.parquet.reader.ParquetDecimalVector; +import org.apache.paimon.format.parquet.reader.ParquetReadState; import org.apache.paimon.format.parquet.reader.ParquetTimestampVector; import org.apache.paimon.format.parquet.type.ParquetField; import org.apache.paimon.fs.Path; import org.apache.paimon.options.Options; -import org.apache.paimon.reader.RecordReader; -import org.apache.paimon.reader.RecordReader.RecordIterator; +import org.apache.paimon.reader.FileRecordReader; import org.apache.paimon.types.ArrayType; import org.apache.paimon.types.DataField; import org.apache.paimon.types.DataType; @@ -88,8 +89,8 @@ public class ParquetReaderFactory implements FormatReaderFactory { private final Options conf; private final RowType projectedType; - private final String[] projectedFields; - private final DataType[] projectedTypes; + private final String[] projectedColumnNames; + private final DataField[] projectedFields; private final int batchSize; private final FilterCompat.Filter filter; private final Set unknownFieldsIndices = new HashSet<>(); @@ -98,14 +99,15 @@ public ParquetReaderFactory( Options conf, RowType projectedType, int batchSize, FilterCompat.Filter filter) { this.conf = conf; this.projectedType = projectedType; - this.projectedFields = projectedType.getFieldNames().toArray(new String[0]); - this.projectedTypes = projectedType.getFieldTypes().toArray(new DataType[0]); + this.projectedColumnNames = projectedType.getFieldNames().toArray(new String[0]); + this.projectedFields = projectedType.getFields().toArray(new DataField[0]); this.batchSize = batchSize; this.filter = filter; } @Override - public ParquetReader createReader(FormatReaderFactory.Context context) throws IOException { + public FileRecordReader createReader(FormatReaderFactory.Context context) + throws IOException { ParquetReadOptions.Builder builder = ParquetReadOptions.builder().withRange(0, context.fileSize()); setReadOptions(builder); @@ -113,7 +115,8 @@ public ParquetReader createReader(FormatReaderFactory.Context context) throws IO ParquetFileReader reader = new ParquetFileReader( ParquetInputFile.fromPath(context.fileIO(), context.filePath()), - builder.build()); + builder.build(), + context.fileIndex()); MessageType fileSchema = reader.getFileMetaData().getSchema(); MessageType requestedSchema = clipParquetSchema(fileSchema); reader.setRequestedSchema(requestedSchema); @@ -128,7 +131,7 @@ public ParquetReader createReader(FormatReaderFactory.Context context) throws IO buildFieldsList(projectedType.getFields(), projectedType.getFieldNames(), columnIO); return new ParquetReader( - reader, requestedSchema, reader.getRecordCount(), poolOfBatches, fields); + reader, requestedSchema, reader.getFilteredRecordCount(), poolOfBatches, fields); } private void setReadOptions(ParquetReadOptions.Builder builder) { @@ -153,20 +156,20 @@ private void setReadOptions(ParquetReadOptions.Builder builder) { /** Clips `parquetSchema` according to `fieldNames`. */ private MessageType clipParquetSchema(GroupType parquetSchema) { - Type[] types = new Type[projectedFields.length]; - for (int i = 0; i < projectedFields.length; ++i) { - String fieldName = projectedFields[i]; + Type[] types = new Type[projectedColumnNames.length]; + for (int i = 0; i < projectedColumnNames.length; ++i) { + String fieldName = projectedColumnNames[i]; if (!parquetSchema.containsField(fieldName)) { LOG.warn( "{} does not exist in {}, will fill the field with null.", fieldName, parquetSchema); types[i] = - ParquetSchemaConverter.convertToParquetType(fieldName, projectedTypes[i]); + ParquetSchemaConverter.convertToParquetType(fieldName, projectedFields[i]); unknownFieldsIndices.add(i); } else { Type parquetType = parquetSchema.getType(fieldName); - types[i] = clipParquetType(projectedTypes[i], parquetType); + types[i] = clipParquetType(projectedFields[i].type(), parquetType); } } @@ -220,7 +223,7 @@ private Type clipParquetType(DataType readType, Type parquetType) { private void checkSchema(MessageType fileSchema, MessageType requestedSchema) throws IOException, UnsupportedOperationException { - if (projectedFields.length != requestedSchema.getFieldCount()) { + if (projectedColumnNames.length != requestedSchema.getFieldCount()) { throw new RuntimeException( "The quality of field type is incompatible with the request schema!"); } @@ -268,13 +271,13 @@ private ParquetReaderBatch createReaderBatch( } private WritableColumnVector[] createWritableVectors(MessageType requestedSchema) { - WritableColumnVector[] columns = new WritableColumnVector[projectedTypes.length]; + WritableColumnVector[] columns = new WritableColumnVector[projectedFields.length]; List types = requestedSchema.getFields(); - for (int i = 0; i < projectedTypes.length; i++) { + for (int i = 0; i < projectedFields.length; i++) { columns[i] = createWritableColumnVector( batchSize, - projectedTypes[i], + projectedFields[i].type(), types.get(i), requestedSchema.getColumns(), 0); @@ -290,9 +293,12 @@ private VectorizedColumnBatch createVectorizedColumnBatch( WritableColumnVector[] writableVectors) { ColumnVector[] vectors = new ColumnVector[writableVectors.length]; for (int i = 0; i < writableVectors.length; i++) { - switch (projectedTypes[i].getTypeRoot()) { + switch (projectedFields[i].type().getTypeRoot()) { case DECIMAL: - vectors[i] = new ParquetDecimalVector(writableVectors[i]); + vectors[i] = + new ParquetDecimalVector( + writableVectors[i], + ((ElementCountable) writableVectors[i]).getLen()); break; case TIMESTAMP_WITHOUT_TIME_ZONE: case TIMESTAMP_WITH_LOCAL_TIME_ZONE: @@ -306,7 +312,7 @@ private VectorizedColumnBatch createVectorizedColumnBatch( return new VectorizedColumnBatch(vectors); } - private class ParquetReader implements RecordReader { + private class ParquetReader implements FileRecordReader { private ParquetFileReader reader; @@ -331,6 +337,10 @@ private class ParquetReader implements RecordReader { private long nextRowPosition; + private ParquetReadState currentRowGroupReadState; + + private long currentRowGroupFirstRowIndex; + /** * For each request column, the reader to read this column. This is NULL if this column is * missing from the file, in which case we populate the attribute with NULL. @@ -354,12 +364,13 @@ private ParquetReader( this.totalCountLoadedSoFar = 0; this.currentRowPosition = 0; this.nextRowPosition = 0; + this.currentRowGroupFirstRowIndex = 0; this.fields = fields; } @Nullable @Override - public RecordIterator readBatch() throws IOException { + public ColumnarRowIterator readBatch() throws IOException { final ParquetReaderBatch batch = getCachedEntry(); if (!nextBatch(batch)) { @@ -385,7 +396,8 @@ private boolean nextBatch(ParquetReaderBatch batch) throws IOException { currentRowPosition = nextRowPosition; } - int num = (int) Math.min(batchSize, totalCountLoadedSoFar - rowsReturned); + int num = getBachSize(); + for (int i = 0; i < columnReaders.length; ++i) { if (columnReaders[i] == null) { batch.writableVectors[i].fillWithNulls(); @@ -395,13 +407,13 @@ private boolean nextBatch(ParquetReaderBatch batch) throws IOException { } } rowsReturned += num; - nextRowPosition = currentRowPosition + num; + nextRowPosition = getNextRowPosition(num); batch.columnarBatch.setNumRows(num); return true; } private void readNextRowGroup() throws IOException { - PageReadStore rowGroup = reader.readNextRowGroup(); + PageReadStore rowGroup = reader.readNextFilteredRowGroup(); if (rowGroup == null) { throw new IOException( "expecting more rows but reached last block. Read " @@ -410,13 +422,16 @@ private void readNextRowGroup() throws IOException { + totalRowCount); } + this.currentRowGroupReadState = + new ParquetReadState(rowGroup.getRowIndexes().orElse(null)); + List types = requestedSchema.getFields(); columnReaders = new ColumnReader[types.size()]; for (int i = 0; i < types.size(); ++i) { if (!unknownFieldsIndices.contains(i)) { columnReaders[i] = createColumnReader( - projectedTypes[i], + projectedFields[i].type(), types.get(i), requestedSchema.getColumns(), rowGroup, @@ -424,18 +439,62 @@ private void readNextRowGroup() throws IOException { 0); } } + totalCountLoadedSoFar += rowGroup.getRowCount(); - if (rowGroup.getRowIndexOffset().isPresent()) { - currentRowPosition = rowGroup.getRowIndexOffset().get(); + + if (rowGroup.getRowIndexOffset().isPresent()) { // filter + currentRowGroupFirstRowIndex = rowGroup.getRowIndexOffset().get(); + long pageIndex = 0; + if (!this.currentRowGroupReadState.isMaxRange()) { + pageIndex = this.currentRowGroupReadState.currentRangeStart(); + } + currentRowPosition = currentRowGroupFirstRowIndex + pageIndex; } else { if (reader.rowGroupsFiltered()) { throw new RuntimeException( "There is a bug, rowIndexOffset must be present when row groups are filtered."); } + currentRowGroupFirstRowIndex = nextRowPosition; currentRowPosition = nextRowPosition; } } + private int getBachSize() throws IOException { + + long rangeBatchSize = Long.MAX_VALUE; + if (this.currentRowGroupReadState.isFinished()) { + throw new IOException( + "expecting more rows but reached last page block. Read " + + rowsReturned + + " out of " + + totalRowCount); + } else if (!this.currentRowGroupReadState.isMaxRange()) { + long pageIndex = this.currentRowPosition - this.currentRowGroupFirstRowIndex; + rangeBatchSize = this.currentRowGroupReadState.currentRangeEnd() - pageIndex + 1; + } + + return (int) + Math.min( + batchSize, + Math.min(rangeBatchSize, totalCountLoadedSoFar - rowsReturned)); + } + + private long getNextRowPosition(int num) { + if (this.currentRowGroupReadState.isMaxRange()) { + return this.currentRowPosition + num; + } else { + long pageIndex = this.currentRowPosition - this.currentRowGroupFirstRowIndex; + long nextIndex = pageIndex + num; + + if (this.currentRowGroupReadState.currentRangeEnd() < nextIndex) { + this.currentRowGroupReadState.nextRange(); + nextIndex = this.currentRowGroupReadState.currentRangeStart(); + } + + return this.currentRowGroupFirstRowIndex + nextIndex; + } + } + private ParquetReaderBatch getCachedEntry() throws IOException { try { return pool.pollEntry(); @@ -487,7 +546,7 @@ public void recycle() { recycler.recycle(this); } - public RecordIterator convertAndGetIterator(long rowNumber) { + public ColumnarRowIterator convertAndGetIterator(long rowNumber) { result.reset(rowNumber); return result; } diff --git a/paimon-format/src/main/java/org/apache/paimon/format/parquet/ParquetSchemaConverter.java b/paimon-format/src/main/java/org/apache/paimon/format/parquet/ParquetSchemaConverter.java index 43489f9d7f9f..708e5eb7ea3d 100644 --- a/paimon-format/src/main/java/org/apache/paimon/format/parquet/ParquetSchemaConverter.java +++ b/paimon-format/src/main/java/org/apache/paimon/format/parquet/ParquetSchemaConverter.java @@ -18,7 +18,9 @@ package org.apache.paimon.format.parquet; +import org.apache.paimon.table.SpecialFields; import org.apache.paimon.types.ArrayType; +import org.apache.paimon.types.DataField; import org.apache.paimon.types.DataType; import org.apache.paimon.types.DecimalType; import org.apache.paimon.types.IntType; @@ -36,9 +38,6 @@ import org.apache.parquet.schema.Type; import org.apache.parquet.schema.Types; -import java.util.ArrayList; -import java.util.List; - import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY; import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.INT32; import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.INT64; @@ -53,93 +52,113 @@ public class ParquetSchemaConverter { static final String LIST_ELEMENT_NAME = "element"; public static MessageType convertToParquetMessageType(String name, RowType rowType) { - Type[] types = new Type[rowType.getFieldCount()]; - for (int i = 0; i < rowType.getFieldCount(); i++) { - types[i] = convertToParquetType(rowType.getFieldNames().get(i), rowType.getTypeAt(i)); - } - return new MessageType(name, types); + return new MessageType(name, convertToParquetTypes(rowType)); } - public static Type convertToParquetType(String name, DataType type) { - Type.Repetition repetition = - type.isNullable() ? Type.Repetition.OPTIONAL : Type.Repetition.REQUIRED; - return convertToParquetType(name, type, repetition); + public static Type convertToParquetType(String name, DataField field) { + return convertToParquetType(name, field.type(), field.id(), 0); + } + + private static Type[] convertToParquetTypes(RowType rowType) { + return rowType.getFields().stream() + .map(f -> convertToParquetType(f.name(), f.type(), f.id(), 0)) + .toArray(Type[]::new); } - private static Type convertToParquetType( - String name, DataType type, Type.Repetition repetition) { + private static Type convertToParquetType(String name, DataType type, int fieldId, int depth) { + Type.Repetition repetition = + type.isNullable() ? Type.Repetition.OPTIONAL : Type.Repetition.REQUIRED; switch (type.getTypeRoot()) { case CHAR: case VARCHAR: return Types.primitive(PrimitiveType.PrimitiveTypeName.BINARY, repetition) .as(LogicalTypeAnnotation.stringType()) - .named(name); + .named(name) + .withId(fieldId); case BOOLEAN: return Types.primitive(PrimitiveType.PrimitiveTypeName.BOOLEAN, repetition) - .named(name); + .named(name) + .withId(fieldId); case BINARY: case VARBINARY: return Types.primitive(PrimitiveType.PrimitiveTypeName.BINARY, repetition) - .named(name); + .named(name) + .withId(fieldId); case DECIMAL: int precision = ((DecimalType) type).getPrecision(); int scale = ((DecimalType) type).getScale(); if (is32BitDecimal(precision)) { return Types.primitive(INT32, repetition) .as(LogicalTypeAnnotation.decimalType(scale, precision)) - .named(name); + .named(name) + .withId(fieldId); } else if (is64BitDecimal(precision)) { return Types.primitive(INT64, repetition) .as(LogicalTypeAnnotation.decimalType(scale, precision)) - .named(name); + .named(name) + .withId(fieldId); } else { return Types.primitive(FIXED_LEN_BYTE_ARRAY, repetition) .as(LogicalTypeAnnotation.decimalType(scale, precision)) .length(computeMinBytesForDecimalPrecision(precision)) - .named(name); + .named(name) + .withId(fieldId); } case TINYINT: return Types.primitive(INT32, repetition) .as(LogicalTypeAnnotation.intType(8, true)) - .named(name); + .named(name) + .withId(fieldId); case SMALLINT: return Types.primitive(INT32, repetition) .as(LogicalTypeAnnotation.intType(16, true)) - .named(name); + .named(name) + .withId(fieldId); case INTEGER: - return Types.primitive(INT32, repetition).named(name); + return Types.primitive(INT32, repetition).named(name).withId(fieldId); case BIGINT: - return Types.primitive(INT64, repetition).named(name); + return Types.primitive(INT64, repetition).named(name).withId(fieldId); case FLOAT: return Types.primitive(PrimitiveType.PrimitiveTypeName.FLOAT, repetition) - .named(name); + .named(name) + .withId(fieldId); case DOUBLE: return Types.primitive(PrimitiveType.PrimitiveTypeName.DOUBLE, repetition) - .named(name); + .named(name) + .withId(fieldId); case DATE: return Types.primitive(INT32, repetition) .as(LogicalTypeAnnotation.dateType()) - .named(name); + .named(name) + .withId(fieldId); case TIME_WITHOUT_TIME_ZONE: return Types.primitive(INT32, repetition) .as( LogicalTypeAnnotation.timeType( true, LogicalTypeAnnotation.TimeUnit.MILLIS)) - .named(name); + .named(name) + .withId(fieldId); case TIMESTAMP_WITHOUT_TIME_ZONE: TimestampType timestampType = (TimestampType) type; return createTimestampWithLogicalType( - name, timestampType.getPrecision(), repetition, false); + name, timestampType.getPrecision(), repetition, false) + .withId(fieldId); case TIMESTAMP_WITH_LOCAL_TIME_ZONE: LocalZonedTimestampType localZonedTimestampType = (LocalZonedTimestampType) type; return createTimestampWithLogicalType( - name, localZonedTimestampType.getPrecision(), repetition, true); + name, localZonedTimestampType.getPrecision(), repetition, true) + .withId(fieldId); case ARRAY: ArrayType arrayType = (ArrayType) type; - return ConversionPatterns.listOfElements( - repetition, - name, - convertToParquetType(LIST_ELEMENT_NAME, arrayType.getElementType())); + Type elementParquetType = + convertToParquetType( + LIST_ELEMENT_NAME, + arrayType.getElementType(), + fieldId, + depth + 1) + .withId(SpecialFields.getArrayElementFieldId(fieldId, depth + 1)); + return ConversionPatterns.listOfElements(repetition, name, elementParquetType) + .withId(fieldId); case MAP: MapType mapType = (MapType) type; DataType keyType = mapType.getKeyType(); @@ -148,12 +167,20 @@ private static Type convertToParquetType( // it as not nullable keyType = keyType.copy(false); } + Type mapKeyParquetType = + convertToParquetType(MAP_KEY_NAME, keyType, fieldId, depth + 1) + .withId(SpecialFields.getMapKeyFieldId(fieldId, depth + 1)); + Type mapValueParquetType = + convertToParquetType( + MAP_VALUE_NAME, mapType.getValueType(), fieldId, depth + 1) + .withId(SpecialFields.getMapValueFieldId(fieldId, depth + 1)); return ConversionPatterns.mapType( - repetition, - name, - MAP_REPEATED_NAME, - convertToParquetType(MAP_KEY_NAME, keyType), - convertToParquetType(MAP_VALUE_NAME, mapType.getValueType())); + repetition, + name, + MAP_REPEATED_NAME, + mapKeyParquetType, + mapValueParquetType) + .withId(fieldId); case MULTISET: MultisetType multisetType = (MultisetType) type; DataType elementType = multisetType.getElementType(); @@ -162,15 +189,23 @@ private static Type convertToParquetType( // so we configure it as not nullable elementType = elementType.copy(false); } + Type multisetKeyParquetType = + convertToParquetType(MAP_KEY_NAME, elementType, fieldId, depth + 1) + .withId(SpecialFields.getMapKeyFieldId(fieldId, depth + 1)); + Type multisetValueParquetType = + convertToParquetType(MAP_VALUE_NAME, new IntType(false), fieldId, depth + 1) + .withId(SpecialFields.getMapValueFieldId(fieldId, depth + 1)); return ConversionPatterns.mapType( - repetition, - name, - MAP_REPEATED_NAME, - convertToParquetType(MAP_KEY_NAME, elementType), - convertToParquetType(MAP_VALUE_NAME, new IntType(false))); + repetition, + name, + MAP_REPEATED_NAME, + multisetKeyParquetType, + multisetValueParquetType) + .withId(fieldId); case ROW: RowType rowType = (RowType) type; - return new GroupType(repetition, name, convertToParquetTypes(rowType)); + return new GroupType(repetition, name, convertToParquetTypes(rowType)) + .withId(fieldId); default: throw new UnsupportedOperationException("Unsupported type: " + type); } @@ -195,14 +230,6 @@ private static Type createTimestampWithLogicalType( } } - private static List convertToParquetTypes(RowType rowType) { - List types = new ArrayList<>(rowType.getFieldCount()); - for (int i = 0; i < rowType.getFieldCount(); i++) { - types.add(convertToParquetType(rowType.getFieldNames().get(i), rowType.getTypeAt(i))); - } - return types; - } - public static int computeMinBytesForDecimalPrecision(int precision) { int numBytes = 1; while (Math.pow(2.0, 8 * numBytes - 1) < Math.pow(10.0, precision)) { diff --git a/paimon-format/src/main/java/org/apache/paimon/format/parquet/ParquetUtil.java b/paimon-format/src/main/java/org/apache/paimon/format/parquet/ParquetUtil.java index 055fe83f7c66..82d19e448878 100644 --- a/paimon-format/src/main/java/org/apache/paimon/format/parquet/ParquetUtil.java +++ b/paimon-format/src/main/java/org/apache/paimon/format/parquet/ParquetUtil.java @@ -18,6 +18,7 @@ package org.apache.paimon.format.parquet; +import org.apache.paimon.fileindex.FileIndexResult; import org.apache.paimon.format.SimpleStatsExtractor; import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.Path; @@ -48,7 +49,7 @@ public class ParquetUtil { */ public static Pair>, SimpleStatsExtractor.FileInfo> extractColumnStats(FileIO fileIO, Path path) throws IOException { - try (ParquetFileReader reader = getParquetReader(fileIO, path)) { + try (ParquetFileReader reader = getParquetReader(fileIO, path, null)) { ParquetMetadata parquetMetadata = reader.getFooter(); List blockMetaDataList = parquetMetadata.getBlocks(); Map> resultStats = new HashMap<>(); @@ -77,9 +78,12 @@ public class ParquetUtil { * @param path the path of parquet files to be read * @return parquet reader, used for reading footer, status, etc. */ - public static ParquetFileReader getParquetReader(FileIO fileIO, Path path) throws IOException { + public static ParquetFileReader getParquetReader( + FileIO fileIO, Path path, FileIndexResult fileIndexResult) throws IOException { return new ParquetFileReader( - ParquetInputFile.fromPath(fileIO, path), ParquetReadOptions.builder().build()); + ParquetInputFile.fromPath(fileIO, path), + ParquetReadOptions.builder().build(), + fileIndexResult); } static void assertStatsClass( diff --git a/paimon-format/src/main/java/org/apache/paimon/format/parquet/position/CollectionPosition.java b/paimon-format/src/main/java/org/apache/paimon/format/parquet/position/CollectionPosition.java index e72a4280f4aa..beb5de7a92e5 100644 --- a/paimon-format/src/main/java/org/apache/paimon/format/parquet/position/CollectionPosition.java +++ b/paimon-format/src/main/java/org/apache/paimon/format/parquet/position/CollectionPosition.java @@ -22,14 +22,14 @@ /** To represent collection's position in repeated type. */ public class CollectionPosition { + @Nullable private final boolean[] isNull; private final long[] offsets; - private final long[] length; - private final int valueCount; - public CollectionPosition(boolean[] isNull, long[] offsets, long[] length, int valueCount) { + public CollectionPosition( + @Nullable boolean[] isNull, long[] offsets, long[] length, int valueCount) { this.isNull = isNull; this.offsets = offsets; this.length = length; diff --git a/paimon-format/src/main/java/org/apache/paimon/format/parquet/position/LevelDelegation.java b/paimon-format/src/main/java/org/apache/paimon/format/parquet/position/LevelDelegation.java index 25bbedc861d1..8e30d90ba2c7 100644 --- a/paimon-format/src/main/java/org/apache/paimon/format/parquet/position/LevelDelegation.java +++ b/paimon-format/src/main/java/org/apache/paimon/format/parquet/position/LevelDelegation.java @@ -20,6 +20,7 @@ /** To delegate repetition level and definition level. */ public class LevelDelegation { + private final int[] repetitionLevel; private final int[] definitionLevel; diff --git a/paimon-format/src/main/java/org/apache/paimon/format/parquet/reader/AbstractColumnReader.java b/paimon-format/src/main/java/org/apache/paimon/format/parquet/reader/AbstractColumnReader.java index 7e2ab6d5e7f0..d4a0ab039b53 100644 --- a/paimon-format/src/main/java/org/apache/paimon/format/parquet/reader/AbstractColumnReader.java +++ b/paimon-format/src/main/java/org/apache/paimon/format/parquet/reader/AbstractColumnReader.java @@ -32,6 +32,7 @@ import org.apache.parquet.column.page.DataPageV1; import org.apache.parquet.column.page.DataPageV2; import org.apache.parquet.column.page.DictionaryPage; +import org.apache.parquet.column.page.PageReadStore; import org.apache.parquet.column.page.PageReader; import org.apache.parquet.column.values.ValuesReader; import org.apache.parquet.io.ParquetDecodingException; @@ -65,20 +66,13 @@ public abstract class AbstractColumnReader protected final ColumnDescriptor descriptor; - /** Total number of values read. */ - private long valuesRead; - - /** - * value that indicates the end of the current page. That is, if valuesRead == - * endOfPageValueCount, we are at the end of the page. - */ - private long endOfPageValueCount; - /** If true, the current page is dictionary encoded. */ private boolean isCurrentPageDictionaryEncoded; - /** Total values in the current page. */ - private int pageValueCount; + /** + * Helper struct to track intermediate states while reading Parquet pages in the column chunk. + */ + private final ParquetReadState readState; /* * Input streams: @@ -93,7 +87,7 @@ public abstract class AbstractColumnReader */ /** Run length decoder for data and dictionary. */ - protected RunLengthDecoder runLenDecoder; + RunLengthDecoder runLenDecoder; /** Data input stream. */ ByteBufferInputStream dataInputStream; @@ -101,12 +95,14 @@ public abstract class AbstractColumnReader /** Dictionary decoder to wrap dictionary ids input stream. */ private RunLengthDecoder dictionaryIdsDecoder; - public AbstractColumnReader(ColumnDescriptor descriptor, PageReader pageReader) + public AbstractColumnReader(ColumnDescriptor descriptor, PageReadStore pageReadStore) throws IOException { this.descriptor = descriptor; - this.pageReader = pageReader; + this.pageReader = pageReadStore.getPageReader(descriptor); this.maxDefLevel = descriptor.getMaxDefinitionLevel(); + this.readState = new ParquetReadState(pageReadStore.getRowIndexes().orElse(null)); + DictionaryPage dictionaryPage = pageReader.readDictionaryPage(); if (dictionaryPage != null) { try { @@ -147,56 +143,136 @@ public final void readToVector(int readNumber, VECTOR vector) throws IOException if (dictionary != null) { dictionaryIds = vector.reserveDictionaryIds(readNumber); } - while (readNumber > 0) { + + readState.resetForNewBatch(readNumber); + + while (readState.rowsToReadInBatch > 0) { // Compute the number of values we want to read in this page. - int leftInPage = (int) (endOfPageValueCount - valuesRead); - if (leftInPage == 0) { - DataPage page = pageReader.readPage(); - if (page instanceof DataPageV1) { - readPageV1((DataPageV1) page); - } else if (page instanceof DataPageV2) { - readPageV2((DataPageV2) page); - } else { - throw new RuntimeException("Unsupported page type: " + page.getClass()); + if (readState.valuesToReadInPage == 0) { + int pageValueCount = readPage(); + if (pageValueCount < 0) { + // we've read all the pages; this could happen when we're reading a repeated + // list and we + // don't know where the list will end until we've seen all the pages. + break; } - leftInPage = (int) (endOfPageValueCount - valuesRead); } - int num = Math.min(readNumber, leftInPage); - if (isCurrentPageDictionaryEncoded) { - // Read and decode dictionary ids. - runLenDecoder.readDictionaryIds( - num, dictionaryIds, vector, rowId, maxDefLevel, this.dictionaryIdsDecoder); - - if (vector.hasDictionary() || (rowId == 0 && supportLazyDecode())) { - // Column vector supports lazy decoding of dictionary values so just set the - // dictionary. - // We can't do this if rowId != 0 AND the column doesn't have a dictionary (i.e. - // some - // non-dictionary encoded values have already been added). - vector.setDictionary(new ParquetDictionary(dictionary)); + + if (readState.isFinished()) { + break; + } + + long pageRowId = readState.rowId; + int leftInBatch = readState.rowsToReadInBatch; + int leftInPage = readState.valuesToReadInPage; + + int readBatch = Math.min(leftInBatch, leftInPage); + + long rangeStart = readState.currentRangeStart(); + long rangeEnd = readState.currentRangeEnd(); + + if (pageRowId < rangeStart) { + int toSkip = (int) (rangeStart - pageRowId); + if (toSkip >= leftInPage) { // drop page + pageRowId += leftInPage; + leftInPage = 0; } else { - readBatchFromDictionaryIds(rowId, num, vector, dictionaryIds); + if (isCurrentPageDictionaryEncoded) { + runLenDecoder.skipDictionaryIds( + toSkip, maxDefLevel, this.dictionaryIdsDecoder); + pageRowId += toSkip; + leftInPage -= toSkip; + } else { + skipBatch(toSkip); + pageRowId += toSkip; + leftInPage -= toSkip; + } } + } else if (pageRowId > rangeEnd) { + readState.nextRange(); } else { - if (vector.hasDictionary() && rowId != 0) { - // This batch already has dictionary encoded values but this new page is not. - // The batch - // does not support a mix of dictionary and not so we will decode the - // dictionary. - readBatchFromDictionaryIds(0, rowId, vector, vector.getDictionaryIds()); + long start = pageRowId; + long end = Math.min(rangeEnd, pageRowId + readBatch - 1); + int num = (int) (end - start + 1); + + if (isCurrentPageDictionaryEncoded) { + // Read and decode dictionary ids. + runLenDecoder.readDictionaryIds( + num, + dictionaryIds, + vector, + rowId, + maxDefLevel, + this.dictionaryIdsDecoder); + + if (vector.hasDictionary() || (rowId == 0 && supportLazyDecode())) { + // Column vector supports lazy decoding of dictionary values so just set the + // dictionary. + // We can't do this if rowId != 0 AND the column doesn't have a dictionary + // (i.e. + // some + // non-dictionary encoded values have already been added). + vector.setDictionary(new ParquetDictionary(dictionary)); + } else { + readBatchFromDictionaryIds(rowId, num, vector, dictionaryIds); + } + } else { + if (vector.hasDictionary() && rowId != 0) { + // This batch already has dictionary encoded values but this new page is + // not. + // The batch + // does not support a mix of dictionary and not so we will decode the + // dictionary. + readBatchFromDictionaryIds(0, rowId, vector, vector.getDictionaryIds()); + } + vector.setDictionary(null); + readBatch(rowId, num, vector); } - vector.setDictionary(null); - readBatch(rowId, num, vector); + leftInBatch -= num; + pageRowId += num; + leftInPage -= num; + rowId += num; } + readState.rowsToReadInBatch = leftInBatch; + readState.valuesToReadInPage = leftInPage; + readState.rowId = pageRowId; + } + } - valuesRead += num; - rowId += num; - readNumber -= num; + private int readPage() { + DataPage page = pageReader.readPage(); + if (page == null) { + return -1; } + long pageFirstRowIndex = page.getFirstRowIndex().orElse(0L); + + int pageValueCount = + page.accept( + new DataPage.Visitor() { + @Override + public Integer visit(DataPageV1 dataPageV1) { + try { + return readPageV1(dataPageV1); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Override + public Integer visit(DataPageV2 dataPageV2) { + try { + return readPageV2(dataPageV2); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + }); + readState.resetForNewPage(pageValueCount, pageFirstRowIndex); + return pageValueCount; } - private void readPageV1(DataPageV1 page) throws IOException { - this.pageValueCount = page.getValueCount(); + private int readPageV1(DataPageV1 page) throws IOException { + int pageValueCount = page.getValueCount(); ValuesReader rlReader = page.getRlEncoding().getValuesReader(descriptor, REPETITION_LEVEL); // Initialize the decoders. @@ -211,30 +287,31 @@ private void readPageV1(DataPageV1 page) throws IOException { ByteBufferInputStream in = bytes.toInputStream(); rlReader.initFromPage(pageValueCount, in); this.runLenDecoder.initFromStream(pageValueCount, in); - prepareNewPage(page.getValueEncoding(), in); + prepareNewPage(page.getValueEncoding(), in, pageValueCount); + return pageValueCount; } catch (IOException e) { throw new IOException("could not read page " + page + " in col " + descriptor, e); } } - private void readPageV2(DataPageV2 page) throws IOException { - this.pageValueCount = page.getValueCount(); + private int readPageV2(DataPageV2 page) throws IOException { + int pageValueCount = page.getValueCount(); int bitWidth = BytesUtils.getWidthFromMaxInt(descriptor.getMaxDefinitionLevel()); // do not read the length from the stream. v2 pages handle dividing the page bytes. this.runLenDecoder = new RunLengthDecoder(bitWidth, false); this.runLenDecoder.initFromStream( - this.pageValueCount, page.getDefinitionLevels().toInputStream()); + pageValueCount, page.getDefinitionLevels().toInputStream()); try { - prepareNewPage(page.getDataEncoding(), page.getData().toInputStream()); + prepareNewPage(page.getDataEncoding(), page.getData().toInputStream(), pageValueCount); + return pageValueCount; } catch (IOException e) { throw new IOException("could not read page " + page + " in col " + descriptor, e); } } - private void prepareNewPage(Encoding dataEncoding, ByteBufferInputStream in) + private void prepareNewPage(Encoding dataEncoding, ByteBufferInputStream in, int pageValueCount) throws IOException { - this.endOfPageValueCount = valuesRead + pageValueCount; if (dataEncoding.usesDictionary()) { if (dictionary == null) { throw new IOException( @@ -269,6 +346,14 @@ private void prepareNewPage(Encoding dataEncoding, ByteBufferInputStream in) afterReadPage(); } + final void skipDataBuffer(int length) { + try { + dataInputStream.skipFully(length); + } catch (IOException e) { + throw new ParquetDecodingException("Failed to skip " + length + " bytes", e); + } + } + final ByteBuffer readDataBuffer(int length) { try { return dataInputStream.slice(length).order(ByteOrder.LITTLE_ENDIAN); @@ -291,6 +376,8 @@ protected boolean supportLazyDecode() { /** Read batch from {@link #runLenDecoder} and {@link #dataInputStream}. */ protected abstract void readBatch(int rowId, int num, VECTOR column); + protected abstract void skipBatch(int num); + /** * Decode dictionary ids to data. From {@link #runLenDecoder} and {@link #dictionaryIdsDecoder}. */ diff --git a/paimon-format/src/main/java/org/apache/paimon/format/parquet/reader/BooleanColumnReader.java b/paimon-format/src/main/java/org/apache/paimon/format/parquet/reader/BooleanColumnReader.java index d5dc231d8436..4355392bf552 100644 --- a/paimon-format/src/main/java/org/apache/paimon/format/parquet/reader/BooleanColumnReader.java +++ b/paimon-format/src/main/java/org/apache/paimon/format/parquet/reader/BooleanColumnReader.java @@ -22,7 +22,7 @@ import org.apache.paimon.data.columnar.writable.WritableIntVector; import org.apache.parquet.column.ColumnDescriptor; -import org.apache.parquet.column.page.PageReader; +import org.apache.parquet.column.page.PageReadStore; import org.apache.parquet.io.ParquetDecodingException; import org.apache.parquet.schema.PrimitiveType; @@ -36,17 +36,12 @@ public class BooleanColumnReader extends AbstractColumnReader 0) { + if (runLenDecoder.currentCount == 0) { + runLenDecoder.readNextGroup(); + } + int n = Math.min(left, runLenDecoder.currentCount); + switch (runLenDecoder.mode) { + case RLE: + if (runLenDecoder.currentValue == maxDefLevel) { + for (int i = 0; i < n; i++) { + readBoolean(); + } + } + break; + case PACKED: + for (int i = 0; i < n; ++i) { + if (runLenDecoder.currentBuffer[runLenDecoder.currentBufferIdx++] + == maxDefLevel) { + readBoolean(); + } + } + break; + } + left -= n; + runLenDecoder.currentCount -= n; + } + } + private boolean readBoolean() { if (bitOffset == 0) { try { diff --git a/paimon-format/src/main/java/org/apache/paimon/format/parquet/reader/ByteColumnReader.java b/paimon-format/src/main/java/org/apache/paimon/format/parquet/reader/ByteColumnReader.java index bed9923d9be3..804b8bc0275e 100644 --- a/paimon-format/src/main/java/org/apache/paimon/format/parquet/reader/ByteColumnReader.java +++ b/paimon-format/src/main/java/org/apache/paimon/format/parquet/reader/ByteColumnReader.java @@ -22,7 +22,7 @@ import org.apache.paimon.data.columnar.writable.WritableIntVector; import org.apache.parquet.column.ColumnDescriptor; -import org.apache.parquet.column.page.PageReader; +import org.apache.parquet.column.page.PageReadStore; import org.apache.parquet.schema.PrimitiveType; import java.io.IOException; @@ -31,8 +31,9 @@ /** Byte {@link ColumnReader}. Using INT32 to store byte, so just cast int to byte. */ public class ByteColumnReader extends AbstractColumnReader { - public ByteColumnReader(ColumnDescriptor descriptor, PageReader pageReader) throws IOException { - super(descriptor, pageReader); + public ByteColumnReader(ColumnDescriptor descriptor, PageReadStore pageReadStore) + throws IOException { + super(descriptor, pageReadStore); checkTypeName(PrimitiveType.PrimitiveTypeName.INT32); } @@ -69,6 +70,38 @@ protected void readBatch(int rowId, int num, WritableByteVector column) { } } + @Override + protected void skipBatch(int num) { + int left = num; + while (left > 0) { + if (runLenDecoder.currentCount == 0) { + runLenDecoder.readNextGroup(); + } + int n = Math.min(left, runLenDecoder.currentCount); + switch (runLenDecoder.mode) { + case RLE: + if (runLenDecoder.currentValue == maxDefLevel) { + skipByte(n); + } + break; + case PACKED: + for (int i = 0; i < n; ++i) { + if (runLenDecoder.currentBuffer[runLenDecoder.currentBufferIdx++] + == maxDefLevel) { + skipByte(1); + } + } + break; + } + left -= n; + runLenDecoder.currentCount -= n; + } + } + + private void skipByte(int num) { + skipDataBuffer(4 * num); + } + @Override protected void readBatchFromDictionaryIds( int rowId, int num, WritableByteVector column, WritableIntVector dictionaryIds) { diff --git a/paimon-format/src/main/java/org/apache/paimon/format/parquet/reader/BytesColumnReader.java b/paimon-format/src/main/java/org/apache/paimon/format/parquet/reader/BytesColumnReader.java index e83115c8a69f..6ee395e58568 100644 --- a/paimon-format/src/main/java/org/apache/paimon/format/parquet/reader/BytesColumnReader.java +++ b/paimon-format/src/main/java/org/apache/paimon/format/parquet/reader/BytesColumnReader.java @@ -22,7 +22,7 @@ import org.apache.paimon.data.columnar.writable.WritableIntVector; import org.apache.parquet.column.ColumnDescriptor; -import org.apache.parquet.column.page.PageReader; +import org.apache.parquet.column.page.PageReadStore; import org.apache.parquet.schema.PrimitiveType; import java.io.IOException; @@ -31,9 +31,9 @@ /** Bytes {@link ColumnReader}. A int length and bytes data. */ public class BytesColumnReader extends AbstractColumnReader { - public BytesColumnReader(ColumnDescriptor descriptor, PageReader pageReader) + public BytesColumnReader(ColumnDescriptor descriptor, PageReadStore pageReadStore) throws IOException { - super(descriptor, pageReader); + super(descriptor, pageReadStore); checkTypeName(PrimitiveType.PrimitiveTypeName.BINARY); } @@ -70,6 +70,41 @@ protected void readBatch(int rowId, int num, WritableBytesVector column) { } } + @Override + protected void skipBatch(int num) { + int left = num; + while (left > 0) { + if (runLenDecoder.currentCount == 0) { + runLenDecoder.readNextGroup(); + } + int n = Math.min(left, runLenDecoder.currentCount); + switch (runLenDecoder.mode) { + case RLE: + if (runLenDecoder.currentValue == maxDefLevel) { + skipBinary(n); + } + break; + case PACKED: + for (int i = 0; i < n; ++i) { + if (runLenDecoder.currentBuffer[runLenDecoder.currentBufferIdx++] + == maxDefLevel) { + skipBinary(1); + } + } + break; + } + left -= n; + runLenDecoder.currentCount -= n; + } + } + + private void skipBinary(int num) { + for (int i = 0; i < num; i++) { + int len = readDataBuffer(4).getInt(); + skipDataBuffer(len); + } + } + @Override protected void readBatchFromDictionaryIds( int rowId, int num, WritableBytesVector column, WritableIntVector dictionaryIds) { diff --git a/paimon-format/src/main/java/org/apache/paimon/format/parquet/reader/DoubleColumnReader.java b/paimon-format/src/main/java/org/apache/paimon/format/parquet/reader/DoubleColumnReader.java index d6d8aa2bbb22..2cffd406248e 100644 --- a/paimon-format/src/main/java/org/apache/paimon/format/parquet/reader/DoubleColumnReader.java +++ b/paimon-format/src/main/java/org/apache/paimon/format/parquet/reader/DoubleColumnReader.java @@ -22,7 +22,7 @@ import org.apache.paimon.data.columnar.writable.WritableIntVector; import org.apache.parquet.column.ColumnDescriptor; -import org.apache.parquet.column.page.PageReader; +import org.apache.parquet.column.page.PageReadStore; import org.apache.parquet.schema.PrimitiveType; import java.io.IOException; @@ -31,9 +31,9 @@ /** Double {@link ColumnReader}. */ public class DoubleColumnReader extends AbstractColumnReader { - public DoubleColumnReader(ColumnDescriptor descriptor, PageReader pageReader) + public DoubleColumnReader(ColumnDescriptor descriptor, PageReadStore pageReadStore) throws IOException { - super(descriptor, pageReader); + super(descriptor, pageReadStore); checkTypeName(PrimitiveType.PrimitiveTypeName.DOUBLE); } @@ -70,6 +70,38 @@ protected void readBatch(int rowId, int num, WritableDoubleVector column) { } } + @Override + protected void skipBatch(int num) { + int left = num; + while (left > 0) { + if (runLenDecoder.currentCount == 0) { + runLenDecoder.readNextGroup(); + } + int n = Math.min(left, runLenDecoder.currentCount); + switch (runLenDecoder.mode) { + case RLE: + if (runLenDecoder.currentValue == maxDefLevel) { + skipDouble(n); + } + break; + case PACKED: + for (int i = 0; i < n; ++i) { + if (runLenDecoder.currentBuffer[runLenDecoder.currentBufferIdx++] + == maxDefLevel) { + skipDouble(1); + } + } + break; + } + left -= n; + runLenDecoder.currentCount -= n; + } + } + + private void skipDouble(int num) { + skipDataBuffer(8 * num); + } + @Override protected void readBatchFromDictionaryIds( int rowId, int num, WritableDoubleVector column, WritableIntVector dictionaryIds) { diff --git a/paimon-format/src/main/java/org/apache/paimon/format/parquet/reader/FixedLenBytesColumnReader.java b/paimon-format/src/main/java/org/apache/paimon/format/parquet/reader/FixedLenBytesColumnReader.java index afce717a6719..25e1b466e465 100644 --- a/paimon-format/src/main/java/org/apache/paimon/format/parquet/reader/FixedLenBytesColumnReader.java +++ b/paimon-format/src/main/java/org/apache/paimon/format/parquet/reader/FixedLenBytesColumnReader.java @@ -25,7 +25,7 @@ import org.apache.paimon.format.parquet.ParquetSchemaConverter; import org.apache.parquet.column.ColumnDescriptor; -import org.apache.parquet.column.page.PageReader; +import org.apache.parquet.column.page.PageReadStore; import org.apache.parquet.io.api.Binary; import org.apache.parquet.schema.PrimitiveType; @@ -39,8 +39,9 @@ public class FixedLenBytesColumnReader private final int precision; public FixedLenBytesColumnReader( - ColumnDescriptor descriptor, PageReader pageReader, int precision) throws IOException { - super(descriptor, pageReader); + ColumnDescriptor descriptor, PageReadStore pageReadStore, int precision) + throws IOException { + super(descriptor, pageReadStore); checkTypeName(PrimitiveType.PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY); this.precision = precision; } @@ -79,6 +80,35 @@ protected void readBatch(int rowId, int num, VECTOR column) { } } + @Override + protected void skipBatch(int num) { + int bytesLen = descriptor.getPrimitiveType().getTypeLength(); + if (ParquetSchemaConverter.is32BitDecimal(precision)) { + for (int i = 0; i < num; i++) { + if (runLenDecoder.readInteger() == maxDefLevel) { + skipDataBinary(bytesLen); + } + } + } else if (ParquetSchemaConverter.is64BitDecimal(precision)) { + + for (int i = 0; i < num; i++) { + if (runLenDecoder.readInteger() == maxDefLevel) { + skipDataBinary(bytesLen); + } + } + } else { + for (int i = 0; i < num; i++) { + if (runLenDecoder.readInteger() == maxDefLevel) { + skipDataBinary(bytesLen); + } + } + } + } + + private void skipDataBinary(int len) { + skipDataBuffer(len); + } + @Override protected void readBatchFromDictionaryIds( int rowId, int num, VECTOR column, WritableIntVector dictionaryIds) { diff --git a/paimon-format/src/main/java/org/apache/paimon/format/parquet/reader/FloatColumnReader.java b/paimon-format/src/main/java/org/apache/paimon/format/parquet/reader/FloatColumnReader.java index 1f4adfa4b9c8..e9eec13df5fc 100644 --- a/paimon-format/src/main/java/org/apache/paimon/format/parquet/reader/FloatColumnReader.java +++ b/paimon-format/src/main/java/org/apache/paimon/format/parquet/reader/FloatColumnReader.java @@ -22,7 +22,7 @@ import org.apache.paimon.data.columnar.writable.WritableIntVector; import org.apache.parquet.column.ColumnDescriptor; -import org.apache.parquet.column.page.PageReader; +import org.apache.parquet.column.page.PageReadStore; import org.apache.parquet.schema.PrimitiveType; import java.io.IOException; @@ -31,9 +31,9 @@ /** Float {@link ColumnReader}. */ public class FloatColumnReader extends AbstractColumnReader { - public FloatColumnReader(ColumnDescriptor descriptor, PageReader pageReader) + public FloatColumnReader(ColumnDescriptor descriptor, PageReadStore pageReadStore) throws IOException { - super(descriptor, pageReader); + super(descriptor, pageReadStore); checkTypeName(PrimitiveType.PrimitiveTypeName.FLOAT); } @@ -70,6 +70,38 @@ protected void readBatch(int rowId, int num, WritableFloatVector column) { } } + @Override + protected void skipBatch(int num) { + int left = num; + while (left > 0) { + if (runLenDecoder.currentCount == 0) { + runLenDecoder.readNextGroup(); + } + int n = Math.min(left, runLenDecoder.currentCount); + switch (runLenDecoder.mode) { + case RLE: + if (runLenDecoder.currentValue == maxDefLevel) { + skipFloat(n); + } + break; + case PACKED: + for (int i = 0; i < n; ++i) { + if (runLenDecoder.currentBuffer[runLenDecoder.currentBufferIdx++] + == maxDefLevel) { + skipFloat(1); + } + } + break; + } + left -= n; + runLenDecoder.currentCount -= n; + } + } + + private void skipFloat(int num) { + skipDataBuffer(4 * num); + } + @Override protected void readBatchFromDictionaryIds( int rowId, int num, WritableFloatVector column, WritableIntVector dictionaryIds) { diff --git a/paimon-format/src/main/java/org/apache/paimon/format/parquet/reader/IntColumnReader.java b/paimon-format/src/main/java/org/apache/paimon/format/parquet/reader/IntColumnReader.java index e38e916d187e..521ad998f6f1 100644 --- a/paimon-format/src/main/java/org/apache/paimon/format/parquet/reader/IntColumnReader.java +++ b/paimon-format/src/main/java/org/apache/paimon/format/parquet/reader/IntColumnReader.java @@ -21,7 +21,7 @@ import org.apache.paimon.data.columnar.writable.WritableIntVector; import org.apache.parquet.column.ColumnDescriptor; -import org.apache.parquet.column.page.PageReader; +import org.apache.parquet.column.page.PageReadStore; import org.apache.parquet.schema.PrimitiveType; import java.io.IOException; @@ -30,8 +30,9 @@ /** Int {@link ColumnReader}. */ public class IntColumnReader extends AbstractColumnReader { - public IntColumnReader(ColumnDescriptor descriptor, PageReader pageReader) throws IOException { - super(descriptor, pageReader); + public IntColumnReader(ColumnDescriptor descriptor, PageReadStore pageReadStore) + throws IOException { + super(descriptor, pageReadStore); checkTypeName(PrimitiveType.PrimitiveTypeName.INT32); } @@ -68,6 +69,38 @@ protected void readBatch(int rowId, int num, WritableIntVector column) { } } + @Override + protected void skipBatch(int num) { + int left = num; + while (left > 0) { + if (runLenDecoder.currentCount == 0) { + runLenDecoder.readNextGroup(); + } + int n = Math.min(left, runLenDecoder.currentCount); + switch (runLenDecoder.mode) { + case RLE: + if (runLenDecoder.currentValue == maxDefLevel) { + skipInteger(n); + } + break; + case PACKED: + for (int i = 0; i < n; ++i) { + if (runLenDecoder.currentBuffer[runLenDecoder.currentBufferIdx++] + == maxDefLevel) { + skipInteger(1); + } + } + break; + } + left -= n; + runLenDecoder.currentCount -= n; + } + } + + private void skipInteger(int num) { + skipDataBuffer(4 * num); + } + @Override protected void readBatchFromDictionaryIds( int rowId, int num, WritableIntVector column, WritableIntVector dictionaryIds) { diff --git a/paimon-format/src/main/java/org/apache/paimon/format/parquet/reader/LongColumnReader.java b/paimon-format/src/main/java/org/apache/paimon/format/parquet/reader/LongColumnReader.java index a8e04eae673a..c4af086a7026 100644 --- a/paimon-format/src/main/java/org/apache/paimon/format/parquet/reader/LongColumnReader.java +++ b/paimon-format/src/main/java/org/apache/paimon/format/parquet/reader/LongColumnReader.java @@ -22,7 +22,7 @@ import org.apache.paimon.data.columnar.writable.WritableLongVector; import org.apache.parquet.column.ColumnDescriptor; -import org.apache.parquet.column.page.PageReader; +import org.apache.parquet.column.page.PageReadStore; import org.apache.parquet.schema.PrimitiveType; import java.io.IOException; @@ -31,8 +31,9 @@ /** Long {@link ColumnReader}. */ public class LongColumnReader extends AbstractColumnReader { - public LongColumnReader(ColumnDescriptor descriptor, PageReader pageReader) throws IOException { - super(descriptor, pageReader); + public LongColumnReader(ColumnDescriptor descriptor, PageReadStore pageReadStore) + throws IOException { + super(descriptor, pageReadStore); checkTypeName(PrimitiveType.PrimitiveTypeName.INT64); } @@ -69,6 +70,38 @@ protected void readBatch(int rowId, int num, WritableLongVector column) { } } + @Override + protected void skipBatch(int num) { + int left = num; + while (left > 0) { + if (runLenDecoder.currentCount == 0) { + runLenDecoder.readNextGroup(); + } + int n = Math.min(left, runLenDecoder.currentCount); + switch (runLenDecoder.mode) { + case RLE: + if (runLenDecoder.currentValue == maxDefLevel) { + skipValue(n); + } + break; + case PACKED: + for (int i = 0; i < n; ++i) { + if (runLenDecoder.currentBuffer[runLenDecoder.currentBufferIdx++] + == maxDefLevel) { + skipValue(1); + } + } + break; + } + left -= n; + runLenDecoder.currentCount -= n; + } + } + + private void skipValue(int num) { + skipDataBuffer(num * 8); + } + @Override protected void readBatchFromDictionaryIds( int rowId, int num, WritableLongVector column, WritableIntVector dictionaryIds) { diff --git a/paimon-format/src/main/java/org/apache/paimon/format/parquet/reader/NestedColumnReader.java b/paimon-format/src/main/java/org/apache/paimon/format/parquet/reader/NestedColumnReader.java index c89c77603dac..8f20be275447 100644 --- a/paimon-format/src/main/java/org/apache/paimon/format/parquet/reader/NestedColumnReader.java +++ b/paimon-format/src/main/java/org/apache/paimon/format/parquet/reader/NestedColumnReader.java @@ -20,6 +20,7 @@ import org.apache.paimon.data.columnar.ColumnVector; import org.apache.paimon.data.columnar.heap.AbstractHeapVector; +import org.apache.paimon.data.columnar.heap.ElementCountable; import org.apache.paimon.data.columnar.heap.HeapArrayVector; import org.apache.paimon.data.columnar.heap.HeapMapVector; import org.apache.paimon.data.columnar.heap.HeapRowVector; @@ -134,7 +135,7 @@ private Pair readRow( String.format("Row field does not have any children: %s.", field)); } - int len = ((AbstractHeapVector) finalChildrenVectors[0]).getLen(); + int len = ((ElementCountable) finalChildrenVectors[0]).getLen(); boolean[] isNull = new boolean[len]; Arrays.fill(isNull, true); boolean hasNull = false; @@ -278,7 +279,7 @@ private Pair readPrimitive( reader = new NestedPrimitiveColumnReader( descriptor, - pages.getPageReader(descriptor), + pages, isUtcTimestamp, descriptor.getPrimitiveType(), field.getType(), diff --git a/paimon-format/src/main/java/org/apache/paimon/format/parquet/reader/NestedPrimitiveColumnReader.java b/paimon-format/src/main/java/org/apache/paimon/format/parquet/reader/NestedPrimitiveColumnReader.java index 7ee33a0bb5cc..f0a82a6d711e 100644 --- a/paimon-format/src/main/java/org/apache/paimon/format/parquet/reader/NestedPrimitiveColumnReader.java +++ b/paimon-format/src/main/java/org/apache/paimon/format/parquet/reader/NestedPrimitiveColumnReader.java @@ -44,6 +44,7 @@ import org.apache.parquet.column.page.DataPageV1; import org.apache.parquet.column.page.DataPageV2; import org.apache.parquet.column.page.DictionaryPage; +import org.apache.parquet.column.page.PageReadStore; import org.apache.parquet.column.page.PageReader; import org.apache.parquet.column.values.ValuesReader; import org.apache.parquet.column.values.rle.RunLengthBitPackingHybridDecoder; @@ -64,6 +65,7 @@ /** Reader to read nested primitive column. */ public class NestedPrimitiveColumnReader implements ColumnReader { + private static final Logger LOG = LoggerFactory.getLogger(NestedPrimitiveColumnReader.class); private final IntArrayList repetitionLevelList = new IntArrayList(0); @@ -82,15 +84,6 @@ public class NestedPrimitiveColumnReader implements ColumnReader valueList = new ArrayList<>(); + int valueIndex = collectDataFromParquetPage(readNumber, valueList); + + return fillColumnVector(valueIndex, valueList); + } + + private int collectDataFromParquetPage(int total, List valueList) throws IOException { + int valueIndex = 0; // repeated type need two loops to read data. - while (!eof && index < readNumber) { + + readState.resetForNewBatch(total); + + while (!eof && readState.rowsToReadInBatch > 0) { + + if (readState.isFinished()) { // finished to read + eof = true; + break; + } + + long pageRowId = readState.rowId; + long rangeStart = readState.currentRangeStart(); + long rangeEnd = readState.currentRangeEnd(); + + if (pageRowId > rangeEnd) { + readState.nextRange(); + continue; + } + + boolean needFilterSkip = pageRowId < rangeStart; + do { - if (!lastValue.shouldSkip) { + + if (!lastValue.shouldSkip && !needFilterSkip) { valueList.add(lastValue.value); valueIndex++; } } while (readValue() && (repetitionLevel != 0)); - index++; + + if (pageRowId == readState.rowId) { + readState.rowId = readState.rowId + 1; + } + + if (!needFilterSkip) { + readState.rowsToReadInBatch = readState.rowsToReadInBatch - 1; + } } - return fillColumnVector(valueIndex, valueList); + return valueIndex; } public LevelDelegation getLevelDelegation() { @@ -255,20 +285,24 @@ private void readAndSaveRepetitionAndDefinitionLevels() { // get the values of repetition and definitionLevel repetitionLevel = repetitionLevelColumn.nextInt(); definitionLevel = definitionLevelColumn.nextInt(); - valuesRead++; + readState.valuesToReadInPage = readState.valuesToReadInPage - 1; repetitionLevelList.add(repetitionLevel); definitionLevelList.add(definitionLevel); } private int readPageIfNeed() throws IOException { // Compute the number of values we want to read in this page. - int leftInPage = (int) (endOfPageValueCount - valuesRead); - if (leftInPage == 0) { - // no data left in current page, load data from new page - readPage(); - leftInPage = (int) (endOfPageValueCount - valuesRead); + if (readState.valuesToReadInPage == 0) { + int pageValueCount = readPage(); + // 返回当前 page 的数据量 + if (pageValueCount < 0) { + // we've read all the pages; this could happen when we're reading a repeated list + // and we + // don't know where the list will end until we've seen all the pages. + return -1; + } } - return leftInPage; + return readState.valuesToReadInPage; } private Object readPrimitiveTypedRow(DataType category) { @@ -495,7 +529,7 @@ private WritableColumnVector fillColumnVector(int total, List valueList) { phiv.vector[i] = ((List) valueList).get(i); } } - return new ParquetDecimalVector(phiv); + return new ParquetDecimalVector(phiv, total); case INT64: HeapLongVector phlv = new HeapLongVector(total); for (int i = 0; i < valueList.size(); i++) { @@ -505,10 +539,10 @@ private WritableColumnVector fillColumnVector(int total, List valueList) { phlv.vector[i] = ((List) valueList).get(i); } } - return new ParquetDecimalVector(phlv); + return new ParquetDecimalVector(phlv, total); default: HeapBytesVector phbv = getHeapBytesVector(total, valueList); - return new ParquetDecimalVector(phbv); + return new ParquetDecimalVector(phbv, total); } default: throw new RuntimeException("Unsupported type in the list: " + type); @@ -528,33 +562,36 @@ private static HeapBytesVector getHeapBytesVector(int total, List valueList) { return phbv; } - protected void readPage() { + protected int readPage() { DataPage page = pageReader.readPage(); if (page == null) { - return; + return -1; } - page.accept( - new DataPage.Visitor() { - @Override - public Void visit(DataPageV1 dataPageV1) { - readPageV1(dataPageV1); - return null; - } + long pageFirstRowIndex = page.getFirstRowIndex().orElse(0L); - @Override - public Void visit(DataPageV2 dataPageV2) { - readPageV2(dataPageV2); - return null; - } - }); + int pageValueCount = + page.accept( + new DataPage.Visitor() { + @Override + public Integer visit(DataPageV1 dataPageV1) { + return readPageV1(dataPageV1); + } + + @Override + public Integer visit(DataPageV2 dataPageV2) { + return readPageV2(dataPageV2); + } + }); + readState.resetForNewPage(pageValueCount, pageFirstRowIndex); + return pageValueCount; } private void initDataReader(Encoding dataEncoding, ByteBufferInputStream in, int valueCount) throws IOException { - this.pageValueCount = valueCount; - this.endOfPageValueCount = valuesRead + pageValueCount; + // this.pageValueCount = valueCount; + // this.endOfPageValueCount = valuesRead + pageValueCount; if (dataEncoding.usesDictionary()) { this.dataColumn = null; if (dictionary == null) { @@ -577,13 +614,14 @@ private void initDataReader(Encoding dataEncoding, ByteBufferInputStream in, int } try { - dataColumn.initFromPage(pageValueCount, in); + dataColumn.initFromPage(valueCount, in); } catch (IOException e) { throw new IOException(String.format("Could not read page in col %s.", descriptor), e); } } - private void readPageV1(DataPageV1 page) { + private int readPageV1(DataPageV1 page) { + int pageValueCount = page.getValueCount(); ValuesReader rlReader = page.getRlEncoding().getValuesReader(descriptor, REPETITION_LEVEL); ValuesReader dlReader = page.getDlEncoding().getValuesReader(descriptor, DEFINITION_LEVEL); this.repetitionLevelColumn = new ValuesReaderIntIterator(rlReader); @@ -597,15 +635,16 @@ private void readPageV1(DataPageV1 page) { LOG.debug("Reading definition levels at {}.", in.position()); dlReader.initFromPage(pageValueCount, in); LOG.debug("Reading data at {}.", in.position()); - initDataReader(page.getValueEncoding(), in, page.getValueCount()); + initDataReader(page.getValueEncoding(), in, pageValueCount); + return pageValueCount; } catch (IOException e) { throw new ParquetDecodingException( String.format("Could not read page %s in col %s.", page, descriptor), e); } } - private void readPageV2(DataPageV2 page) { - this.pageValueCount = page.getValueCount(); + private int readPageV2(DataPageV2 page) { + int pageValueCount = page.getValueCount(); this.repetitionLevelColumn = newRLEIterator(descriptor.getMaxRepetitionLevel(), page.getRepetitionLevels()); this.definitionLevelColumn = @@ -615,8 +654,8 @@ private void readPageV2(DataPageV2 page) { "Page data size {} bytes and {} records.", page.getData().size(), pageValueCount); - initDataReader( - page.getDataEncoding(), page.getData().toInputStream(), page.getValueCount()); + initDataReader(page.getDataEncoding(), page.getData().toInputStream(), pageValueCount); + return pageValueCount; } catch (IOException e) { throw new ParquetDecodingException( String.format("Could not read page %s in col %s.", page, descriptor), e); diff --git a/paimon-format/src/main/java/org/apache/paimon/format/parquet/reader/ParquetDecimalVector.java b/paimon-format/src/main/java/org/apache/paimon/format/parquet/reader/ParquetDecimalVector.java index 28d308bac61f..42714ab066da 100644 --- a/paimon-format/src/main/java/org/apache/paimon/format/parquet/reader/ParquetDecimalVector.java +++ b/paimon-format/src/main/java/org/apache/paimon/format/parquet/reader/ParquetDecimalVector.java @@ -25,6 +25,7 @@ import org.apache.paimon.data.columnar.Dictionary; import org.apache.paimon.data.columnar.IntColumnVector; import org.apache.paimon.data.columnar.LongColumnVector; +import org.apache.paimon.data.columnar.heap.ElementCountable; import org.apache.paimon.data.columnar.writable.WritableBytesVector; import org.apache.paimon.data.columnar.writable.WritableColumnVector; import org.apache.paimon.data.columnar.writable.WritableIntVector; @@ -38,12 +39,18 @@ * {@link DecimalColumnVector} interface. */ public class ParquetDecimalVector - implements DecimalColumnVector, WritableLongVector, WritableIntVector, WritableBytesVector { + implements DecimalColumnVector, + WritableLongVector, + WritableIntVector, + WritableBytesVector, + ElementCountable { private final ColumnVector vector; + private final int len; - public ParquetDecimalVector(ColumnVector vector) { + public ParquetDecimalVector(ColumnVector vector, int len) { this.vector = vector; + this.len = len; } @Override @@ -225,4 +232,9 @@ public void fill(long value) { ((WritableLongVector) vector).fill(value); } } + + @Override + public int getLen() { + return len; + } } diff --git a/paimon-format/src/main/java/org/apache/paimon/format/parquet/reader/ParquetReadState.java b/paimon-format/src/main/java/org/apache/paimon/format/parquet/reader/ParquetReadState.java new file mode 100644 index 000000000000..aa89ea982144 --- /dev/null +++ b/paimon-format/src/main/java/org/apache/paimon/format/parquet/reader/ParquetReadState.java @@ -0,0 +1,145 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.format.parquet.reader; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.PrimitiveIterator; + +/** Parquet reader state for column index. */ +public class ParquetReadState { + + /** A special row range used when there is no row indexes (hence all rows must be included). */ + private static final RowRange MAX_ROW_RANGE = new RowRange(Long.MIN_VALUE, Long.MAX_VALUE); + + /** + * A special row range used when the row indexes are present AND all the row ranges have been + * processed. This serves as a sentinel at the end indicating that all rows come after the last + * row range should be skipped. + */ + private static final RowRange END_ROW_RANGE = new RowRange(Long.MAX_VALUE, Long.MIN_VALUE); + + private final Iterator rowRanges; + + private RowRange currentRange; + + /** row index for the next read. */ + long rowId; + + int valuesToReadInPage; + int rowsToReadInBatch; + + public ParquetReadState(PrimitiveIterator.OfLong rowIndexes) { + this.rowRanges = constructRanges(rowIndexes); + nextRange(); + } + + /** + * Construct a list of row ranges from the given `rowIndexes`. For example, suppose the + * `rowIndexes` are `[0, 1, 2, 4, 5, 7, 8, 9]`, it will be converted into 3 row ranges: `[0-2], + * [4-5], [7-9]`. + */ + private Iterator constructRanges(PrimitiveIterator.OfLong rowIndexes) { + if (rowIndexes == null) { + return null; + } + + List rowRanges = new ArrayList<>(); + long currentStart = Long.MIN_VALUE; + long previous = Long.MIN_VALUE; + + while (rowIndexes.hasNext()) { + long idx = rowIndexes.nextLong(); + if (currentStart == Long.MIN_VALUE) { + currentStart = idx; + } else if (previous + 1 != idx) { + RowRange range = new RowRange(currentStart, previous); + rowRanges.add(range); + currentStart = idx; + } + previous = idx; + } + + if (previous != Long.MIN_VALUE) { + rowRanges.add(new RowRange(currentStart, previous)); + } + + return rowRanges.iterator(); + } + + /** Must be called at the beginning of reading a new batch. */ + void resetForNewBatch(int batchSize) { + this.rowsToReadInBatch = batchSize; + } + + /** Must be called at the beginning of reading a new page. */ + void resetForNewPage(int totalValuesInPage, long pageFirstRowIndex) { + this.valuesToReadInPage = totalValuesInPage; + this.rowId = pageFirstRowIndex; + } + + /** Returns the start index of the current row range. */ + public long currentRangeStart() { + return currentRange.start; + } + + /** Returns the end index of the current row range. */ + public long currentRangeEnd() { + return currentRange.end; + } + + public boolean isFinished() { + return this.currentRange.equals(END_ROW_RANGE); + } + + public boolean isMaxRange() { + return this.currentRange.equals(MAX_ROW_RANGE); + } + + /** Advance to the next range. */ + public void nextRange() { + if (rowRanges == null) { + currentRange = MAX_ROW_RANGE; + } else if (!rowRanges.hasNext()) { + currentRange = END_ROW_RANGE; + } else { + currentRange = rowRanges.next(); + } + } + + /** Helper struct to represent a range of row indexes `[start, end]`. */ + public static class RowRange { + final long start; + final long end; + + RowRange(long start, long end) { + this.start = start; + this.end = end; + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof RowRange)) { + return false; + } + return ((RowRange) obj).start == this.start && ((RowRange) obj).end == this.end; + } + } +} diff --git a/paimon-format/src/main/java/org/apache/paimon/format/parquet/reader/ParquetSplitReaderUtil.java b/paimon-format/src/main/java/org/apache/paimon/format/parquet/reader/ParquetSplitReaderUtil.java index 860ec54fa88b..a2be77414d5a 100644 --- a/paimon-format/src/main/java/org/apache/paimon/format/parquet/reader/ParquetSplitReaderUtil.java +++ b/paimon-format/src/main/java/org/apache/paimon/format/parquet/reader/ParquetSplitReaderUtil.java @@ -87,58 +87,45 @@ public static ColumnReader createColumnReader( getAllColumnDescriptorByType(depth, type, columnDescriptors); switch (fieldType.getTypeRoot()) { case BOOLEAN: - return new BooleanColumnReader( - descriptors.get(0), pages.getPageReader(descriptors.get(0))); + return new BooleanColumnReader(descriptors.get(0), pages); case TINYINT: - return new ByteColumnReader( - descriptors.get(0), pages.getPageReader(descriptors.get(0))); + return new ByteColumnReader(descriptors.get(0), pages); case DOUBLE: - return new DoubleColumnReader( - descriptors.get(0), pages.getPageReader(descriptors.get(0))); + return new DoubleColumnReader(descriptors.get(0), pages); case FLOAT: - return new FloatColumnReader( - descriptors.get(0), pages.getPageReader(descriptors.get(0))); + return new FloatColumnReader(descriptors.get(0), pages); case INTEGER: case DATE: case TIME_WITHOUT_TIME_ZONE: - return new IntColumnReader( - descriptors.get(0), pages.getPageReader(descriptors.get(0))); + return new IntColumnReader(descriptors.get(0), pages); case BIGINT: - return new LongColumnReader( - descriptors.get(0), pages.getPageReader(descriptors.get(0))); + return new LongColumnReader(descriptors.get(0), pages); case SMALLINT: - return new ShortColumnReader( - descriptors.get(0), pages.getPageReader(descriptors.get(0))); + return new ShortColumnReader(descriptors.get(0), pages); case CHAR: case VARCHAR: case BINARY: case VARBINARY: - return new BytesColumnReader( - descriptors.get(0), pages.getPageReader(descriptors.get(0))); + return new BytesColumnReader(descriptors.get(0), pages); case TIMESTAMP_WITHOUT_TIME_ZONE: case TIMESTAMP_WITH_LOCAL_TIME_ZONE: if (descriptors.get(0).getPrimitiveType().getPrimitiveTypeName() == PrimitiveType.PrimitiveTypeName.INT64) { - return new LongColumnReader( - descriptors.get(0), pages.getPageReader(descriptors.get(0))); + return new LongColumnReader(descriptors.get(0), pages); } - return new TimestampColumnReader( - true, descriptors.get(0), pages.getPageReader(descriptors.get(0))); + return new TimestampColumnReader(true, descriptors.get(0), pages); case DECIMAL: switch (descriptors.get(0).getPrimitiveType().getPrimitiveTypeName()) { case INT32: - return new IntColumnReader( - descriptors.get(0), pages.getPageReader(descriptors.get(0))); + return new IntColumnReader(descriptors.get(0), pages); case INT64: - return new LongColumnReader( - descriptors.get(0), pages.getPageReader(descriptors.get(0))); + return new LongColumnReader(descriptors.get(0), pages); case BINARY: - return new BytesColumnReader( - descriptors.get(0), pages.getPageReader(descriptors.get(0))); + return new BytesColumnReader(descriptors.get(0), pages); case FIXED_LEN_BYTE_ARRAY: return new FixedLenBytesColumnReader( descriptors.get(0), - pages.getPageReader(descriptors.get(0)), + pages, ((DecimalType) fieldType).getPrecision()); } case ARRAY: diff --git a/paimon-format/src/main/java/org/apache/paimon/format/parquet/reader/RowColumnReader.java b/paimon-format/src/main/java/org/apache/paimon/format/parquet/reader/RowColumnReader.java deleted file mode 100644 index fa2da03ef312..000000000000 --- a/paimon-format/src/main/java/org/apache/paimon/format/parquet/reader/RowColumnReader.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.paimon.format.parquet.reader; - -import org.apache.paimon.data.columnar.heap.HeapRowVector; -import org.apache.paimon.data.columnar.writable.WritableColumnVector; - -import java.io.IOException; -import java.util.List; - -/** Row {@link ColumnReader}. */ -public class RowColumnReader implements ColumnReader { - - private final List fieldReaders; - - public RowColumnReader(List fieldReaders) { - this.fieldReaders = fieldReaders; - } - - @Override - public void readToVector(int readNumber, WritableColumnVector vector) throws IOException { - HeapRowVector rowVector = (HeapRowVector) vector; - WritableColumnVector[] vectors = rowVector.getFields(); - // row vector null array - boolean[] isNulls = new boolean[readNumber]; - for (int i = 0; i < vectors.length; i++) { - fieldReaders.get(i).readToVector(readNumber, vectors[i]); - - for (int j = 0; j < readNumber; j++) { - if (i == 0) { - isNulls[j] = vectors[i].isNullAt(j); - } else { - isNulls[j] = isNulls[j] && vectors[i].isNullAt(j); - } - if (i == vectors.length - 1 && isNulls[j]) { - // rowColumnVector[j] is null only when all fields[j] of rowColumnVector[j] is - // null - rowVector.setNullAt(j); - } - } - } - } -} diff --git a/paimon-format/src/main/java/org/apache/paimon/format/parquet/reader/RunLengthDecoder.java b/paimon-format/src/main/java/org/apache/paimon/format/parquet/reader/RunLengthDecoder.java index 2dd1655d571f..ebb8f28fa1ee 100644 --- a/paimon-format/src/main/java/org/apache/paimon/format/parquet/reader/RunLengthDecoder.java +++ b/paimon-format/src/main/java/org/apache/paimon/format/parquet/reader/RunLengthDecoder.java @@ -194,6 +194,51 @@ private void readDictionaryIdData(int total, WritableIntVector c, int rowId) { } } + void skipDictionaryIds(int total, int level, RunLengthDecoder data) { + int left = total; + while (left > 0) { + if (this.currentCount == 0) { + this.readNextGroup(); + } + int n = Math.min(left, this.currentCount); + switch (mode) { + case RLE: + if (currentValue == level) { + data.skipDictionaryIdData(n); + } + break; + case PACKED: + for (int i = 0; i < n; ++i) { + if (currentBuffer[currentBufferIdx++] == level) { + data.readInteger(); + } + } + break; + } + left -= n; + currentCount -= n; + } + } + + private void skipDictionaryIdData(int total) { + int left = total; + while (left > 0) { + if (this.currentCount == 0) { + this.readNextGroup(); + } + int n = Math.min(left, this.currentCount); + switch (mode) { + case RLE: + break; + case PACKED: + currentBufferIdx += n; + break; + } + left -= n; + currentCount -= n; + } + } + /** Reads the next varint encoded int. */ private int readUnsignedVarInt() throws IOException { int value = 0; diff --git a/paimon-format/src/main/java/org/apache/paimon/format/parquet/reader/ShortColumnReader.java b/paimon-format/src/main/java/org/apache/paimon/format/parquet/reader/ShortColumnReader.java index 7b32232261a7..bdb2f401fa3f 100644 --- a/paimon-format/src/main/java/org/apache/paimon/format/parquet/reader/ShortColumnReader.java +++ b/paimon-format/src/main/java/org/apache/paimon/format/parquet/reader/ShortColumnReader.java @@ -22,7 +22,7 @@ import org.apache.paimon.data.columnar.writable.WritableShortVector; import org.apache.parquet.column.ColumnDescriptor; -import org.apache.parquet.column.page.PageReader; +import org.apache.parquet.column.page.PageReadStore; import org.apache.parquet.schema.PrimitiveType; import java.io.IOException; @@ -30,9 +30,9 @@ /** Short {@link ColumnReader}. Using INT32 to store short, so just cast int to short. */ public class ShortColumnReader extends AbstractColumnReader { - public ShortColumnReader(ColumnDescriptor descriptor, PageReader pageReader) + public ShortColumnReader(ColumnDescriptor descriptor, PageReadStore pageReadStore) throws IOException { - super(descriptor, pageReader); + super(descriptor, pageReadStore); checkTypeName(PrimitiveType.PrimitiveTypeName.INT32); } @@ -71,6 +71,38 @@ protected void readBatch(int rowId, int num, WritableShortVector column) { } } + @Override + protected void skipBatch(int num) { + int left = num; + while (left > 0) { + if (runLenDecoder.currentCount == 0) { + runLenDecoder.readNextGroup(); + } + int n = Math.min(left, runLenDecoder.currentCount); + switch (runLenDecoder.mode) { + case RLE: + if (runLenDecoder.currentValue == maxDefLevel) { + skipShot(n); + } + break; + case PACKED: + for (int i = 0; i < n; ++i) { + if (runLenDecoder.currentBuffer[runLenDecoder.currentBufferIdx++] + == maxDefLevel) { + skipShot(1); + } + } + break; + } + left -= n; + runLenDecoder.currentCount -= n; + } + } + + private void skipShot(int num) { + skipDataBuffer(4 * num); + } + @Override protected void readBatchFromDictionaryIds( int rowId, int num, WritableShortVector column, WritableIntVector dictionaryIds) { diff --git a/paimon-format/src/main/java/org/apache/paimon/format/parquet/reader/TimestampColumnReader.java b/paimon-format/src/main/java/org/apache/paimon/format/parquet/reader/TimestampColumnReader.java index 4a279ff90e15..d6ac96ea4458 100644 --- a/paimon-format/src/main/java/org/apache/paimon/format/parquet/reader/TimestampColumnReader.java +++ b/paimon-format/src/main/java/org/apache/paimon/format/parquet/reader/TimestampColumnReader.java @@ -23,7 +23,7 @@ import org.apache.paimon.data.columnar.writable.WritableTimestampVector; import org.apache.parquet.column.ColumnDescriptor; -import org.apache.parquet.column.page.PageReader; +import org.apache.parquet.column.page.PageReadStore; import org.apache.parquet.io.api.Binary; import org.apache.parquet.schema.PrimitiveType; @@ -36,8 +36,9 @@ /** * Timestamp {@link ColumnReader}. We only support INT96 bytes now, julianDay(4) + nanosOfDay(8). - * See https://github.com/apache/parquet-format/blob/master/DataTypes.md#timestamp TIMESTAMP_MILLIS - * and TIMESTAMP_MICROS are the deprecated ConvertedType. + * See Parquet + * Timestamp TIMESTAMP_MILLIS and TIMESTAMP_MICROS are the deprecated ConvertedType. */ public class TimestampColumnReader extends AbstractColumnReader { @@ -49,9 +50,9 @@ public class TimestampColumnReader extends AbstractColumnReader createWriter(OutputFile out, String compression) throws IOException { - return new ParquetRowDataBuilder(out, rowType) - .withConf(conf) - .withCompressionCodec(CompressionCodecName.fromConf(getCompression(compression))) - .withRowGroupSize( - conf.getLong( - ParquetOutputFormat.BLOCK_SIZE, ParquetWriter.DEFAULT_BLOCK_SIZE)) - .withPageSize( - conf.getInt(ParquetOutputFormat.PAGE_SIZE, ParquetWriter.DEFAULT_PAGE_SIZE)) - .withDictionaryPageSize( - conf.getInt( - ParquetOutputFormat.DICTIONARY_PAGE_SIZE, - ParquetProperties.DEFAULT_DICTIONARY_PAGE_SIZE)) - .withMaxPaddingSize( - conf.getInt( - ParquetOutputFormat.MAX_PADDING_BYTES, - ParquetWriter.MAX_PADDING_SIZE_DEFAULT)) - .withDictionaryEncoding( - conf.getBoolean( - ParquetOutputFormat.ENABLE_DICTIONARY, - ParquetProperties.DEFAULT_IS_DICTIONARY_ENABLED)) - .withValidation(conf.getBoolean(ParquetOutputFormat.VALIDATION, false)) - .withWriterVersion( - ParquetProperties.WriterVersion.fromString( - conf.get( - ParquetOutputFormat.WRITER_VERSION, - ParquetProperties.DEFAULT_WRITER_VERSION.toString()))) - .build(); + ParquetRowDataBuilder builder = + new ParquetRowDataBuilder(out, rowType) + .withConf(conf) + .withCompressionCodec( + CompressionCodecName.fromConf(getCompression(compression))) + .withRowGroupSize( + conf.getLong( + ParquetOutputFormat.BLOCK_SIZE, + ParquetWriter.DEFAULT_BLOCK_SIZE)) + .withPageSize( + conf.getInt( + ParquetOutputFormat.PAGE_SIZE, + ParquetWriter.DEFAULT_PAGE_SIZE)) + .withPageRowCountLimit( + conf.getInt( + ParquetOutputFormat.PAGE_ROW_COUNT_LIMIT, + ParquetProperties.DEFAULT_PAGE_ROW_COUNT_LIMIT)) + .withDictionaryPageSize( + conf.getInt( + ParquetOutputFormat.DICTIONARY_PAGE_SIZE, + ParquetProperties.DEFAULT_DICTIONARY_PAGE_SIZE)) + .withMaxPaddingSize( + conf.getInt( + ParquetOutputFormat.MAX_PADDING_BYTES, + ParquetWriter.MAX_PADDING_SIZE_DEFAULT)) + .withDictionaryEncoding( + conf.getBoolean( + ParquetOutputFormat.ENABLE_DICTIONARY, + ParquetProperties.DEFAULT_IS_DICTIONARY_ENABLED)) + .withValidation(conf.getBoolean(ParquetOutputFormat.VALIDATION, false)) + .withWriterVersion( + ParquetProperties.WriterVersion.fromString( + conf.get( + ParquetOutputFormat.WRITER_VERSION, + ParquetProperties.DEFAULT_WRITER_VERSION + .toString()))) + .withBloomFilterEnabled( + conf.getBoolean( + ParquetOutputFormat.BLOOM_FILTER_ENABLED, + ParquetProperties.DEFAULT_BLOOM_FILTER_ENABLED)); + new ColumnConfigParser() + .withColumnConfig( + ParquetOutputFormat.ENABLE_DICTIONARY, + key -> conf.getBoolean(key, false), + builder::withDictionaryEncoding) + .withColumnConfig( + ParquetOutputFormat.BLOOM_FILTER_ENABLED, + key -> conf.getBoolean(key, false), + builder::withBloomFilterEnabled) + .withColumnConfig( + ParquetOutputFormat.BLOOM_FILTER_EXPECTED_NDV, + key -> conf.getLong(key, -1L), + builder::withBloomFilterNDV) + .withColumnConfig( + ParquetOutputFormat.BLOOM_FILTER_FPP, + key -> conf.getDouble(key, ParquetProperties.DEFAULT_BLOOM_FILTER_FPP), + builder::withBloomFilterFPP) + .parseConfig(conf); + return builder.build(); } public String getCompression(String compression) { diff --git a/paimon-format/src/main/java/org/apache/parquet/filter2/predicate/ParquetFilters.java b/paimon-format/src/main/java/org/apache/parquet/filter2/predicate/ParquetFilters.java index cacc241fd24b..96cf2fe726cf 100644 --- a/paimon-format/src/main/java/org/apache/parquet/filter2/predicate/ParquetFilters.java +++ b/paimon-format/src/main/java/org/apache/parquet/filter2/predicate/ParquetFilters.java @@ -172,6 +172,11 @@ private static Comparable toParquetObject(Object value) { } if (value instanceof Number) { + if (value instanceof Byte) { + return ((Byte) value).intValue(); + } else if (value instanceof Short) { + return ((Short) value).intValue(); + } return (Comparable) value; } else if (value instanceof String) { return Binary.fromString((String) value); diff --git a/paimon-format/src/main/java/org/apache/parquet/hadoop/ParquetFileReader.java b/paimon-format/src/main/java/org/apache/parquet/hadoop/ParquetFileReader.java index aca1f021b98f..e3fc118ad674 100644 --- a/paimon-format/src/main/java/org/apache/parquet/hadoop/ParquetFileReader.java +++ b/paimon-format/src/main/java/org/apache/parquet/hadoop/ParquetFileReader.java @@ -18,10 +18,13 @@ package org.apache.parquet.hadoop; +import org.apache.paimon.fileindex.FileIndexResult; +import org.apache.paimon.fileindex.bitmap.BitmapIndexResult; import org.apache.paimon.format.parquet.ParquetInputFile; import org.apache.paimon.format.parquet.ParquetInputStream; import org.apache.paimon.fs.FileRange; import org.apache.paimon.fs.VectoredReadable; +import org.apache.paimon.utils.RoaringBitmap32; import org.apache.hadoop.fs.Path; import org.apache.parquet.ParquetReadOptions; @@ -92,6 +95,7 @@ import java.util.Objects; import java.util.Set; import java.util.concurrent.ExecutionException; +import java.util.stream.Collectors; import java.util.zip.CRC32; import static org.apache.paimon.utils.Preconditions.checkArgument; @@ -225,10 +229,14 @@ private static ParquetMetadata readFooter( private DictionaryPageReader nextDictionaryReader = null; private InternalFileDecryptor fileDecryptor = null; + private FileIndexResult fileIndexResult; - public ParquetFileReader(InputFile file, ParquetReadOptions options) throws IOException { + public ParquetFileReader( + InputFile file, ParquetReadOptions options, FileIndexResult fileIndexResult) + throws IOException { this.converter = new ParquetMetadataConverter(options); this.file = (ParquetInputFile) file; + this.fileIndexResult = fileIndexResult; this.f = this.file.newStream(); this.options = options; try { @@ -333,22 +341,38 @@ public String getFile() { private List filterRowGroups(List blocks) throws IOException { FilterCompat.Filter recordFilter = options.getRecordFilter(); - if (checkRowIndexOffsetExists(blocks) && FilterCompat.isFilteringRequired(recordFilter)) { - // set up data filters based on configured levels - List levels = new ArrayList<>(); + if (checkRowIndexOffsetExists(blocks)) { + if (FilterCompat.isFilteringRequired(recordFilter)) { + // set up data filters based on configured levels + List levels = new ArrayList<>(); - if (options.useStatsFilter()) { - levels.add(STATISTICS); - } + if (options.useStatsFilter()) { + levels.add(STATISTICS); + } - if (options.useDictionaryFilter()) { - levels.add(DICTIONARY); - } + if (options.useDictionaryFilter()) { + levels.add(DICTIONARY); + } - if (options.useBloomFilter()) { - levels.add(BLOOMFILTER); + if (options.useBloomFilter()) { + levels.add(BLOOMFILTER); + } + blocks = RowGroupFilter.filterRowGroups(levels, recordFilter, blocks, this); + } + if (fileIndexResult instanceof BitmapIndexResult) { + RoaringBitmap32 bitmap = ((BitmapIndexResult) fileIndexResult).get(); + blocks = + blocks.stream() + .filter( + it -> { + long rowIndexOffset = it.getRowIndexOffset(); + return bitmap.rangeCardinality( + rowIndexOffset, + rowIndexOffset + it.getRowCount()) + > 0; + }) + .collect(Collectors.toList()); } - return RowGroupFilter.filterRowGroups(levels, recordFilter, blocks, this); } return blocks; diff --git a/paimon-format/src/test/java/org/apache/paimon/format/avro/AvroFileFormatTest.java b/paimon-format/src/test/java/org/apache/paimon/format/avro/AvroFileFormatTest.java index 3f6486baaef2..9c0dbb43fe62 100644 --- a/paimon-format/src/test/java/org/apache/paimon/format/avro/AvroFileFormatTest.java +++ b/paimon-format/src/test/java/org/apache/paimon/format/avro/AvroFileFormatTest.java @@ -198,4 +198,16 @@ private void checkException() throws IOException { .isInstanceOf(IOException.class) .hasMessageContaining("Artificial exception"); } + + @Test + void testCompression() throws IOException { + RowType rowType = DataTypes.ROW(DataTypes.INT().notNull()); + AvroFileFormat format = new AvroFileFormat(new FormatContext(new Options(), 1024, 1024)); + LocalFileIO localFileIO = LocalFileIO.create(); + Path file = new Path(new Path(tempPath.toUri()), UUID.randomUUID().toString()); + try (PositionOutputStream out = localFileIO.newOutputStream(file, false)) { + assertThatThrownBy(() -> format.createWriterFactory(rowType).create(out, "unsupported")) + .hasMessageContaining("Unrecognized codec: unsupported"); + } + } } diff --git a/paimon-format/src/test/java/org/apache/paimon/format/orc/OrcReaderFactoryTest.java b/paimon-format/src/test/java/org/apache/paimon/format/orc/OrcReaderFactoryTest.java index 1efd984965bf..63b391b44c44 100644 --- a/paimon-format/src/test/java/org/apache/paimon/format/orc/OrcReaderFactoryTest.java +++ b/paimon-format/src/test/java/org/apache/paimon/format/orc/OrcReaderFactoryTest.java @@ -277,7 +277,8 @@ protected OrcReaderFactory createFormat( new Configuration(), Projection.of(selectedFields).project(formatType), conjunctPredicates, - BATCH_SIZE); + BATCH_SIZE, + false); } private RecordReader createReader(OrcReaderFactory format, Path split) diff --git a/paimon-format/src/test/java/org/apache/paimon/format/orc/OrcSplitReaderUtilTest.java b/paimon-format/src/test/java/org/apache/paimon/format/orc/OrcSplitReaderUtilTest.java deleted file mode 100644 index c07838dfa34c..000000000000 --- a/paimon-format/src/test/java/org/apache/paimon/format/orc/OrcSplitReaderUtilTest.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.paimon.format.orc; - -import org.apache.paimon.format.orc.reader.OrcSplitReaderUtil; -import org.apache.paimon.types.DataType; -import org.apache.paimon.types.DataTypes; - -import org.junit.jupiter.api.Test; - -import static org.apache.paimon.format.orc.reader.OrcSplitReaderUtil.toOrcType; -import static org.assertj.core.api.Assertions.assertThat; - -/** Test for {@link OrcSplitReaderUtil}. */ -class OrcSplitReaderUtilTest { - - @Test - void testDataTypeToOrcType() { - test("boolean", DataTypes.BOOLEAN()); - test("char(123)", DataTypes.CHAR(123)); - test("varchar(123)", DataTypes.VARCHAR(123)); - test("string", DataTypes.STRING()); - test("binary", DataTypes.BYTES()); - test("tinyint", DataTypes.TINYINT()); - test("smallint", DataTypes.SMALLINT()); - test("int", DataTypes.INT()); - test("bigint", DataTypes.BIGINT()); - test("float", DataTypes.FLOAT()); - test("double", DataTypes.DOUBLE()); - test("date", DataTypes.DATE()); - test("timestamp", DataTypes.TIMESTAMP()); - test("array", DataTypes.ARRAY(DataTypes.FLOAT())); - test("map", DataTypes.MAP(DataTypes.FLOAT(), DataTypes.BIGINT())); - test( - "struct>", - DataTypes.ROW( - DataTypes.FIELD(0, "int0", DataTypes.INT()), - DataTypes.FIELD(1, "str1", DataTypes.STRING()), - DataTypes.FIELD(2, "double2", DataTypes.DOUBLE()), - DataTypes.FIELD( - 3, - "row3", - DataTypes.ROW( - DataTypes.FIELD(4, "int0", DataTypes.INT()), - DataTypes.FIELD(5, "int1", DataTypes.INT()))))); - test("decimal(4,2)", DataTypes.DECIMAL(4, 2)); - } - - private void test(String expected, DataType type) { - assertThat(toOrcType(type)).hasToString(expected); - } -} diff --git a/paimon-format/src/test/java/org/apache/paimon/format/orc/OrcTypeUtilTest.java b/paimon-format/src/test/java/org/apache/paimon/format/orc/OrcTypeUtilTest.java new file mode 100644 index 000000000000..5669ac33d443 --- /dev/null +++ b/paimon-format/src/test/java/org/apache/paimon/format/orc/OrcTypeUtilTest.java @@ -0,0 +1,206 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.format.orc; + +import org.apache.paimon.format.FileFormatFactory; +import org.apache.paimon.format.FormatWriter; +import org.apache.paimon.fs.FileIO; +import org.apache.paimon.fs.Path; +import org.apache.paimon.fs.PositionOutputStream; +import org.apache.paimon.fs.local.LocalFileIO; +import org.apache.paimon.options.Options; +import org.apache.paimon.types.DataType; +import org.apache.paimon.types.DataTypes; +import org.apache.paimon.types.RowType; + +import org.apache.paimon.shade.guava30.com.google.common.base.Objects; + +import org.apache.hadoop.conf.Configuration; +import org.apache.orc.Reader; +import org.apache.orc.TypeDescription; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.IOException; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.apache.paimon.format.orc.OrcFileFormat.refineDataType; +import static org.apache.paimon.format.orc.OrcTypeUtil.PAIMON_ORC_FIELD_ID_KEY; +import static org.apache.paimon.format.orc.OrcTypeUtil.convertToOrcSchema; +import static org.apache.paimon.format.orc.OrcTypeUtil.convertToOrcType; +import static org.apache.paimon.utils.Preconditions.checkArgument; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatNoException; + +/** Test for {@link OrcTypeUtil}. */ +class OrcTypeUtilTest { + + @Test + void testDataTypeToOrcType() { + test("boolean", DataTypes.BOOLEAN()); + test("char(123)", DataTypes.CHAR(123)); + test("varchar(123)", DataTypes.VARCHAR(123)); + test("string", DataTypes.STRING()); + test("binary", DataTypes.BYTES()); + test("tinyint", DataTypes.TINYINT()); + test("smallint", DataTypes.SMALLINT()); + test("int", DataTypes.INT()); + test("bigint", DataTypes.BIGINT()); + test("float", DataTypes.FLOAT()); + test("double", DataTypes.DOUBLE()); + test("date", DataTypes.DATE()); + test("timestamp", DataTypes.TIMESTAMP()); + test("array", DataTypes.ARRAY(DataTypes.FLOAT())); + test("map", DataTypes.MAP(DataTypes.FLOAT(), DataTypes.BIGINT())); + test( + "struct>", + DataTypes.ROW( + DataTypes.FIELD(0, "int0", DataTypes.INT()), + DataTypes.FIELD(1, "str1", DataTypes.STRING()), + DataTypes.FIELD(2, "double2", DataTypes.DOUBLE()), + DataTypes.FIELD( + 3, + "row3", + DataTypes.ROW( + DataTypes.FIELD(4, "int0", DataTypes.INT()), + DataTypes.FIELD(5, "int1", DataTypes.INT()))))); + test("decimal(4,2)", DataTypes.DECIMAL(4, 2)); + } + + private void test(String expected, DataType type) { + assertThat(convertToOrcType(type, -1, -1)).hasToString(expected); + } + + @Test + void testFieldIdAttribute(@TempDir java.nio.file.Path tempPath) throws IOException { + RowType rowType = + RowType.builder() + .field("a", DataTypes.INT()) + .field( + "b", + RowType.builder(true, new AtomicInteger(10)) + .field("f0", DataTypes.STRING()) + .field("f1", DataTypes.INT()) + .build()) + .field("c", DataTypes.ARRAY(DataTypes.INT())) + .field("d", DataTypes.MAP(DataTypes.INT(), DataTypes.STRING())) + .field( + "e", + DataTypes.ARRAY( + RowType.builder(true, new AtomicInteger(20)) + .field("f0", DataTypes.STRING()) + .field("f1", DataTypes.INT()) + .build())) + .field( + "f", + RowType.builder(true, new AtomicInteger(30)) + .field("f0", DataTypes.ARRAY(DataTypes.INT())) + .build()) + .build(); + + // write schema to orc file then get + FileIO fileIO = LocalFileIO.create(); + Path tempFile = new Path(new Path(tempPath.toUri()), UUID.randomUUID().toString()); + + OrcFileFormat format = + new OrcFileFormat(new FileFormatFactory.FormatContext(new Options(), 1024, 1024)); + PositionOutputStream out = fileIO.newOutputStream(tempFile, false); + FormatWriter writer = format.createWriterFactory(rowType).create(out, "zstd"); + writer.close(); + out.close(); + + Reader orcReader = + OrcReaderFactory.createReader(new Configuration(), fileIO, tempFile, null); + TypeDescription orcSchema = orcReader.getSchema(); + + RowType refined = (RowType) refineDataType(rowType); + + assertThatNoException() + .isThrownBy(() -> checkStruct(convertToOrcSchema(refined), orcSchema)); + + assertThatNoException() + .isThrownBy( + () -> + checkStruct( + convertToOrcSchema(refined.project("c", "b", "d")), + orcSchema)); + + assertThatNoException() + .isThrownBy( + () -> + checkStruct( + convertToOrcSchema(refined.project("a", "e", "f")), + orcSchema)); + } + + private void checkStruct(TypeDescription requiredStruct, TypeDescription orcStruct) { + List requiredFields = requiredStruct.getFieldNames(); + List requiredTypes = requiredStruct.getChildren(); + List orcFields = orcStruct.getFieldNames(); + List orcTypes = orcStruct.getChildren(); + + for (int i = 0; i < requiredFields.size(); i++) { + String field = requiredFields.get(i); + int orcIndex = orcFields.indexOf(field); + checkArgument(orcIndex != -1, "Cannot find field %s in orc file meta.", field); + TypeDescription requiredType = requiredTypes.get(i); + TypeDescription orcType = orcTypes.get(orcIndex); + checkField(field, requiredType, orcType); + } + } + + private void checkField( + String fieldName, TypeDescription requiredType, TypeDescription orcType) { + checkFieldIdAttribute(fieldName, requiredType, orcType); + if (requiredType.getCategory().isPrimitive()) { + return; + } + + switch (requiredType.getCategory()) { + case LIST: + checkField( + "_elem", requiredType.getChildren().get(0), orcType.getChildren().get(0)); + return; + case MAP: + checkField("_key", requiredType.getChildren().get(0), orcType.getChildren().get(0)); + checkField( + "_value", requiredType.getChildren().get(1), orcType.getChildren().get(1)); + return; + case STRUCT: + checkStruct(requiredType, orcType); + return; + default: + throw new UnsupportedOperationException("Unsupported orc type: " + requiredType); + } + } + + private void checkFieldIdAttribute( + String fieldName, TypeDescription requiredType, TypeDescription orcType) { + String requiredId = requiredType.getAttributeValue(PAIMON_ORC_FIELD_ID_KEY); + String orcId = orcType.getAttributeValue(PAIMON_ORC_FIELD_ID_KEY); + checkArgument( + Objects.equal(requiredId, orcId), + "Field %s has different id: read type id is %s but orc type id is %s. This is unexpected.", + fieldName, + requiredId, + orcId); + } +} diff --git a/paimon-format/src/test/java/org/apache/paimon/format/orc/OrcWriterFactoryTest.java b/paimon-format/src/test/java/org/apache/paimon/format/orc/OrcWriterFactoryTest.java index 2511d7ed7a9e..52df5afb4efd 100644 --- a/paimon-format/src/test/java/org/apache/paimon/format/orc/OrcWriterFactoryTest.java +++ b/paimon-format/src/test/java/org/apache/paimon/format/orc/OrcWriterFactoryTest.java @@ -28,6 +28,7 @@ import org.apache.hadoop.fs.Path; import org.apache.orc.MemoryManager; import org.apache.orc.OrcFile; +import org.apache.orc.TypeDescription; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -47,7 +48,7 @@ void testNotOverrideInMemoryManager(@TempDir java.nio.file.Path tmpDir) throws I OrcWriterFactory factory = new TestOrcWriterFactory( new RowDataVectorizer( - "struct<_col0:string,_col1:int>", + TypeDescription.fromString("struct<_col0:string,_col1:int>"), new DataType[] {DataTypes.STRING(), DataTypes.INT()}), memoryManager); factory.create(new LocalPositionOutputStream(tmpDir.resolve("file1").toFile()), "LZ4"); diff --git a/paimon-format/src/test/java/org/apache/paimon/format/parquet/ParquetFormatReadWriteTest.java b/paimon-format/src/test/java/org/apache/paimon/format/parquet/ParquetFormatReadWriteTest.java index d5338b1e78be..e0d1d240a9fd 100644 --- a/paimon-format/src/test/java/org/apache/paimon/format/parquet/ParquetFormatReadWriteTest.java +++ b/paimon-format/src/test/java/org/apache/paimon/format/parquet/ParquetFormatReadWriteTest.java @@ -18,10 +18,27 @@ package org.apache.paimon.format.parquet; +import org.apache.paimon.data.GenericRow; import org.apache.paimon.format.FileFormat; import org.apache.paimon.format.FileFormatFactory; import org.apache.paimon.format.FormatReadWriteTest; +import org.apache.paimon.format.FormatWriter; +import org.apache.paimon.fs.PositionOutputStream; import org.apache.paimon.options.Options; +import org.apache.paimon.types.DataTypes; +import org.apache.paimon.types.RowType; + +import org.apache.parquet.column.values.bloomfilter.BloomFilter; +import org.apache.parquet.hadoop.ParquetFileReader; +import org.apache.parquet.hadoop.metadata.BlockMetaData; +import org.apache.parquet.hadoop.metadata.ColumnChunkMetaData; +import org.apache.parquet.hadoop.metadata.ParquetMetadata; +import org.assertj.core.api.Assertions; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import java.util.List; +import java.util.concurrent.ThreadLocalRandom; /** A parquet {@link FormatReadWriteTest}. */ public class ParquetFormatReadWriteTest extends FormatReadWriteTest { @@ -35,4 +52,39 @@ protected FileFormat fileFormat() { return new ParquetFileFormat( new FileFormatFactory.FormatContext(new Options(), 1024, 1024)); } + + @ParameterizedTest + @ValueSource(booleans = {true, false}) + public void testEnableBloomFilter(boolean enabled) throws Exception { + Options options = new Options(); + options.set("parquet.bloom.filter.enabled", String.valueOf(enabled)); + ParquetFileFormat format = + new ParquetFileFormat(new FileFormatFactory.FormatContext(options, 1024, 1024)); + + RowType rowType = DataTypes.ROW(DataTypes.INT().notNull(), DataTypes.BIGINT()); + + if (ThreadLocalRandom.current().nextBoolean()) { + rowType = (RowType) rowType.notNull(); + } + + PositionOutputStream out = fileIO.newOutputStream(file, false); + FormatWriter writer = format.createWriterFactory(rowType).create(out, "zstd"); + writer.addElement(GenericRow.of(1, 1L)); + writer.addElement(GenericRow.of(2, 2L)); + writer.addElement(GenericRow.of(3, null)); + writer.close(); + out.close(); + + try (ParquetFileReader reader = ParquetUtil.getParquetReader(fileIO, file, null)) { + ParquetMetadata parquetMetadata = reader.getFooter(); + List blockMetaDataList = parquetMetadata.getBlocks(); + for (BlockMetaData blockMetaData : blockMetaDataList) { + List columnChunkMetaDataList = blockMetaData.getColumns(); + for (ColumnChunkMetaData columnChunkMetaData : columnChunkMetaDataList) { + BloomFilter filter = reader.readBloomFilter(columnChunkMetaData); + Assertions.assertThat(enabled == (filter != null)).isTrue(); + } + } + } + } } diff --git a/paimon-format/src/test/java/org/apache/paimon/format/parquet/ParquetReadWriteTest.java b/paimon-format/src/test/java/org/apache/paimon/format/parquet/ParquetReadWriteTest.java index 0ccf3fe30842..ffe4d6008296 100644 --- a/paimon-format/src/test/java/org/apache/paimon/format/parquet/ParquetReadWriteTest.java +++ b/paimon-format/src/test/java/org/apache/paimon/format/parquet/ParquetReadWriteTest.java @@ -40,6 +40,7 @@ import org.apache.paimon.types.BooleanType; import org.apache.paimon.types.DataField; import org.apache.paimon.types.DataType; +import org.apache.paimon.types.DataTypes; import org.apache.paimon.types.DecimalType; import org.apache.paimon.types.DoubleType; import org.apache.paimon.types.FloatType; @@ -61,7 +62,13 @@ import org.apache.parquet.hadoop.ParquetWriter; import org.apache.parquet.hadoop.example.ExampleParquetWriter; import org.apache.parquet.hadoop.util.HadoopOutputFile; +import org.apache.parquet.schema.ConversionPatterns; +import org.apache.parquet.schema.GroupType; +import org.apache.parquet.schema.LogicalTypeAnnotation; import org.apache.parquet.schema.MessageType; +import org.apache.parquet.schema.PrimitiveType; +import org.apache.parquet.schema.Type; +import org.apache.parquet.schema.Types; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.RepeatedTest; import org.junit.jupiter.api.Test; @@ -89,6 +96,8 @@ import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; +import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.INT32; +import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.INT64; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -470,6 +479,78 @@ public void testNestedNullMapKey() { .isInstanceOf(RuntimeException.class); } + @Test + public void testConvertToParquetTypeWithId() { + List nestedFields = + Arrays.asList( + new DataField(3, "v1", DataTypes.INT()), + new DataField(4, "v2", DataTypes.STRING())); + List fields = + Arrays.asList( + new DataField(0, "a", DataTypes.INT()), + new DataField(1, "b", DataTypes.ARRAY(DataTypes.STRING())), + new DataField( + 2, + "c", + DataTypes.MAP( + DataTypes.INT(), + DataTypes.MAP( + DataTypes.BIGINT(), new RowType(nestedFields))))); + RowType rowType = new RowType(fields); + + int baseId = 536870911; + int depthLimit = 1 << 10; + Type innerMapValueType = + new GroupType( + Type.Repetition.OPTIONAL, + "value", + Types.primitive(INT32, Type.Repetition.OPTIONAL) + .named("v1") + .withId(3), + Types.primitive( + PrimitiveType.PrimitiveTypeName.BINARY, + Type.Repetition.OPTIONAL) + .as(LogicalTypeAnnotation.stringType()) + .named("v2") + .withId(4)) + .withId(baseId + depthLimit * 2 + 2); + Type outerMapValueType = + ConversionPatterns.mapType( + Type.Repetition.OPTIONAL, + "value", + "key_value", + Types.primitive(INT64, Type.Repetition.REQUIRED) + .named("key") + .withId(baseId - depthLimit * 2 - 2), + innerMapValueType) + .withId(baseId + depthLimit * 2 + 1); + Type expected = + new MessageType( + "table", + Types.primitive(INT32, Type.Repetition.OPTIONAL).named("a").withId(0), + ConversionPatterns.listOfElements( + Type.Repetition.OPTIONAL, + "b", + Types.primitive( + PrimitiveType.PrimitiveTypeName.BINARY, + Type.Repetition.OPTIONAL) + .as(LogicalTypeAnnotation.stringType()) + .named("element") + .withId(baseId + depthLimit + 1)) + .withId(1), + ConversionPatterns.mapType( + Type.Repetition.OPTIONAL, + "c", + "key_value", + Types.primitive(INT32, Type.Repetition.REQUIRED) + .named("key") + .withId(baseId - depthLimit * 2 - 1), + outerMapValueType) + .withId(2)); + Type actual = ParquetSchemaConverter.convertToParquetMessageType("table", rowType); + assertThat(actual).isEqualTo(expected); + } + private void innerTestTypes(File folder, List records, int rowGroupSize) throws IOException { List rows = records.stream().map(this::newRow).collect(Collectors.toList()); diff --git a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveAlterTableUtils.java b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveAlterTableUtils.java new file mode 100644 index 000000000000..8f77499486fd --- /dev/null +++ b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveAlterTableUtils.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.hive; + +import org.apache.paimon.catalog.Identifier; + +import org.apache.hadoop.hive.common.StatsSetupConst; +import org.apache.hadoop.hive.metastore.IMetaStoreClient; +import org.apache.hadoop.hive.metastore.api.EnvironmentContext; +import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.thrift.TException; + +/** Utils for hive alter table. */ +public class HiveAlterTableUtils { + + public static void alterTable(IMetaStoreClient client, Identifier identifier, Table table) + throws TException { + try { + alterTableWithEnv(client, identifier, table); + } catch (NoClassDefFoundError | NoSuchMethodError e) { + alterTableWithoutEnv(client, identifier, table); + } + } + + private static void alterTableWithEnv( + IMetaStoreClient client, Identifier identifier, Table table) throws TException { + EnvironmentContext environmentContext = new EnvironmentContext(); + environmentContext.putToProperties(StatsSetupConst.CASCADE, "true"); + environmentContext.putToProperties(StatsSetupConst.DO_NOT_UPDATE_STATS, "false"); + client.alter_table_with_environmentContext( + identifier.getDatabaseName(), identifier.getTableName(), table, environmentContext); + } + + private static void alterTableWithoutEnv( + IMetaStoreClient client, Identifier identifier, Table table) throws TException { + client.alter_table(identifier.getDatabaseName(), identifier.getTableName(), table, true); + } +} diff --git a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java index 4d9c482a20da..c74ede981546 100644 --- a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java +++ b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java @@ -40,18 +40,25 @@ import org.apache.paimon.schema.SchemaChange; import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.schema.TableSchema; +import org.apache.paimon.table.CatalogEnvironment; import org.apache.paimon.table.CatalogTableType; import org.apache.paimon.table.FileStoreTable; +import org.apache.paimon.table.FileStoreTableFactory; import org.apache.paimon.table.FormatTable; import org.apache.paimon.types.DataField; import org.apache.paimon.types.DataTypes; import org.apache.paimon.types.RowType; +import org.apache.paimon.utils.Pair; +import org.apache.paimon.utils.Preconditions; +import org.apache.paimon.view.View; +import org.apache.paimon.view.ViewImpl; import org.apache.flink.table.hive.LegacyHiveClasses; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.IMetaStoreClient; +import org.apache.hadoop.hive.metastore.TableType; import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; @@ -77,7 +84,6 @@ import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; -import java.util.Locale; import java.util.Map; import java.util.Optional; import java.util.Set; @@ -85,20 +91,24 @@ import java.util.stream.Collectors; import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.METASTOREWAREHOUSE; +import static org.apache.hadoop.hive.serde.serdeConstants.FIELD_DELIM; import static org.apache.paimon.CoreOptions.FILE_FORMAT; +import static org.apache.paimon.CoreOptions.PARTITION_EXPIRATION_TIME; import static org.apache.paimon.CoreOptions.TYPE; import static org.apache.paimon.TableType.FORMAT_TABLE; import static org.apache.paimon.hive.HiveCatalogLock.acquireTimeout; import static org.apache.paimon.hive.HiveCatalogLock.checkMaxSleep; -import static org.apache.paimon.hive.HiveCatalogOptions.FORMAT_TABLE_ENABLED; import static org.apache.paimon.hive.HiveCatalogOptions.HADOOP_CONF_DIR; import static org.apache.paimon.hive.HiveCatalogOptions.HIVE_CONF_DIR; import static org.apache.paimon.hive.HiveCatalogOptions.IDENTIFIER; import static org.apache.paimon.hive.HiveCatalogOptions.LOCATION_IN_PROPERTIES; +import static org.apache.paimon.hive.HiveTableUtils.convertToFormatTable; import static org.apache.paimon.options.CatalogOptions.ALLOW_UPPER_CASE; +import static org.apache.paimon.options.CatalogOptions.FORMAT_TABLE_ENABLED; import static org.apache.paimon.options.CatalogOptions.SYNC_ALL_PROPERTIES; import static org.apache.paimon.options.CatalogOptions.TABLE_TYPE; import static org.apache.paimon.options.OptionsUtils.convertToPropertiesPrefixKey; +import static org.apache.paimon.table.FormatTableOptions.FIELD_DELIMITER; import static org.apache.paimon.utils.BranchManager.DEFAULT_MAIN_BRANCH; import static org.apache.paimon.utils.HadoopUtils.addHadoopConfIfFound; import static org.apache.paimon.utils.Preconditions.checkArgument; @@ -111,7 +121,7 @@ public class HiveCatalog extends AbstractCatalog { // Reserved properties public static final String TABLE_TYPE_PROP = "table_type"; - public static final String PAIMON_TABLE_TYPE_VALUE = "paimon"; + public static final String PAIMON_TABLE_IDENTIFIER = "PAIMON"; // we don't include paimon-hive-connector as dependencies because it depends on // hive-exec @@ -124,6 +134,7 @@ public class HiveCatalog extends AbstractCatalog { "org.apache.paimon.hive.PaimonStorageHandler"; private static final String HIVE_PREFIX = "hive."; public static final String HIVE_SITE_FILE = "hive-site.xml"; + private static final String HIVE_EXTERNAL_TABLE_PROP = "EXTERNAL"; private final HiveConf hiveConf; private final String clientClassName; @@ -163,8 +174,8 @@ public HiveCatalog( this.clients = new CachedClientPool(hiveConf, options, clientClassName); } - private boolean formatTableEnabled() { - return options.get(FORMAT_TABLE_ENABLED); + private boolean formatTableDisabled() { + return !options.get(FORMAT_TABLE_ENABLED); } @Override @@ -180,35 +191,48 @@ public Optional lockContext() { } @Override - public Optional metastoreClientFactory(Identifier identifier) { + public Optional metastoreClientFactory( + Identifier identifier, TableSchema schema) { Identifier tableIdentifier = new Identifier(identifier.getDatabaseName(), identifier.getTableName()); - try { - return Optional.of( - new HiveMetastoreClient.Factory( - tableIdentifier, - getDataTableSchema(tableIdentifier), - hiveConf, - clientClassName, - options)); - } catch (TableNotExistException e) { - throw new RuntimeException( - "Table " + identifier + " does not exist. This is unexpected.", e); - } + return Optional.of( + new HiveMetastoreClient.Factory( + tableIdentifier, schema, hiveConf, clientClassName, options)); } @Override public Path getTableLocation(Identifier identifier) { + Table table = null; + try { + table = getHmsTable(identifier); + } catch (TableNotExistException ignored) { + } + return getTableLocation(identifier, table); + } + + private Pair initialTableLocation( + Map tableOptions, Identifier identifier) { + boolean externalTable; + Path location; + if (tableOptions.containsKey(CoreOptions.PATH.key())) { + externalTable = true; + location = new Path(tableOptions.get(CoreOptions.PATH.key())); + } else { + externalTable = usingExternalTable(tableOptions); + location = getTableLocation(identifier, null); + } + return Pair.of(location, externalTable); + } + + private Path getTableLocation(Identifier identifier, @Nullable Table table) { try { String databaseName = identifier.getDatabaseName(); String tableName = identifier.getTableName(); Optional tablePath = clients.run( client -> { - if (client.tableExists(databaseName, tableName)) { - String location = - locationHelper.getTableLocation( - client.getTable(databaseName, tableName)); + if (table != null) { + String location = locationHelper.getTableLocation(table); if (location != null) { return Optional.of(new Path(location)); } @@ -273,6 +297,8 @@ private Database convertToHiveDatabase(String name, Map properti (key, value) -> { if (key.equals(COMMENT_PROP)) { database.setDescription(value); + } else if (key.equals(OWNER_PROP)) { + database.setOwnerName(value); } else if (key.equals(DB_LOCATION_PROP)) { database.setLocationUri(value); } else if (value != null) { @@ -284,10 +310,21 @@ private Database convertToHiveDatabase(String name, Map properti } @Override - public Map loadDatabasePropertiesImpl(String name) + public org.apache.paimon.catalog.Database getDatabaseImpl(String name) throws DatabaseNotExistException { try { - return convertToProperties(clients.run(client -> client.getDatabase(name))); + Database database = clients.run(client -> client.getDatabase(name)); + Map options = new HashMap<>(database.getParameters()); + if (database.getDescription() != null) { + options.put(COMMENT_PROP, database.getDescription()); + } + if (database.getOwnerName() != null) { + options.put(OWNER_PROP, database.getOwnerName()); + } + if (database.getLocationUri() != null) { + options.put(DB_LOCATION_PROP, database.getLocationUri()); + } + return org.apache.paimon.catalog.Database.of(name, options, database.getDescription()); } catch (NoSuchObjectException e) { throw new DatabaseNotExistException(name); } catch (TException e) { @@ -299,17 +336,6 @@ public Map loadDatabasePropertiesImpl(String name) } } - private Map convertToProperties(Database database) { - Map properties = new HashMap<>(database.getParameters()); - if (database.getLocationUri() != null) { - properties.put(DB_LOCATION_PROP, database.getLocationUri()); - } - if (database.getDescription() != null) { - properties.put(COMMENT_PROP, database.getDescription()); - } - return properties; - } - @Override public void dropPartition(Identifier identifier, Map partitionSpec) throws TableNotExistException { @@ -382,16 +408,20 @@ protected void dropDatabaseImpl(String name) { @Override protected List listTablesImpl(String databaseName) { try { - return clients.run( - client -> - client.getAllTables(databaseName).stream() - .filter( - tableName -> { - Identifier identifier = - new Identifier(databaseName, tableName); - return tableExists(identifier); - }) - .collect(Collectors.toList())); + List allTables = clients.run(client -> client.getAllTables(databaseName)); + List result = new ArrayList<>(allTables.size()); + for (String t : allTables) { + try { + Identifier identifier = new Identifier(databaseName, t); + Table table = getHmsTable(identifier); + if (isPaimonTable(identifier, table) + || (!formatTableDisabled() && isFormatTable(table))) { + result.add(t); + } + } catch (TableNotExistException ignored) { + } + } + return result; } catch (TException e) { throw new RuntimeException("Failed to list all tables in database " + databaseName, e); } catch (InterruptedException e) { @@ -401,102 +431,203 @@ protected List listTablesImpl(String databaseName) { } @Override - public boolean tableExists(Identifier identifier) { - if (isSystemTable(identifier)) { - return super.tableExists(identifier); + protected TableMeta getDataTableMeta(Identifier identifier) throws TableNotExistException { + return getDataTableMeta(identifier, getHmsTable(identifier)); + } + + private TableMeta getDataTableMeta(Identifier identifier, Table table) + throws TableNotExistException { + return new TableMeta( + getDataTableSchema(identifier, table), + identifier.getFullName() + "." + table.getCreateTime()); + } + + @Override + public TableSchema getDataTableSchema(Identifier identifier) throws TableNotExistException { + Table table = getHmsTable(identifier); + return getDataTableSchema(identifier, table); + } + + private TableSchema getDataTableSchema(Identifier identifier, Table table) + throws TableNotExistException { + if (!isPaimonTable(identifier, table)) { + throw new TableNotExistException(identifier); } + return tableSchemaInFileSystem( + getTableLocation(identifier, table), identifier.getBranchNameOrDefault()) + .orElseThrow(() -> new TableNotExistException(identifier)); + } + + @Override + public View getView(Identifier identifier) throws ViewNotExistException { Table table; try { - table = - clients.run( - client -> - client.getTable( - identifier.getDatabaseName(), - identifier.getTableName())); - } catch (NoSuchObjectException e) { - return false; - } catch (TException e) { - throw new RuntimeException( - "Cannot determine if table " + identifier.getFullName() + " is a paimon table.", - e); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new RuntimeException( - "Interrupted in call to tableExists " + identifier.getFullName(), e); + table = getHmsTable(identifier); + } catch (TableNotExistException e) { + throw new ViewNotExistException(identifier); } - boolean isDataTable = - isPaimonTable(table) - && tableSchemaInFileSystem( - getTableLocation(identifier), - identifier.getBranchNameOrDefault()) - .isPresent(); - if (isDataTable) { - return true; + if (!isView(table)) { + throw new ViewNotExistException(identifier); } - if (formatTableEnabled()) { - try { - HiveFormatTableUtils.convertToFormatTable(table); - return true; - } catch (UnsupportedOperationException e) { - return false; + RowType rowType = HiveTableUtils.createRowType(table); + Map options = new HashMap<>(table.getParameters()); + String comment = options.remove(COMMENT_PROP); + return new ViewImpl(identifier, rowType, table.getViewExpandedText(), comment, options); + } + + @Override + public void createView(Identifier identifier, View view, boolean ignoreIfExists) + throws ViewAlreadyExistException, DatabaseNotExistException { + getDatabase(identifier.getDatabaseName()); + + try { + getView(identifier); + if (ignoreIfExists) { + return; } + throw new ViewAlreadyExistException(identifier); + } catch (ViewNotExistException ignored) { } - return false; - } + Table hiveTable = + org.apache.hadoop.hive.ql.metadata.Table.getEmptyTable( + identifier.getDatabaseName(), identifier.getObjectName()); + hiveTable.setCreateTime((int) (System.currentTimeMillis() / 1000)); - private static boolean isPaimonTable(Table table) { - boolean isPaimonTable = - INPUT_FORMAT_CLASS_NAME.equals(table.getSd().getInputFormat()) - && OUTPUT_FORMAT_CLASS_NAME.equals(table.getSd().getOutputFormat()); - return isPaimonTable || LegacyHiveClasses.isPaimonTable(table); + Map properties = new HashMap<>(view.options()); + // Table comment + if (view.comment().isPresent()) { + properties.put(COMMENT_PROP, view.comment().get()); + } + hiveTable.setParameters(properties); + hiveTable.setPartitionKeys(new ArrayList<>()); + hiveTable.setViewOriginalText(view.query()); + hiveTable.setViewExpandedText(view.query()); + hiveTable.setTableType(TableType.VIRTUAL_VIEW.name()); + + StorageDescriptor sd = hiveTable.getSd(); + List columns = + view.rowType().getFields().stream() + .map(this::convertToFieldSchema) + .collect(Collectors.toList()); + sd.setCols(columns); + + try { + clients.execute(client -> client.createTable(hiveTable)); + } catch (Exception e) { + // we don't need to delete directories since HMS will roll back db and fs if failed. + throw new RuntimeException("Failed to create table " + identifier.getFullName(), e); + } } @Override - public TableSchema getDataTableSchema(Identifier identifier) throws TableNotExistException { - if (!tableExists(identifier)) { - throw new TableNotExistException(identifier); + public void dropView(Identifier identifier, boolean ignoreIfNotExists) + throws ViewNotExistException { + try { + getView(identifier); + } catch (ViewNotExistException e) { + if (ignoreIfNotExists) { + return; + } + throw e; } - return tableSchemaInFileSystem( - getTableLocation(identifier), identifier.getBranchNameOrDefault()) - .orElseThrow(() -> new TableNotExistException(identifier)); + try { + clients.execute( + client -> + client.dropTable( + identifier.getDatabaseName(), + identifier.getTableName(), + false, + false, + false)); + } catch (TException e) { + throw new RuntimeException("Failed to drop view " + identifier.getFullName(), e); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException( + "Interrupted in call to drop view " + identifier.getFullName(), e); + } } @Override - public FormatTable getFormatTable(Identifier identifier) throws TableNotExistException { - if (!formatTableEnabled()) { - throw new TableNotExistException(identifier); + public List listViews(String databaseName) throws DatabaseNotExistException { + if (isSystemDatabase(databaseName)) { + return Collections.emptyList(); } + getDatabase(databaseName); - Table table; try { - table = - clients.run( - client -> - client.getTable( - identifier.getDatabaseName(), - identifier.getTableName())); - } catch (NoSuchObjectException e) { - throw new TableNotExistException(identifier); + return clients.run( + client -> client.getTables(databaseName, "*", TableType.VIRTUAL_VIEW)); } catch (TException e) { - throw new RuntimeException(e); + throw new RuntimeException("Failed to list views in database " + databaseName, e); } catch (InterruptedException e) { Thread.currentThread().interrupt(); - throw new RuntimeException(e); + throw new RuntimeException("Interrupted in call to getTables " + databaseName, e); } + } + + @Override + public void renameView(Identifier fromView, Identifier toView, boolean ignoreIfNotExists) + throws ViewNotExistException, ViewAlreadyExistException { + try { + getView(fromView); + } catch (ViewNotExistException e) { + if (ignoreIfNotExists) { + return; + } + throw new ViewNotExistException(fromView); + } + + try { + getView(toView); + throw new ViewAlreadyExistException(toView); + } catch (ViewNotExistException ignored) { + } + + renameHiveTable(fromView, toView); + } + + @Override + public org.apache.paimon.table.Table getDataOrFormatTable(Identifier identifier) + throws TableNotExistException { + Preconditions.checkArgument(identifier.getSystemTableName() == null); + Table table = getHmsTable(identifier); try { - return HiveFormatTableUtils.convertToFormatTable(table); + TableMeta tableMeta = getDataTableMeta(identifier, table); + return FileStoreTableFactory.create( + fileIO, + getTableLocation(identifier, table), + tableMeta.schema(), + new CatalogEnvironment( + identifier, + tableMeta.uuid(), + Lock.factory( + lockFactory().orElse(null), + lockContext().orElse(null), + identifier), + metastoreClientFactory(identifier, tableMeta.schema()).orElse(null))); + } catch (TableNotExistException ignore) { + } + + if (formatTableDisabled()) { + throw new TableNotExistException(identifier); + } + + try { + return convertToFormatTable(table); } catch (UnsupportedOperationException e) { throw new TableNotExistException(identifier); } } + @Override public void createFormatTable(Identifier identifier, Schema schema) { - if (!formatTableEnabled()) { + if (formatTableDisabled()) { throw new UnsupportedOperationException( "Format table is not enabled for " + identifier); } @@ -516,7 +647,10 @@ public void createFormatTable(Identifier identifier, Schema schema) { options, schema.comment()); try { - Table hiveTable = createHiveTable(identifier, newSchema); + Pair pair = initialTableLocation(schema.options(), identifier); + Path location = pair.getLeft(); + boolean externalTable = pair.getRight(); + Table hiveTable = createHiveFormatTable(identifier, newSchema, location, externalTable); clients.execute(client -> client.createTable(hiveTable)); } catch (Exception e) { // we don't need to delete directories since HMS will roll back db and fs if failed. @@ -524,29 +658,36 @@ public void createFormatTable(Identifier identifier, Schema schema) { } } - private boolean usingExternalTable() { + private boolean usingExternalTable(Map tableOptions) { CatalogTableType tableType = OptionsUtils.convertToEnum( hiveConf.get(TABLE_TYPE.key(), CatalogTableType.MANAGED.toString()), CatalogTableType.class); - return CatalogTableType.EXTERNAL.equals(tableType); + + String externalPropValue = + tableOptions.getOrDefault( + HIVE_EXTERNAL_TABLE_PROP.toLowerCase(), + tableOptions.get(HIVE_EXTERNAL_TABLE_PROP.toUpperCase())); + return CatalogTableType.EXTERNAL.equals(tableType) + || "TRUE".equalsIgnoreCase(externalPropValue); } @Override protected void dropTableImpl(Identifier identifier) { try { + boolean externalTable = isExternalTable(getHmsTable(identifier)); clients.execute( client -> client.dropTable( identifier.getDatabaseName(), identifier.getTableName(), - true, + !externalTable, false, true)); // When drop a Hive external table, only the hive metadata is deleted and the data files // are not deleted. - if (usingExternalTable()) { + if (externalTable) { return; } @@ -561,7 +702,7 @@ protected void dropTableImpl(Identifier identifier) { } catch (Exception ee) { LOG.error("Delete directory[{}] fail for table {}", path, identifier, ee); } - } catch (TException e) { + } catch (TException | TableNotExistException e) { throw new RuntimeException("Failed to drop table " + identifier.getFullName(), e); } catch (InterruptedException e) { Thread.currentThread().interrupt(); @@ -572,67 +713,88 @@ protected void dropTableImpl(Identifier identifier) { @Override protected void createTableImpl(Identifier identifier, Schema schema) { - // first commit changes to underlying files - // if changes on Hive fails there is no harm to perform the same changes to files again + Pair pair = initialTableLocation(schema.options(), identifier); + Path location = pair.getLeft(); + boolean externalTable = pair.getRight(); TableSchema tableSchema; try { - tableSchema = schemaManager(identifier).createTable(schema, usingExternalTable()); + tableSchema = schemaManager(identifier, location).createTable(schema, externalTable); } catch (Exception e) { - throw new RuntimeException( - "Failed to commit changes of table " - + identifier.getFullName() - + " to underlying files.", - e); + throw new RuntimeException("Failed to create table " + identifier.getFullName(), e); } try { - clients.execute(client -> client.createTable(createHiveTable(identifier, tableSchema))); + clients.execute( + client -> + client.createTable( + createHiveTable( + identifier, tableSchema, location, externalTable))); } catch (Exception e) { - Path path = getTableLocation(identifier); try { - fileIO.deleteDirectoryQuietly(path); + if (!externalTable) { + fileIO.deleteDirectoryQuietly(location); + } } catch (Exception ee) { - LOG.error("Delete directory[{}] fail for table {}", path, identifier, ee); + LOG.error("Delete directory[{}] fail for table {}", location, identifier, ee); } throw new RuntimeException("Failed to create table " + identifier.getFullName(), e); } } - private Table createHiveTable(Identifier identifier, TableSchema tableSchema) { + private Table createHiveTable( + Identifier identifier, TableSchema tableSchema, Path location, boolean externalTable) { + checkArgument(Options.fromMap(tableSchema.options()).get(TYPE) != FORMAT_TABLE); + Map tblProperties; - String provider = "paimon"; - if (Options.fromMap(tableSchema.options()).get(TYPE) == FORMAT_TABLE) { - provider = tableSchema.options().get(FILE_FORMAT.key()); - } - if (syncAllProperties() || !provider.equals("paimon")) { + if (syncAllProperties()) { tblProperties = new HashMap<>(tableSchema.options()); // add primary-key, partition-key to tblproperties tblProperties.putAll(convertToPropertiesTableKey(tableSchema)); } else { tblProperties = convertToPropertiesPrefixKey(tableSchema.options(), HIVE_PREFIX); + if (tableSchema.options().containsKey(PARTITION_EXPIRATION_TIME.key())) { + // This property will be stored in the 'table_params' table of the HMS database for + // querying by other engines or products. + tblProperties.put( + PARTITION_EXPIRATION_TIME.key(), + tableSchema.options().get(PARTITION_EXPIRATION_TIME.key())); + } } - Table table = newHmsTable(identifier, tblProperties, provider); - updateHmsTable(table, identifier, tableSchema, provider); + Table table = newHmsTable(identifier, tblProperties, null, externalTable); + updateHmsTable(table, identifier, tableSchema, null, location); + return table; + } + + private Table createHiveFormatTable( + Identifier identifier, TableSchema tableSchema, Path location, boolean externalTable) { + CoreOptions coreOptions = new CoreOptions(tableSchema.options()); + checkArgument(coreOptions.type() == FORMAT_TABLE); + + // file.format option has a default value and cannot be empty. + FormatTable.Format provider = FormatTable.parseFormat(coreOptions.formatType()); + + Map tblProperties = new HashMap<>(); + + Table table = newHmsTable(identifier, tblProperties, provider, externalTable); + updateHmsTable(table, identifier, tableSchema, provider, location); + return table; } @Override protected void renameTableImpl(Identifier fromTable, Identifier toTable) { try { - String fromDB = fromTable.getDatabaseName(); - String fromTableName = fromTable.getTableName(); - Table table = clients.run(client -> client.getTable(fromDB, fromTableName)); - table.setDbName(toTable.getDatabaseName()); - table.setTableName(toTable.getTableName()); - clients.execute(client -> client.alter_table(fromDB, fromTableName, table)); - + // Get fromTable's location before rename Path fromPath = getTableLocation(fromTable); - if (!new SchemaManager(fileIO, fromPath).listAllIds().isEmpty()) { + Table table = renameHiveTable(fromTable, toTable); + Path toPath = getTableLocation(toTable); + if (!isExternalTable(table) + && !fromPath.equals(toPath) + && !new SchemaManager(fileIO, fromPath).listAllIds().isEmpty()) { // Rename the file system's table directory. Maintain consistency between tables in // the file system and tables in the Hive Metastore. - Path toPath = getTableLocation(toTable); try { fileIO.rename(fromPath, toPath); } catch (IOException e) { @@ -658,10 +820,33 @@ protected void renameTableImpl(Identifier fromTable, Identifier toTable) { } } + private Table renameHiveTable(Identifier fromTable, Identifier toTable) { + try { + String fromDB = fromTable.getDatabaseName(); + String fromTableName = fromTable.getTableName(); + Table table = clients.run(client -> client.getTable(fromDB, fromTableName)); + table.setDbName(toTable.getDatabaseName()); + table.setTableName(toTable.getTableName()); + clients.execute(client -> client.alter_table(fromDB, fromTableName, table)); + + return table; + } catch (TException e) { + throw new RuntimeException("Failed to rename table " + fromTable.getFullName(), e); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException("Interrupted in call to renameTable", e); + } + } + @Override protected void alterTableImpl(Identifier identifier, List changes) throws TableNotExistException, ColumnAlreadyExistException, ColumnNotExistException { - final SchemaManager schemaManager = schemaManager(identifier); + Table table = getHmsTable(identifier); + if (!isPaimonTable(identifier, table)) { + throw new UnsupportedOperationException("Only data table support alter table."); + } + + final SchemaManager schemaManager = schemaManager(identifier, getTableLocation(identifier)); // first commit changes to underlying files TableSchema schema = schemaManager.commitChanges(changes); @@ -670,12 +855,6 @@ protected void alterTableImpl(Identifier identifier, List changes) return; } try { - Table table = - clients.run( - client -> - client.getTable( - identifier.getDatabaseName(), - identifier.getTableName())); alterTableToHms(table, identifier, schema); } catch (Exception te) { schemaManager.deleteSchema(schema.id()); @@ -686,14 +865,10 @@ protected void alterTableImpl(Identifier identifier, List changes) private void alterTableToHms(Table table, Identifier identifier, TableSchema newSchema) throws TException, InterruptedException { updateHmsTablePars(table, newSchema); - updateHmsTable(table, identifier, newSchema, newSchema.options().get("provider")); - clients.execute( - client -> - client.alter_table( - identifier.getDatabaseName(), - identifier.getTableName(), - table, - true)); + Path location = getTableLocation(identifier, table); + // file format is null, because only data table support alter table. + updateHmsTable(table, identifier, newSchema, null, location); + clients.execute(client -> HiveAlterTableUtils.alterTable(client, identifier, table)); } @Override @@ -701,13 +876,18 @@ public boolean allowUpperCase() { return catalogOptions.getOptional(ALLOW_UPPER_CASE).orElse(false); } + @Override + protected boolean allowCustomTablePath() { + return true; + } + public boolean syncAllProperties() { return catalogOptions.get(SYNC_ALL_PROPERTIES); } @Override public void repairCatalog() { - List databases = null; + List databases; try { databases = listDatabasesInFileSystem(new Path(warehouse)); } catch (IOException e) { @@ -723,7 +903,9 @@ public void repairDatabase(String databaseName) { checkNotSystemDatabase(databaseName); // create database if needed - if (!databaseExists(databaseName)) { + try { + getDatabase(databaseName); + } catch (DatabaseNotExistException e) { createDatabaseImpl(databaseName, Collections.emptyMap()); } @@ -751,19 +933,17 @@ public void repairTable(Identifier identifier) throws TableNotExistException { checkNotSystemTable(identifier, "repairTable"); validateIdentifierNameCaseInsensitive(identifier); + Path location = getTableLocation(identifier); TableSchema tableSchema = - tableSchemaInFileSystem( - getTableLocation(identifier), identifier.getBranchNameOrDefault()) + tableSchemaInFileSystem(location, identifier.getBranchNameOrDefault()) .orElseThrow(() -> new TableNotExistException(identifier)); - Table newTable = createHiveTable(identifier, tableSchema); + try { + Table newTable = null; try { - Table table = - clients.run( - client -> - client.getTable( - identifier.getDatabaseName(), - identifier.getTableName())); + Table table = getHmsTable(identifier); + newTable = + createHiveTable(identifier, tableSchema, location, isExternalTable(table)); checkArgument( isPaimonTable(table), "Table %s is not a paimon table in hive metastore.", @@ -772,9 +952,18 @@ public void repairTable(Identifier identifier) throws TableNotExistException { || !newTable.getParameters().equals(table.getParameters())) { alterTableToHms(table, identifier, tableSchema); } - } catch (NoSuchObjectException e) { + } catch (TableNotExistException e) { // hive table does not exist. - clients.execute(client -> client.createTable(newTable)); + if (newTable == null) { + newTable = + createHiveTable( + identifier, + tableSchema, + location, + usingExternalTable(tableSchema.options())); + } + Table finalNewTable = newTable; + clients.execute(client -> client.createTable(finalNewTable)); } // repair partitions @@ -804,16 +993,61 @@ public String warehouse() { return warehouse; } + public Table getHmsTable(Identifier identifier) throws TableNotExistException { + try { + return clients.run( + client -> + client.getTable( + identifier.getDatabaseName(), identifier.getTableName())); + } catch (NoSuchObjectException e) { + throw new TableNotExistException(identifier); + } catch (TException e) { + throw new RuntimeException( + "Cannot determine if table " + identifier.getFullName() + " is a paimon table.", + e); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException( + "Interrupted in call to tableExists " + identifier.getFullName(), e); + } + } + + private boolean isPaimonTable(Identifier identifier, Table table) { + return isPaimonTable(table) + && tableExistsInFileSystem( + getTableLocation(identifier, table), identifier.getBranchNameOrDefault()); + } + + private static boolean isPaimonTable(Table table) { + boolean isPaimonTable = + INPUT_FORMAT_CLASS_NAME.equals(table.getSd().getInputFormat()) + && OUTPUT_FORMAT_CLASS_NAME.equals(table.getSd().getOutputFormat()); + return isPaimonTable || LegacyHiveClasses.isPaimonTable(table); + } + + private boolean isFormatTable(Table table) { + try { + convertToFormatTable(table); + return true; + } catch (UnsupportedOperationException e) { + return false; + } + } + + public static boolean isView(Table table) { + return table != null && TableType.VIRTUAL_VIEW.name().equals(table.getTableType()); + } + + private boolean isExternalTable(Table table) { + return table != null && TableType.EXTERNAL_TABLE.name().equals(table.getTableType()); + } + private Table newHmsTable( - Identifier identifier, Map tableParameters, String provider) { + Identifier identifier, + Map tableParameters, + @Nullable FormatTable.Format provider, + boolean externalTable) { long currentTimeMillis = System.currentTimeMillis(); - CatalogTableType tableType = - OptionsUtils.convertToEnum( - hiveConf.get(TABLE_TYPE.key(), CatalogTableType.MANAGED.toString()), - CatalogTableType.class); - if (provider == null) { - provider = "paimon"; - } Table table = new Table( identifier.getTableName(), @@ -828,72 +1062,94 @@ private Table newHmsTable( tableParameters, null, null, - tableType.toString().toUpperCase(Locale.ROOT) + "_TABLE"); - table.getParameters().put(TABLE_TYPE_PROP, provider.toUpperCase()); - if ("paimon".equalsIgnoreCase(provider)) { + externalTable + ? TableType.EXTERNAL_TABLE.name() + : TableType.MANAGED_TABLE.name()); + + if (provider == null) { + // normal paimon table + table.getParameters().put(TABLE_TYPE_PROP, PAIMON_TABLE_IDENTIFIER); table.getParameters() .put(hive_metastoreConstants.META_TABLE_STORAGE, STORAGE_HANDLER_CLASS_NAME); } else { - table.getParameters().put(FILE_FORMAT.key(), provider.toLowerCase()); + // format table + table.getParameters().put(TABLE_TYPE_PROP, provider.name()); + table.getParameters().put(FILE_FORMAT.key(), provider.name().toLowerCase()); table.getParameters().put(TYPE.key(), FORMAT_TABLE.toString()); } - if (CatalogTableType.EXTERNAL.equals(tableType)) { - table.getParameters().put("EXTERNAL", "TRUE"); + + if (externalTable) { + table.getParameters().put(HIVE_EXTERNAL_TABLE_PROP, "TRUE"); } return table; } - private String getSerdeClassName(String provider) { - if (provider == null || provider.equalsIgnoreCase("paimon")) { - return SERDE_CLASS_NAME; - } else if (provider.equalsIgnoreCase("csv")) { - return "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"; - } else if (provider.equalsIgnoreCase("parquet")) { - return "org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe"; - } else if (provider.equalsIgnoreCase("orc")) { - return "org.apache.hadoop.hive.ql.io.orc.OrcSerde"; - } else { + private String getSerdeClassName(@Nullable FormatTable.Format provider) { + if (provider == null) { return SERDE_CLASS_NAME; } + switch (provider) { + case CSV: + return "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"; + case PARQUET: + return "org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe"; + case ORC: + return "org.apache.hadoop.hive.ql.io.orc.OrcSerde"; + } + return SERDE_CLASS_NAME; } - private String getInputFormatName(String provider) { - if (provider == null || provider.equalsIgnoreCase("paimon")) { - return INPUT_FORMAT_CLASS_NAME; - } else if (provider.equalsIgnoreCase("csv")) { - return "org.apache.hadoop.mapred.TextInputFormat"; - } else if (provider.equalsIgnoreCase("parquet")) { - return "org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat"; - } else if (provider.equalsIgnoreCase("orc")) { - return "org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"; - } else { + private String getInputFormatName(@Nullable FormatTable.Format provider) { + if (provider == null) { return INPUT_FORMAT_CLASS_NAME; } + switch (provider) { + case CSV: + return "org.apache.hadoop.mapred.TextInputFormat"; + case PARQUET: + return "org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat"; + case ORC: + return "org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"; + } + return INPUT_FORMAT_CLASS_NAME; } - private String getOutputFormatClassName(String provider) { - if (provider == null || provider.equalsIgnoreCase("paimon")) { - return OUTPUT_FORMAT_CLASS_NAME; - } else if (provider.equalsIgnoreCase("csv")) { - return "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat"; - } else if (provider.equalsIgnoreCase("parquet")) { - return "org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat"; - } else if (provider.equalsIgnoreCase("orc")) { - return "org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat"; - } else { + private String getOutputFormatClassName(@Nullable FormatTable.Format provider) { + if (provider == null) { return OUTPUT_FORMAT_CLASS_NAME; } + switch (provider) { + case CSV: + return "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat"; + case PARQUET: + return "org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat"; + case ORC: + return "org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat"; + } + return OUTPUT_FORMAT_CLASS_NAME; + } + + private Map setSerDeInfoParam(@Nullable FormatTable.Format provider) { + Map param = new HashMap<>(); + if (provider == FormatTable.Format.CSV) { + param.put(FIELD_DELIM, options.get(FIELD_DELIMITER)); + } + return param; } private void updateHmsTable( - Table table, Identifier identifier, TableSchema schema, String provider) { + Table table, + Identifier identifier, + TableSchema schema, + @Nullable FormatTable.Format provider, + Path location) { StorageDescriptor sd = table.getSd() != null ? table.getSd() : new StorageDescriptor(); sd.setInputFormat(getInputFormatName(provider)); sd.setOutputFormat(getOutputFormatClassName(provider)); SerDeInfo serDeInfo = sd.getSerdeInfo() != null ? sd.getSerdeInfo() : new SerDeInfo(); - serDeInfo.setParameters(new HashMap<>()); + serDeInfo.setParameters(setSerDeInfoParam(provider)); serDeInfo.setSerializationLib(getSerdeClassName(provider)); sd.setSerdeInfo(serDeInfo); @@ -943,7 +1199,10 @@ private void updateHmsTable( } // update location - locationHelper.specifyTableLocation(table, getTableLocation(identifier).toString()); + if (location == null) { + location = getTableLocation(identifier, table); + } + locationHelper.specifyTableLocation(table, location.toString()); } private void updateHmsTablePars(Table table, TableSchema schema) { @@ -989,9 +1248,8 @@ private FieldSchema convertToFieldSchema(DataField dataField) { dataField.description()); } - private SchemaManager schemaManager(Identifier identifier) { - return new SchemaManager( - fileIO, getTableLocation(identifier), identifier.getBranchNameOrDefault()) + private SchemaManager schemaManager(Identifier identifier, Path location) { + return new SchemaManager(fileIO, location, identifier.getBranchNameOrDefault()) .withLock(lock(identifier)); } @@ -1089,7 +1347,7 @@ public static Catalog createHiveCatalog(CatalogContext context) { return new HiveCatalog( fileIO, hiveConf, - options.get(HiveCatalogFactory.METASTORE_CLIENT_CLASS), + options.get(HiveCatalogOptions.METASTORE_CLIENT_CLASS), options, warehouse.toUri().toString()); } diff --git a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalogFactory.java b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalogFactory.java index 95da0037168c..eff06831dd4f 100644 --- a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalogFactory.java +++ b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalogFactory.java @@ -21,23 +21,12 @@ import org.apache.paimon.catalog.Catalog; import org.apache.paimon.catalog.CatalogContext; import org.apache.paimon.catalog.CatalogFactory; -import org.apache.paimon.options.ConfigOption; -import org.apache.paimon.options.ConfigOptions; import static org.apache.paimon.hive.HiveCatalogOptions.IDENTIFIER; /** Factory to create {@link HiveCatalog}. */ public class HiveCatalogFactory implements CatalogFactory { - public static final ConfigOption METASTORE_CLIENT_CLASS = - ConfigOptions.key("metastore.client.class") - .stringType() - .defaultValue("org.apache.hadoop.hive.metastore.HiveMetaStoreClient") - .withDescription( - "Class name of Hive metastore client.\n" - + "NOTE: This class must directly implements " - + "org.apache.hadoop.hive.metastore.IMetaStoreClient."); - @Override public String identifier() { return IDENTIFIER; diff --git a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalogOptions.java b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalogOptions.java index c74fa447ea46..ceab49836820 100644 --- a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalogOptions.java +++ b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalogOptions.java @@ -48,6 +48,15 @@ public final class HiveCatalogOptions { + "If not configured, try to load from 'HADOOP_CONF_DIR' or 'HADOOP_HOME' system environment.\n" + "Configure Priority: 1.from 'hadoop-conf-dir' 2.from HADOOP_CONF_DIR 3.from HADOOP_HOME/conf 4.HADOOP_HOME/etc/hadoop.\n"); + public static final ConfigOption METASTORE_CLIENT_CLASS = + ConfigOptions.key("metastore.client.class") + .stringType() + .defaultValue("org.apache.hadoop.hive.metastore.HiveMetaStoreClient") + .withDescription( + "Class name of Hive metastore client.\n" + + "NOTE: This class must directly implements " + + "org.apache.hadoop.hive.metastore.IMetaStoreClient."); + public static final ConfigOption LOCATION_IN_PROPERTIES = ConfigOptions.key("location-in-properties") .booleanType() @@ -85,14 +94,5 @@ public final class HiveCatalogOptions { + "E.g. specifying \"conf:a.b.c\" will add \"a.b.c\" to the key, and so that configurations with different default catalog wouldn't share the same client pool. Multiple conf elements can be specified.")) .build()); - public static final ConfigOption FORMAT_TABLE_ENABLED = - ConfigOptions.key("format-table.enabled") - .booleanType() - .defaultValue(false) - .withDescription( - "Whether to support format tables, format table corresponds to a regular Hive table, allowing read and write operations. " - + "However, during these processes, it does not connect to the metastore; hence, newly added partitions will not be reflected in" - + " the metastore and need to be manually added as separate partition operations."); - private HiveCatalogOptions() {} } diff --git a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveMetastoreClient.java b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveMetastoreClient.java index b006f09c99a4..885fa463e5a7 100644 --- a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveMetastoreClient.java +++ b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveMetastoreClient.java @@ -31,6 +31,7 @@ import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.IMetaStoreClient; +import org.apache.hadoop.hive.metastore.api.AlreadyExistsException; import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; import org.apache.hadoop.hive.metastore.api.Partition; import org.apache.hadoop.hive.metastore.api.PartitionEventType; @@ -40,6 +41,8 @@ import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; /** {@link MetastoreClient} for Hive tables. */ public class HiveMetastoreClient implements MetastoreClient { @@ -56,11 +59,13 @@ public class HiveMetastoreClient implements MetastoreClient { ClientPool clients) throws TException, InterruptedException { this.identifier = identifier; + CoreOptions options = new CoreOptions(schema.options()); this.partitionComputer = new InternalRowPartitionComputer( - new CoreOptions(schema.options()).partitionDefaultName(), + options.partitionDefaultName(), schema.logicalPartitionType(), - schema.partitionKeys().toArray(new String[0])); + schema.partitionKeys().toArray(new String[0]), + options.legacyPartitionName()); this.clients = clients; this.sd = @@ -78,36 +83,73 @@ public void addPartition(BinaryRow partition) throws Exception { addPartition(partitionComputer.generatePartValues(partition)); } + @Override + public void addPartitions(List partitions) throws Exception { + addPartitionsSpec( + partitions.stream() + .map(partitionComputer::generatePartValues) + .collect(Collectors.toList())); + } + @Override public void addPartition(LinkedHashMap partitionSpec) throws Exception { + Partition hivePartition = + toHivePartition(partitionSpec, (int) (System.currentTimeMillis() / 1000)); + clients.execute( + client -> { + try { + client.add_partition(hivePartition); + } catch (AlreadyExistsException ignore) { + } + }); + } + + @Override + public void addPartitionsSpec(List> partitionSpecsList) + throws Exception { + int currentTime = (int) (System.currentTimeMillis() / 1000); + List hivePartitions = + partitionSpecsList.stream() + .map(partitionSpec -> toHivePartition(partitionSpec, currentTime)) + .collect(Collectors.toList()); + clients.execute(client -> client.add_partitions(hivePartitions, true, false)); + } + + @Override + public void alterPartition( + LinkedHashMap partitionSpec, + Map parameters, + long modifyTime, + boolean ignoreIfNotExist) + throws Exception { List partitionValues = new ArrayList<>(partitionSpec.values()); + int currentTime = (int) (modifyTime / 1000); + Partition hivePartition; try { - clients.execute( - client -> - client.getPartition( - identifier.getDatabaseName(), - identifier.getTableName(), - partitionValues)); - // do nothing if the partition already exists + hivePartition = + clients.run( + client -> + client.getPartition( + identifier.getDatabaseName(), + identifier.getObjectName(), + partitionValues)); } catch (NoSuchObjectException e) { - // partition not found, create new partition - StorageDescriptor newSd = new StorageDescriptor(sd); - newSd.setLocation( - sd.getLocation() - + "/" - + PartitionPathUtils.generatePartitionPath(partitionSpec)); - - Partition hivePartition = new Partition(); - hivePartition.setDbName(identifier.getDatabaseName()); - hivePartition.setTableName(identifier.getTableName()); - hivePartition.setValues(partitionValues); - hivePartition.setSd(newSd); - int currentTime = (int) (System.currentTimeMillis() / 1000); - hivePartition.setCreateTime(currentTime); - hivePartition.setLastAccessTime(currentTime); - - clients.execute(client -> client.add_partition(hivePartition)); + if (ignoreIfNotExist) { + return; + } else { + throw e; + } } + + hivePartition.setValues(partitionValues); + hivePartition.setLastAccessTime(currentTime); + hivePartition.getParameters().putAll(parameters); + clients.execute( + client -> + client.alter_partition( + identifier.getDatabaseName(), + identifier.getObjectName(), + hivePartition)); } @Override @@ -150,6 +192,21 @@ public IMetaStoreClient client() throws TException, InterruptedException { return clients.run(client -> client); } + private Partition toHivePartition( + LinkedHashMap partitionSpec, int currentTime) { + Partition hivePartition = new Partition(); + StorageDescriptor newSd = new StorageDescriptor(sd); + newSd.setLocation( + sd.getLocation() + "/" + PartitionPathUtils.generatePartitionPath(partitionSpec)); + hivePartition.setDbName(identifier.getDatabaseName()); + hivePartition.setTableName(identifier.getTableName()); + hivePartition.setValues(new ArrayList<>(partitionSpec.values())); + hivePartition.setSd(newSd); + hivePartition.setCreateTime(currentTime); + hivePartition.setLastAccessTime(currentTime); + return hivePartition; + } + /** Factory to create {@link HiveMetastoreClient}. */ public static class Factory implements MetastoreClient.Factory { diff --git a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveFormatTableUtils.java b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveTableUtils.java similarity index 68% rename from paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveFormatTableUtils.java rename to paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveTableUtils.java index 6ec9d3863700..5e5af75e52b9 100644 --- a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveFormatTableUtils.java +++ b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveTableUtils.java @@ -19,14 +19,12 @@ package org.apache.paimon.hive; import org.apache.paimon.catalog.Identifier; -import org.apache.paimon.options.Options; import org.apache.paimon.table.FormatTable; import org.apache.paimon.table.FormatTable.Format; import org.apache.paimon.types.DataType; import org.apache.paimon.types.RowType; import org.apache.paimon.utils.Pair; -import org.apache.hadoop.hive.metastore.TableType; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.SerDeInfo; import org.apache.hadoop.hive.metastore.api.Table; @@ -38,47 +36,42 @@ import java.util.Map; import static org.apache.hadoop.hive.serde.serdeConstants.FIELD_DELIM; -import static org.apache.paimon.CoreOptions.FILE_FORMAT; -import static org.apache.paimon.CoreOptions.TYPE; -import static org.apache.paimon.TableType.FORMAT_TABLE; import static org.apache.paimon.catalog.Catalog.COMMENT_PROP; +import static org.apache.paimon.hive.HiveCatalog.isView; import static org.apache.paimon.table.FormatTableOptions.FIELD_DELIMITER; -class HiveFormatTableUtils { +class HiveTableUtils { public static FormatTable convertToFormatTable(Table hiveTable) { - if (TableType.valueOf(hiveTable.getTableType()) == TableType.VIRTUAL_VIEW) { + if (isView(hiveTable)) { throw new UnsupportedOperationException("Hive view is not supported."); } Identifier identifier = new Identifier(hiveTable.getDbName(), hiveTable.getTableName()); Map options = new HashMap<>(hiveTable.getParameters()); List partitionKeys = getFieldNames(hiveTable.getPartitionKeys()); - RowType rowType = createRowType(hiveTable.getSd().getCols(), hiveTable.getPartitionKeys()); + RowType rowType = createRowType(hiveTable); String comment = options.remove(COMMENT_PROP); String location = hiveTable.getSd().getLocation(); + Format format; - if (Options.fromMap(options).get(TYPE) == FORMAT_TABLE) { - format = Format.valueOf(options.get(FILE_FORMAT.key()).toUpperCase()); - // field delimiter for csv leaves untouched + SerDeInfo serdeInfo = hiveTable.getSd().getSerdeInfo(); + String serLib = serdeInfo.getSerializationLib().toLowerCase(); + String inputFormat = hiveTable.getSd().getInputFormat(); + if (serLib.contains("parquet")) { + format = Format.PARQUET; + } else if (serLib.contains("orc")) { + format = Format.ORC; + } else if (inputFormat.contains("Text")) { + format = Format.CSV; + // hive default field delimiter is '\u0001' + options.put( + FIELD_DELIMITER.key(), + serdeInfo.getParameters().getOrDefault(FIELD_DELIM, "\u0001")); } else { - SerDeInfo serdeInfo = hiveTable.getSd().getSerdeInfo(); - String serLib = serdeInfo.getSerializationLib().toLowerCase(); - String inputFormat = hiveTable.getSd().getInputFormat(); - if (serLib.contains("parquet")) { - format = Format.PARQUET; - } else if (serLib.contains("orc")) { - format = Format.ORC; - } else if (inputFormat.contains("Text")) { - format = Format.CSV; - // hive default field delimiter is '\u0001' - options.put( - FIELD_DELIMITER.key(), - serdeInfo.getParameters().getOrDefault(FIELD_DELIM, "\u0001")); - } else { - throw new UnsupportedOperationException("Unsupported table: " + hiveTable); - } + throw new UnsupportedOperationException("Unsupported table: " + hiveTable); } + return FormatTable.builder() .identifier(identifier) .rowType(rowType) @@ -100,10 +93,9 @@ private static List getFieldNames(List fieldSchemas) { } /** Create a Paimon's Schema from Hive table's columns and partition keys. */ - private static RowType createRowType( - List nonPartCols, List partitionKeys) { - List allCols = new ArrayList<>(nonPartCols); - allCols.addAll(partitionKeys); + public static RowType createRowType(Table table) { + List allCols = new ArrayList<>(table.getSd().getCols()); + allCols.addAll(table.getPartitionKeys()); Pair columnInformation = extractColumnInformation(allCols); return RowType.builder() .fields(columnInformation.getRight(), columnInformation.getLeft()) diff --git a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/RetryingMetaStoreClientFactory.java b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/RetryingMetaStoreClientFactory.java index bc5a887057d8..0ac665fa4e11 100644 --- a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/RetryingMetaStoreClientFactory.java +++ b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/RetryingMetaStoreClientFactory.java @@ -21,6 +21,7 @@ import org.apache.paimon.shade.guava30.com.google.common.collect.ImmutableMap; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.common.JavaUtils; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.HiveMetaHookLoader; import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; @@ -29,7 +30,9 @@ import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; +import java.util.Arrays; import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.concurrent.ConcurrentHashMap; @@ -75,6 +78,16 @@ public class RetryingMetaStoreClientFactory { new ConcurrentHashMap<>(), clientClassName, true)) + .put( + new Class[] { + HiveConf.class, + Class[].class, + Object[].class, + ConcurrentHashMap.class, + String.class + }, + RetryingMetaStoreClientFactory + ::constructorDetectedHiveMetastoreProxySupplier) // for hive 3.x .put( new Class[] { @@ -103,23 +116,8 @@ public class RetryingMetaStoreClientFactory { ConcurrentHashMap.class, String.class }, - (getProxyMethod, hiveConf, clientClassName) -> - (IMetaStoreClient) - getProxyMethod.invoke( - null, - hiveConf, - new Class[] { - HiveConf.class, - HiveMetaHookLoader.class, - Boolean.class - }, - new Object[] { - hiveConf, - (HiveMetaHookLoader) (tbl -> null), - true - }, - new ConcurrentHashMap<>(), - clientClassName)) + RetryingMetaStoreClientFactory + ::constructorDetectedHiveMetastoreProxySupplier) .build(); // If clientClassName is HiveMetaStoreClient, @@ -175,4 +173,50 @@ public interface HiveMetastoreProxySupplier { IMetaStoreClient get(Method getProxyMethod, Configuration conf, String clientClassName) throws IllegalAccessException, IllegalArgumentException, InvocationTargetException; } + + /** Detect the client class whether it has the proper constructor. */ + private static IMetaStoreClient constructorDetectedHiveMetastoreProxySupplier( + Method getProxyMethod, Configuration hiveConf, String clientClassName) + throws IllegalAccessException, IllegalArgumentException, InvocationTargetException { + + try { + Class baseClass = Class.forName(clientClassName, false, JavaUtils.getClassLoader()); + + // Configuration.class or HiveConf.class + List> possibleFirstParamTypes = + Arrays.asList(getProxyMethod.getParameterTypes()[0], hiveConf.getClass()); + + for (Class possibleFirstParamType : possibleFirstParamTypes) { + Class[] fullParams = + new Class[] { + possibleFirstParamType, HiveMetaHookLoader.class, Boolean.TYPE + }; + Object[] fullParamValues = + new Object[] {hiveConf, (HiveMetaHookLoader) (tbl -> null), Boolean.TRUE}; + + for (int i = fullParams.length; i >= 1; i--) { + try { + baseClass.getConstructor(Arrays.copyOfRange(fullParams, 0, i)); + return (IMetaStoreClient) + getProxyMethod.invoke( + null, + hiveConf, + Arrays.copyOfRange(fullParams, 0, i), + Arrays.copyOfRange(fullParamValues, 0, i), + new ConcurrentHashMap<>(), + clientClassName); + } catch (NoSuchMethodException ignored) { + } + } + } + + } catch (ClassNotFoundException e) { + throw new RuntimeException(e); + } + + throw new IllegalArgumentException( + "Failed to create the desired metastore client with proper constructors (class name: " + + clientClassName + + ")"); + } } diff --git a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/migrate/HiveMigrator.java b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/migrate/HiveMigrator.java index b9928ce7311b..d1478830ac6d 100644 --- a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/migrate/HiveMigrator.java +++ b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/migrate/HiveMigrator.java @@ -19,6 +19,7 @@ package org.apache.paimon.hive.migrate; import org.apache.paimon.CoreOptions; +import org.apache.paimon.catalog.Catalog; import org.apache.paimon.catalog.Identifier; import org.apache.paimon.data.BinaryRow; import org.apache.paimon.data.BinaryWriter; @@ -81,8 +82,8 @@ public class HiveMigrator implements Migrator { private final String targetDatabase; private final String targetTable; private final CoreOptions coreOptions; - private Boolean delete = true; - private Integer parallelism; + + private Boolean deleteOriginTable = true; public HiveMigrator( HiveCatalog hiveCatalog, @@ -99,7 +100,6 @@ public HiveMigrator( this.sourceTable = sourceTable; this.targetDatabase = targetDatabase; this.targetTable = targetTable; - this.parallelism = parallelism; this.coreOptions = new CoreOptions(options); this.executor = createCachedThreadPool(parallelism, "HIVE_MIGRATOR"); } @@ -129,8 +129,8 @@ public static List databaseMigrators( } @Override - public void deleteOriginTable(boolean delete) { - this.delete = delete; + public void deleteOriginTable(boolean deleteOriginTable) { + this.deleteOriginTable = deleteOriginTable; } @Override @@ -145,14 +145,18 @@ public void executeMigrate() throws Exception { // create paimon table if not exists Identifier identifier = Identifier.create(targetDatabase, targetTable); - boolean alreadyExist = hiveCatalog.tableExists(identifier); - if (!alreadyExist) { + + boolean deleteIfFail = false; + try { + hiveCatalog.getTable(identifier); + } catch (Catalog.TableNotExistException e) { Schema schema = from( client.getSchema(sourceDatabase, sourceTable), sourceHiveTable.getPartitionKeys(), properties); hiveCatalog.createTable(identifier, schema, false); + deleteIfFail = true; } try { @@ -211,14 +215,14 @@ public void executeMigrate() throws Exception { commit.commit(new ArrayList<>(commitMessages)); } } catch (Exception e) { - if (!alreadyExist) { + if (deleteIfFail) { hiveCatalog.dropTable(identifier, true); } throw new RuntimeException("Migrating failed", e); } // if all success, drop the origin table according the delete field - if (delete) { + if (deleteOriginTable) { client.dropTable(sourceDatabase, sourceTable, true, true); } } diff --git a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/iceberg/IcebergHiveMetadataCommitter.java b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/iceberg/IcebergHiveMetadataCommitter.java new file mode 100644 index 000000000000..ddd21384cbc8 --- /dev/null +++ b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/iceberg/IcebergHiveMetadataCommitter.java @@ -0,0 +1,216 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.iceberg; + +import org.apache.paimon.catalog.Identifier; +import org.apache.paimon.client.ClientPool; +import org.apache.paimon.fs.Path; +import org.apache.paimon.hive.HiveCatalog; +import org.apache.paimon.hive.HiveTypeUtils; +import org.apache.paimon.hive.pool.CachedClientPool; +import org.apache.paimon.options.Options; +import org.apache.paimon.table.FileStoreTable; +import org.apache.paimon.types.DataField; +import org.apache.paimon.utils.Preconditions; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.metastore.IMetaStoreClient; +import org.apache.hadoop.hive.metastore.api.Database; +import org.apache.hadoop.hive.metastore.api.FieldSchema; +import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; +import org.apache.hadoop.hive.metastore.api.SerDeInfo; +import org.apache.hadoop.hive.metastore.api.StorageDescriptor; +import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.thrift.TException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.annotation.Nullable; + +import java.util.Collections; +import java.util.HashMap; +import java.util.stream.Collectors; + +import static org.apache.paimon.iceberg.AbstractIcebergCommitCallback.catalogDatabasePath; + +/** + * {@link IcebergMetadataCommitter} to commit Iceberg metadata to Hive metastore, so the table can + * be visited by Iceberg's Hive catalog. + */ +public class IcebergHiveMetadataCommitter implements IcebergMetadataCommitter { + + private static final Logger LOG = LoggerFactory.getLogger(IcebergHiveMetadataCommitter.class); + + private final FileStoreTable table; + private final Identifier identifier; + private final ClientPool clients; + + public IcebergHiveMetadataCommitter(FileStoreTable table) { + this.table = table; + this.identifier = + Preconditions.checkNotNull( + table.catalogEnvironment().identifier(), + "If you want to sync Paimon Iceberg compatible metadata to Hive, " + + "you must use a Paimon table created from a Paimon catalog, " + + "instead of a temporary table."); + Preconditions.checkArgument( + identifier.getBranchName() == null, + "Paimon Iceberg compatibility currently does not support branches."); + + Options options = new Options(table.options()); + String uri = options.get(IcebergOptions.URI); + String hiveConfDir = options.get(IcebergOptions.HIVE_CONF_DIR); + String hadoopConfDir = options.get(IcebergOptions.HADOOP_CONF_DIR); + Configuration hadoopConf = new Configuration(); + hadoopConf.setClassLoader(IcebergHiveMetadataCommitter.class.getClassLoader()); + HiveConf hiveConf = HiveCatalog.createHiveConf(hiveConfDir, hadoopConfDir, hadoopConf); + + table.options().forEach(hiveConf::set); + if (uri != null) { + hiveConf.set(HiveConf.ConfVars.METASTOREURIS.varname, uri); + } + + if (hiveConf.get(HiveConf.ConfVars.METASTOREURIS.varname) == null) { + LOG.error( + "Can't find hive metastore uri to connect: " + + "either set {} for paimon table or set hive.metastore.uris " + + "in hive-site.xml or hadoop configurations. " + + "Will use empty metastore uris, which means we may use a embedded metastore. " + + "This may cause unpredictable consensus problem.", + IcebergOptions.URI.key()); + } + + this.clients = + new CachedClientPool( + hiveConf, options, options.getString(IcebergOptions.HIVE_CLIENT_CLASS)); + } + + @Override + public void commitMetadata(Path newMetadataPath, @Nullable Path baseMetadataPath) { + try { + commitMetadataImpl(newMetadataPath, baseMetadataPath); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + private void commitMetadataImpl(Path newMetadataPath, @Nullable Path baseMetadataPath) + throws Exception { + if (!databaseExists(identifier.getDatabaseName())) { + createDatabase(identifier.getDatabaseName()); + } + + Table hiveTable; + if (tableExists(identifier)) { + hiveTable = + clients.run( + client -> + client.getTable( + identifier.getDatabaseName(), + identifier.getTableName())); + } else { + hiveTable = createTable(newMetadataPath); + } + + hiveTable.getParameters().put("metadata_location", newMetadataPath.toString()); + if (baseMetadataPath != null) { + hiveTable + .getParameters() + .put("previous_metadata_location", baseMetadataPath.toString()); + } + + clients.execute( + client -> + client.alter_table( + identifier.getDatabaseName(), + identifier.getTableName(), + hiveTable, + true)); + } + + private boolean databaseExists(String databaseName) throws Exception { + try { + clients.run(client -> client.getDatabase(databaseName)); + return true; + } catch (NoSuchObjectException ignore) { + return false; + } + } + + private void createDatabase(String databaseName) throws Exception { + Database database = new Database(); + database.setName(databaseName); + database.setLocationUri(catalogDatabasePath(table).toString()); + clients.execute(client -> client.createDatabase(database)); + } + + private boolean tableExists(Identifier identifier) throws Exception { + return clients.run( + client -> + client.tableExists( + identifier.getDatabaseName(), identifier.getTableName())); + } + + private Table createTable(Path metadataPath) throws Exception { + long currentTimeMillis = System.currentTimeMillis(); + Table hiveTable = + new Table( + identifier.getTableName(), + identifier.getDatabaseName(), + // current linux user + System.getProperty("user.name"), + (int) (currentTimeMillis / 1000), + (int) (currentTimeMillis / 1000), + Integer.MAX_VALUE, + new StorageDescriptor(), + Collections.emptyList(), + new HashMap<>(), + null, + null, + "EXTERNAL_TABLE"); + + hiveTable.getParameters().put("DO_NOT_UPDATE_STATS", "true"); + hiveTable.getParameters().put("EXTERNAL", "TRUE"); + hiveTable.getParameters().put("table_type", "ICEBERG"); + + StorageDescriptor sd = hiveTable.getSd(); + sd.setLocation(metadataPath.getParent().getParent().toString()); + sd.setCols( + table.schema().fields().stream() + .map(this::convertToFieldSchema) + .collect(Collectors.toList())); + sd.setInputFormat("org.apache.hadoop.mapred.FileInputFormat"); + sd.setOutputFormat("org.apache.hadoop.mapred.FileOutputFormat"); + + SerDeInfo serDeInfo = new SerDeInfo(); + serDeInfo.setSerializationLib("org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"); + hiveTable.getSd().setSerdeInfo(serDeInfo); + + clients.execute(client -> client.createTable(hiveTable)); + return hiveTable; + } + + private FieldSchema convertToFieldSchema(DataField dataField) { + return new FieldSchema( + dataField.name(), + HiveTypeUtils.toTypeInfo(dataField.type()).getTypeName(), + dataField.description()); + } +} diff --git a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/iceberg/IcebergHiveMetadataCommitterFactory.java b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/iceberg/IcebergHiveMetadataCommitterFactory.java new file mode 100644 index 000000000000..2ae279d3d7af --- /dev/null +++ b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/iceberg/IcebergHiveMetadataCommitterFactory.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.iceberg; + +import org.apache.paimon.table.FileStoreTable; + +/** Factory to create {@link IcebergHiveMetadataCommitter}. */ +public class IcebergHiveMetadataCommitterFactory implements IcebergMetadataCommitterFactory { + + @Override + public String identifier() { + return IcebergOptions.StorageType.HIVE_CATALOG.toString(); + } + + @Override + public IcebergMetadataCommitter create(FileStoreTable table) { + return new IcebergHiveMetadataCommitter(table); + } +} diff --git a/paimon-hive/paimon-hive-catalog/src/main/resources/META-INF/services/org.apache.paimon.factories.Factory b/paimon-hive/paimon-hive-catalog/src/main/resources/META-INF/services/org.apache.paimon.factories.Factory index baab92184129..26f0944d916e 100644 --- a/paimon-hive/paimon-hive-catalog/src/main/resources/META-INF/services/org.apache.paimon.factories.Factory +++ b/paimon-hive/paimon-hive-catalog/src/main/resources/META-INF/services/org.apache.paimon.factories.Factory @@ -15,3 +15,4 @@ org.apache.paimon.hive.HiveCatalogFactory org.apache.paimon.hive.HiveCatalogLockFactory +org.apache.paimon.iceberg.IcebergHiveMetadataCommitterFactory diff --git a/paimon-hive/paimon-hive-catalog/src/test/java/org/apache/paimon/hive/HiveCatalogTest.java b/paimon-hive/paimon-hive-catalog/src/test/java/org/apache/paimon/hive/HiveCatalogTest.java index 6b13a80e801a..267bdf0c7100 100644 --- a/paimon-hive/paimon-hive-catalog/src/test/java/org/apache/paimon/hive/HiveCatalogTest.java +++ b/paimon-hive/paimon-hive-catalog/src/test/java/org/apache/paimon/hive/HiveCatalogTest.java @@ -18,9 +18,12 @@ package org.apache.paimon.hive; +import org.apache.paimon.CoreOptions; +import org.apache.paimon.catalog.Catalog; import org.apache.paimon.catalog.CatalogTestBase; import org.apache.paimon.catalog.Identifier; import org.apache.paimon.client.ClientPool; +import org.apache.paimon.options.CatalogOptions; import org.apache.paimon.options.Options; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaChange; @@ -29,6 +32,7 @@ import org.apache.paimon.utils.CommonTestUtils; import org.apache.paimon.utils.HadoopUtils; +import org.apache.paimon.shade.guava30.com.google.common.collect.ImmutableMap; import org.apache.paimon.shade.guava30.com.google.common.collect.Lists; import org.apache.hadoop.hive.conf.HiveConf; @@ -37,8 +41,10 @@ import org.apache.thrift.TException; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; import java.lang.reflect.Field; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; @@ -46,9 +52,10 @@ import java.util.Locale; import java.util.Map; import java.util.UUID; +import java.util.concurrent.atomic.AtomicBoolean; import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.METASTORECONNECTURLKEY; -import static org.apache.paimon.hive.HiveCatalog.PAIMON_TABLE_TYPE_VALUE; +import static org.apache.paimon.hive.HiveCatalog.PAIMON_TABLE_IDENTIFIER; import static org.apache.paimon.hive.HiveCatalog.TABLE_TYPE_PROP; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -211,7 +218,7 @@ public void testAddHiveTableParameters() { assertThat(tableProperties).containsEntry("comment", "this is a hive table"); assertThat(tableProperties) .containsEntry( - TABLE_TYPE_PROP, PAIMON_TABLE_TYPE_VALUE.toUpperCase(Locale.ROOT)); + TABLE_TYPE_PROP, PAIMON_TABLE_IDENTIFIER.toUpperCase(Locale.ROOT)); } catch (Exception e) { fail("Test failed due to exception: " + e.getMessage()); } @@ -268,4 +275,124 @@ public void testAlterHiveTableParameters() { fail("Test failed due to exception: " + e.getMessage()); } } + + @Test + public void testListTablesLock() { + try { + String databaseName = "test_db"; + catalog.createDatabase(databaseName, false); + + Map options = new HashMap<>(); + Schema addHiveTableParametersSchema = + new Schema( + Lists.newArrayList( + new DataField(0, "pk", DataTypes.INT()), + new DataField(1, "col1", DataTypes.STRING()), + new DataField(2, "col2", DataTypes.STRING())), + Collections.emptyList(), + Collections.emptyList(), + options, + "this is a hive table"); + + for (int i = 0; i < 100; i++) { + String tableName = "new_table" + i; + catalog.createTable( + Identifier.create(databaseName, tableName), + addHiveTableParametersSchema, + false); + } + List tables1 = new ArrayList<>(); + List tables2 = new ArrayList<>(); + + Thread thread1 = + new Thread( + () -> { + System.out.println( + "First thread started at " + System.currentTimeMillis()); + try { + tables1.addAll(catalog.listTables(databaseName)); + } catch (Catalog.DatabaseNotExistException e) { + throw new RuntimeException(e); + } + }); + Thread thread2 = + new Thread( + () -> { + System.out.println( + "Second thread started at " + System.currentTimeMillis()); + try { + tables2.addAll(catalog.listTables(databaseName)); + } catch (Catalog.DatabaseNotExistException e) { + throw new RuntimeException(e); + } + }); + + thread1.start(); + thread2.start(); + + long timeout = 5000; + long startTime = System.currentTimeMillis(); + + AtomicBoolean deadlockDetected = new AtomicBoolean(false); + while (thread1.isAlive() || thread2.isAlive()) { + if (System.currentTimeMillis() - startTime > timeout) { + deadlockDetected.set(true); + thread1.interrupt(); + thread2.interrupt(); + break; + } + + Thread.sleep(100); + } + + assertThat(deadlockDetected).isFalse(); + assertThat(tables1).size().isEqualTo(100); + assertThat(tables1).containsAll(tables2); + assertThat(tables2).containsAll(tables1); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + @Override + protected boolean supportsView() { + return true; + } + + @Override + protected boolean supportsFormatTable() { + return true; + } + + @Test + public void testCreateExternalTableWithLocation(@TempDir java.nio.file.Path tempDir) + throws Exception { + HiveConf hiveConf = new HiveConf(); + String jdoConnectionURL = "jdbc:derby:memory:" + UUID.randomUUID(); + hiveConf.setVar(METASTORECONNECTURLKEY, jdoConnectionURL + ";create=true"); + hiveConf.set(CatalogOptions.TABLE_TYPE.key(), "external"); + String metastoreClientClass = "org.apache.hadoop.hive.metastore.HiveMetaStoreClient"; + HiveCatalog externalWarehouseCatalog = + new HiveCatalog(fileIO, hiveConf, metastoreClientClass, warehouse); + + String externalTablePath = tempDir.toString(); + + Schema schema = + new Schema( + Lists.newArrayList(new DataField(0, "foo", DataTypes.INT())), + Collections.emptyList(), + Collections.emptyList(), + ImmutableMap.of("path", externalTablePath), + ""); + + Identifier identifier = Identifier.create("default", "my_table"); + externalWarehouseCatalog.createTable(identifier, schema, true); + + org.apache.paimon.table.Table table = externalWarehouseCatalog.getTable(identifier); + assertThat(table.options()) + .extracting(CoreOptions.PATH.key()) + .isEqualTo("file:" + externalTablePath); + + externalWarehouseCatalog.close(); + } } diff --git a/paimon-hive/paimon-hive-catalog/src/test/java/org/apache/paimon/hive/HiveTableStatsTest.java b/paimon-hive/paimon-hive-catalog/src/test/java/org/apache/paimon/hive/HiveTableStatsTest.java new file mode 100644 index 000000000000..33016fd08361 --- /dev/null +++ b/paimon-hive/paimon-hive-catalog/src/test/java/org/apache/paimon/hive/HiveTableStatsTest.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.hive; + +import org.apache.paimon.catalog.Catalog; +import org.apache.paimon.catalog.CatalogContext; +import org.apache.paimon.catalog.Identifier; +import org.apache.paimon.fs.FileIO; +import org.apache.paimon.fs.Path; +import org.apache.paimon.options.CatalogOptions; +import org.apache.paimon.options.Options; +import org.apache.paimon.schema.Schema; +import org.apache.paimon.schema.SchemaChange; +import org.apache.paimon.types.DataField; +import org.apache.paimon.types.DataTypes; + +import org.apache.paimon.shade.guava30.com.google.common.collect.Lists; +import org.apache.paimon.shade.guava30.com.google.common.collect.Maps; + +import org.apache.hadoop.hive.common.StatsSetupConst; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.metastore.api.Table; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.util.Collections; +import java.util.UUID; + +import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.METASTORECONNECTURLKEY; +import static org.assertj.core.api.Assertions.assertThat; + +/** Verify that table stats has been updated. */ +public class HiveTableStatsTest { + @TempDir java.nio.file.Path tempFile; + protected Catalog catalog; + + @BeforeEach + public void setUp() throws Exception { + String warehouse = tempFile.toUri().toString(); + HiveConf hiveConf = new HiveConf(); + String jdoConnectionURL = "jdbc:derby:memory:" + UUID.randomUUID(); + hiveConf.setVar(METASTORECONNECTURLKEY, jdoConnectionURL + ";create=true"); + String metastoreClientClass = "org.apache.hadoop.hive.metastore.HiveMetaStoreClient"; + Options catalogOptions = new Options(); + catalogOptions.set(StatsSetupConst.DO_NOT_UPDATE_STATS, "true"); + catalogOptions.set(CatalogOptions.WAREHOUSE, warehouse); + CatalogContext catalogContext = CatalogContext.create(catalogOptions); + FileIO fileIO = FileIO.get(new Path(warehouse), catalogContext); + catalog = + new HiveCatalog(fileIO, hiveConf, metastoreClientClass, catalogOptions, warehouse); + } + + @Test + public void testAlterTable() throws Exception { + catalog.createDatabase("test_db", false); + // Alter table adds a new column to an existing table,but do not update stats + Identifier identifier = Identifier.create("test_db", "test_table"); + catalog.createTable( + identifier, + new Schema( + Lists.newArrayList(new DataField(0, "col1", DataTypes.STRING())), + Collections.emptyList(), + Collections.emptyList(), + Maps.newHashMap(), + ""), + false); + catalog.alterTable( + identifier, + Lists.newArrayList( + SchemaChange.addColumn("col2", DataTypes.DATE()), + SchemaChange.addColumn("col3", DataTypes.STRING(), "col3 field")), + false); + HiveCatalog hiveCatalog = (HiveCatalog) catalog; + Table table = hiveCatalog.getHmsTable(identifier); + assertThat(table.getParameters().get("COLUMN_STATS_ACCURATE")).isEqualTo(null); + } +} diff --git a/paimon-hive/paimon-hive-common/src/main/java/org/apache/paimon/hive/HiveTypeUtils.java b/paimon-hive/paimon-hive-common/src/main/java/org/apache/paimon/hive/HiveTypeUtils.java index f00d675f3750..33cd45a351a4 100644 --- a/paimon-hive/paimon-hive-common/src/main/java/org/apache/paimon/hive/HiveTypeUtils.java +++ b/paimon-hive/paimon-hive-common/src/main/java/org/apache/paimon/hive/HiveTypeUtils.java @@ -185,7 +185,7 @@ public TypeInfo visit(TimestampType timestampType) { @Override public TypeInfo visit(LocalZonedTimestampType localZonedTimestampType) { - return LocalZonedTimestampTypeUtils.toHiveType(localZonedTimestampType); + return LocalZonedTimestampTypeUtils.hiveLocalZonedTimestampType(); } @Override @@ -254,7 +254,7 @@ static DataType visit(TypeInfo type, HiveToPaimonTypeVisitor visitor) { } public DataType atomic(TypeInfo atomic) { - if (LocalZonedTimestampTypeUtils.isLocalZonedTimestampType(atomic)) { + if (LocalZonedTimestampTypeUtils.isHiveLocalZonedTimestampType(atomic)) { return DataTypes.TIMESTAMP_WITH_LOCAL_TIME_ZONE(); } diff --git a/paimon-hive/paimon-hive-common/src/main/java/org/apache/paimon/hive/LocalZonedTimestampTypeUtils.java b/paimon-hive/paimon-hive-common/src/main/java/org/apache/paimon/hive/LocalZonedTimestampTypeUtils.java index fe76debfc333..b143fcd8caec 100644 --- a/paimon-hive/paimon-hive-common/src/main/java/org/apache/paimon/hive/LocalZonedTimestampTypeUtils.java +++ b/paimon-hive/paimon-hive-common/src/main/java/org/apache/paimon/hive/LocalZonedTimestampTypeUtils.java @@ -23,14 +23,27 @@ import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; -/** To maintain compatibility with Hive 3. */ +import java.lang.reflect.Field; + +/** + * Utils to convert between Hive TimestampLocalTZTypeInfo and Paimon {@link + * LocalZonedTimestampType}, using reflection to solve compatibility between Hive 2 and Hive 3. + */ public class LocalZonedTimestampTypeUtils { - public static boolean isLocalZonedTimestampType(TypeInfo hiveTypeInfo) { - return false; + public static boolean isHiveLocalZonedTimestampType(TypeInfo hiveTypeInfo) { + return "org.apache.hadoop.hive.serde2.typeinfo.TimestampLocalTZTypeInfo" + .equals(hiveTypeInfo.getClass().getName()); } - public static TypeInfo toHiveType(LocalZonedTimestampType paimonType) { - return TypeInfoFactory.timestampTypeInfo; + public static TypeInfo hiveLocalZonedTimestampType() { + try { + Class typeInfoFactoryClass = + Class.forName("org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory"); + Field field = typeInfoFactoryClass.getField("timestampLocalTZTypeInfo"); + return (TypeInfo) field.get(null); + } catch (Exception e) { + return TypeInfoFactory.timestampTypeInfo; + } } } diff --git a/paimon-hive/paimon-hive-connector-2.3/pom.xml b/paimon-hive/paimon-hive-connector-2.3/pom.xml index e61e493d3a5c..a0f509b53375 100644 --- a/paimon-hive/paimon-hive-connector-2.3/pom.xml +++ b/paimon-hive/paimon-hive-connector-2.3/pom.xml @@ -81,7 +81,7 @@ under the License. org.apache.flink - flink-table-planner_${scala.binary.version} + flink-table-planner_${flink.scala.binary.version} ${test.flink.version} test @@ -562,6 +562,13 @@ under the License. test + + + org.apache.iceberg + iceberg-flink-${iceberg.flink.version} + ${iceberg.version} + test + diff --git a/paimon-hive/paimon-hive-connector-2.3/src/test/java/org/apache/paimon/hive/AlterFailHiveMetaStoreClient.java b/paimon-hive/paimon-hive-connector-2.3/src/test/java/org/apache/paimon/hive/AlterFailHiveMetaStoreClient.java index ebd4684edf1b..55e6d74084d8 100644 --- a/paimon-hive/paimon-hive-connector-2.3/src/test/java/org/apache/paimon/hive/AlterFailHiveMetaStoreClient.java +++ b/paimon-hive/paimon-hive-connector-2.3/src/test/java/org/apache/paimon/hive/AlterFailHiveMetaStoreClient.java @@ -22,6 +22,7 @@ import org.apache.hadoop.hive.metastore.HiveMetaHookLoader; import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; import org.apache.hadoop.hive.metastore.IMetaStoreClient; +import org.apache.hadoop.hive.metastore.api.EnvironmentContext; import org.apache.hadoop.hive.metastore.api.InvalidOperationException; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.Table; @@ -51,4 +52,11 @@ public void alter_table( throws InvalidOperationException, MetaException, TException { throw new TException(); } + + @Override + public void alter_table_with_environmentContext( + String defaultDatabaseName, String tblName, Table table, EnvironmentContext env) + throws InvalidOperationException, MetaException, TException { + throw new TException(); + } } diff --git a/paimon-hive/paimon-hive-connector-2.3/src/test/java/org/apache/paimon/hive/CustomConstructorMetastoreClient.java b/paimon-hive/paimon-hive-connector-2.3/src/test/java/org/apache/paimon/hive/CustomConstructorMetastoreClient.java new file mode 100644 index 000000000000..c9ddc1be468f --- /dev/null +++ b/paimon-hive/paimon-hive-connector-2.3/src/test/java/org/apache/paimon/hive/CustomConstructorMetastoreClient.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.hive; + +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.metastore.HiveMetaHookLoader; +import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; +import org.apache.hadoop.hive.metastore.IMetaStoreClient; +import org.apache.hadoop.hive.metastore.api.MetaException; + +/** A {@link HiveMetaStoreClient} Factory to test custom Hive metastore client. */ +public class CustomConstructorMetastoreClient { + + /** + * A {@link HiveMetaStoreClient} to test custom Hive metastore client with (HiveConf, + * HiveMetaHookLoader) constructor. + */ + public static class TwoParameterConstructorMetastoreClient extends HiveMetaStoreClient + implements IMetaStoreClient { + + public TwoParameterConstructorMetastoreClient(HiveConf conf, HiveMetaHookLoader hookLoader) + throws MetaException { + super(conf, hookLoader); + } + } + + /** + * A {@link HiveMetaStoreClient} to test custom Hive metastore client with (HiveConf) + * constructor. + */ + public static class OneParameterConstructorMetastoreClient extends HiveMetaStoreClient + implements IMetaStoreClient { + + public OneParameterConstructorMetastoreClient(HiveConf conf) throws MetaException { + super(conf); + } + } +} diff --git a/paimon-hive/paimon-hive-connector-2.3/src/test/java/org/apache/paimon/hive/Hive23CatalogITCase.java b/paimon-hive/paimon-hive-connector-2.3/src/test/java/org/apache/paimon/hive/Hive23CatalogITCase.java index 204f779d71b5..8a4745a09022 100644 --- a/paimon-hive/paimon-hive-connector-2.3/src/test/java/org/apache/paimon/hive/Hive23CatalogITCase.java +++ b/paimon-hive/paimon-hive-connector-2.3/src/test/java/org/apache/paimon/hive/Hive23CatalogITCase.java @@ -85,7 +85,35 @@ public void testCustomMetastoreClient() throws Exception { } @Test - public void testCreateExistTableInHive() throws Exception { + public void testCustomConstructorMetastoreClient() throws Exception { + path = folder.newFolder().toURI().toString(); + EnvironmentSettings settings = EnvironmentSettings.newInstance().inBatchMode().build(); + Class[] customConstructorMetastoreClientClass = { + CustomConstructorMetastoreClient.TwoParameterConstructorMetastoreClient.class, + CustomConstructorMetastoreClient.OneParameterConstructorMetastoreClient.class + }; + for (Class clazz : customConstructorMetastoreClientClass) { + tEnv = TableEnvironmentImpl.create(settings); + tEnv.executeSql( + String.join( + "\n", + "CREATE CATALOG my_hive WITH (", + " 'type' = 'paimon',", + " 'metastore' = 'hive',", + " 'uri' = '',", + " 'default-database' = 'test_db',", + " 'warehouse' = '" + path + "',", + " 'metastore.client.class' = '" + clazz.getName() + "'", + ")")) + .await(); + tEnv.executeSql("USE CATALOG my_hive").await(); + assertThat(collect("SHOW DATABASES")) + .isEqualTo(Arrays.asList(Row.of("default"), Row.of("test_db"))); + } + } + + @Test + public void testCreateExistTableInHive() { tEnv.executeSql( String.join( "\n", @@ -105,7 +133,6 @@ public void testCreateExistTableInHive() throws Exception { tEnv.executeSql( "CREATE TABLE hive_table(a INT, b INT, c INT, d INT)") .await()) - .isInstanceOf(TableException.class) .hasMessage( "Could not execute CreateTable in path `my_hive_custom_client`.`test_db`.`hive_table`"); assertThat( diff --git a/paimon-hive/paimon-hive-connector-2.3/src/test/java/org/apache/paimon/iceberg/IcebergHive23MetadataCommitterITCase.java b/paimon-hive/paimon-hive-connector-2.3/src/test/java/org/apache/paimon/iceberg/IcebergHive23MetadataCommitterITCase.java new file mode 100644 index 000000000000..7d726e75a17d --- /dev/null +++ b/paimon-hive/paimon-hive-connector-2.3/src/test/java/org/apache/paimon/iceberg/IcebergHive23MetadataCommitterITCase.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.iceberg; + +import org.apache.paimon.hive.CreateFailHiveMetaStoreClient; + +/** IT cases for {@link IcebergHiveMetadataCommitter} in Hive 2.3. */ +public class IcebergHive23MetadataCommitterITCase extends IcebergHiveMetadataCommitterITCaseBase { + @Override + protected String createFailHiveMetaStoreClient() { + return CreateFailHiveMetaStoreClient.class.getName(); + } +} diff --git a/paimon-hive/paimon-hive-connector-3.1/pom.xml b/paimon-hive/paimon-hive-connector-3.1/pom.xml index 3e08196a76af..5383af90c3e5 100644 --- a/paimon-hive/paimon-hive-connector-3.1/pom.xml +++ b/paimon-hive/paimon-hive-connector-3.1/pom.xml @@ -88,7 +88,7 @@ under the License. org.apache.flink - flink-table-planner_${scala.binary.version} + flink-table-planner_${flink.scala.binary.version} ${test.flink.version} test @@ -592,6 +592,13 @@ under the License. test + + + org.apache.iceberg + iceberg-flink-${iceberg.flink.version} + ${iceberg.version} + test + diff --git a/paimon-hive/paimon-hive-connector-3.1/src/test/java/org/apache/paimon/hive/AlterFailHiveMetaStoreClient.java b/paimon-hive/paimon-hive-connector-3.1/src/test/java/org/apache/paimon/hive/AlterFailHiveMetaStoreClient.java index ae6a1bb85ac4..eab18feadaca 100644 --- a/paimon-hive/paimon-hive-connector-3.1/src/test/java/org/apache/paimon/hive/AlterFailHiveMetaStoreClient.java +++ b/paimon-hive/paimon-hive-connector-3.1/src/test/java/org/apache/paimon/hive/AlterFailHiveMetaStoreClient.java @@ -22,6 +22,7 @@ import org.apache.hadoop.hive.metastore.HiveMetaHookLoader; import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; import org.apache.hadoop.hive.metastore.IMetaStoreClient; +import org.apache.hadoop.hive.metastore.api.EnvironmentContext; import org.apache.hadoop.hive.metastore.api.InvalidOperationException; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.Table; @@ -51,4 +52,11 @@ public void alter_table( throws InvalidOperationException, MetaException, TException { throw new TException(); } + + @Override + public void alter_table_with_environmentContext( + String defaultDatabaseName, String tblName, Table table, EnvironmentContext env) + throws InvalidOperationException, MetaException, TException { + throw new TException(); + } } diff --git a/paimon-hive/paimon-hive-connector-3.1/src/test/java/org/apache/paimon/hive/CustomConstructorMetastoreClient.java b/paimon-hive/paimon-hive-connector-3.1/src/test/java/org/apache/paimon/hive/CustomConstructorMetastoreClient.java new file mode 100644 index 000000000000..6b08f16d4ab6 --- /dev/null +++ b/paimon-hive/paimon-hive-connector-3.1/src/test/java/org/apache/paimon/hive/CustomConstructorMetastoreClient.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.hive; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.metastore.HiveMetaHookLoader; +import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; +import org.apache.hadoop.hive.metastore.IMetaStoreClient; +import org.apache.hadoop.hive.metastore.api.MetaException; + +/** A {@link HiveMetaStoreClient} Factory to test custom Hive metastore client. */ +public class CustomConstructorMetastoreClient { + + /** + * A {@link HiveMetaStoreClient} to test custom Hive metastore client with (Configuration, + * HiveMetaHookLoader) constructor. + */ + public static class TwoParameterConstructorMetastoreClient extends HiveMetaStoreClient + implements IMetaStoreClient { + + public TwoParameterConstructorMetastoreClient( + Configuration conf, HiveMetaHookLoader hookLoader) throws MetaException { + super(conf, hookLoader); + } + } + + /** + * A {@link HiveMetaStoreClient} to test custom Hive metastore client with (Configuration) + * constructor. + */ + public static class OneParameterConstructorMetastoreClient extends HiveMetaStoreClient + implements IMetaStoreClient { + + public OneParameterConstructorMetastoreClient(Configuration conf) throws MetaException { + super(conf); + } + } + + /** + * A {@link HiveMetaStoreClient} to test custom Hive metastore client with (HiveConf) + * constructor. + */ + public static class OtherParameterConstructorMetastoreClient extends HiveMetaStoreClient + implements IMetaStoreClient { + + public OtherParameterConstructorMetastoreClient(HiveConf conf) throws MetaException { + super(conf); + } + } +} diff --git a/paimon-hive/paimon-hive-connector-3.1/src/test/java/org/apache/paimon/hive/Hive31CatalogITCase.java b/paimon-hive/paimon-hive-connector-3.1/src/test/java/org/apache/paimon/hive/Hive31CatalogITCase.java index 1cd17553fd3f..48d41d27e8d8 100644 --- a/paimon-hive/paimon-hive-connector-3.1/src/test/java/org/apache/paimon/hive/Hive31CatalogITCase.java +++ b/paimon-hive/paimon-hive-connector-3.1/src/test/java/org/apache/paimon/hive/Hive31CatalogITCase.java @@ -83,6 +83,35 @@ public void testCustomMetastoreClient() throws Exception { Row.of(TestHiveMetaStoreClient.MOCK_DATABASE))); } + @Test + public void testCustomConstructorMetastoreClient() throws Exception { + path = folder.newFolder().toURI().toString(); + EnvironmentSettings settings = EnvironmentSettings.newInstance().inBatchMode().build(); + Class[] customConstructorMetastoreClientClass = { + CustomConstructorMetastoreClient.TwoParameterConstructorMetastoreClient.class, + CustomConstructorMetastoreClient.OneParameterConstructorMetastoreClient.class, + CustomConstructorMetastoreClient.OtherParameterConstructorMetastoreClient.class + }; + + for (Class clazz : customConstructorMetastoreClientClass) { + tEnv = TableEnvironmentImpl.create(settings); + tEnv.executeSql( + String.join( + "\n", + "CREATE CATALOG my_hive WITH (", + " 'type' = 'paimon',", + " 'metastore' = 'hive',", + " 'uri' = '',", + " 'warehouse' = '" + path + "',", + " 'metastore.client.class' = '" + clazz.getName() + "'", + ")")) + .await(); + tEnv.executeSql("USE CATALOG my_hive").await(); + assertThat(collect("SHOW DATABASES")) + .isEqualTo(Arrays.asList(Row.of("default"), Row.of("test_db"))); + } + } + @Test public void testCreateExistTableInHive() throws Exception { tEnv.executeSql( @@ -104,7 +133,6 @@ public void testCreateExistTableInHive() throws Exception { tEnv.executeSql( "CREATE TABLE hive_table(a INT, b INT, c INT, d INT)") .await()) - .isInstanceOf(TableException.class) .hasMessage( "Could not execute CreateTable in path `my_hive_custom_client`.`test_db`.`hive_table`"); assertThat( diff --git a/paimon-hive/paimon-hive-connector-3.1/src/test/java/org/apache/paimon/iceberg/IcebergHive31MetadataCommitterITCase.java b/paimon-hive/paimon-hive-connector-3.1/src/test/java/org/apache/paimon/iceberg/IcebergHive31MetadataCommitterITCase.java new file mode 100644 index 000000000000..0634adfad357 --- /dev/null +++ b/paimon-hive/paimon-hive-connector-3.1/src/test/java/org/apache/paimon/iceberg/IcebergHive31MetadataCommitterITCase.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.iceberg; + +import org.apache.paimon.hive.CreateFailHiveMetaStoreClient; + +/** IT cases for {@link IcebergHiveMetadataCommitter} in Hive 3.1. */ +public class IcebergHive31MetadataCommitterITCase extends IcebergHiveMetadataCommitterITCaseBase { + @Override + protected String createFailHiveMetaStoreClient() { + return CreateFailHiveMetaStoreClient.class.getName(); + } +} diff --git a/paimon-hive/paimon-hive-connector-common/pom.xml b/paimon-hive/paimon-hive-connector-common/pom.xml index 0f7da151b884..f22b73dea71e 100644 --- a/paimon-hive/paimon-hive-connector-common/pom.xml +++ b/paimon-hive/paimon-hive-connector-common/pom.xml @@ -111,14 +111,14 @@ under the License. org.apache.flink - flink-table-planner_${scala.binary.version} + flink-table-planner_${flink.scala.binary.version} ${test.flink.version} test org.apache.flink - flink-connector-hive_${scala.binary.version} + flink-connector-hive_${flink.scala.binary.version} ${test.flink.version} test diff --git a/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/HiveSchema.java b/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/HiveSchema.java index f637651413ed..108315a96103 100644 --- a/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/HiveSchema.java +++ b/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/HiveSchema.java @@ -233,9 +233,10 @@ private static void checkFieldsMatched( } } - if (schemaFieldNames.size() != hiveFieldNames.size()) { + // It is OK that hive is a subset of paimon + if (schemaFieldNames.size() < hiveFieldNames.size()) { throw new IllegalArgumentException( - "Hive DDL and paimon schema mismatched! " + "Hive DDL is a superset of paimon schema! " + "It is recommended not to write any column definition " + "as Paimon external table can read schema from the specified location.\n" + "There are " diff --git a/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/utils/HiveSplitGenerator.java b/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/utils/HiveSplitGenerator.java index 33cbc19e0326..144afab8e1fa 100644 --- a/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/utils/HiveSplitGenerator.java +++ b/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/utils/HiveSplitGenerator.java @@ -96,7 +96,8 @@ public static InputSplit[] generateSplits(FileStoreTable table, JobConf jobConf) scan.withFilter(PredicateBuilder.and(predicatePerPartition)); } } - scan.plan() + scan.dropStats() + .plan() .splits() .forEach( split -> diff --git a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/FlinkGenericCatalogITCase.java b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/FlinkGenericCatalogITCase.java index f2ae892059b8..8507c4c21aee 100644 --- a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/FlinkGenericCatalogITCase.java +++ b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/FlinkGenericCatalogITCase.java @@ -201,4 +201,20 @@ public void testReadPaimonAllProcedures() { assertThat(result) .contains(Row.of("compact"), Row.of("merge_into"), Row.of("migrate_table")); } + + @Test + public void testCreateTag() { + sql( + "CREATE TABLE paimon_t ( " + + "f0 INT, " + + "f1 INT " + + ") WITH ('connector'='paimon', 'file.format' = 'avro' )"); + sql("INSERT INTO paimon_t VALUES (1, 1), (2, 2)"); + assertThat(sql("SELECT * FROM paimon_t")) + .containsExactlyInAnyOrder(Row.of(1, 1), Row.of(2, 2)); + sql("CALL sys.create_tag('test_db.paimon_t', 'tag_1')"); + + List result = sql("SELECT tag_name FROM paimon_t$tags"); + assertThat(result).contains(Row.of("tag_1")); + } } diff --git a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveCatalogFormatTableITCaseBase.java b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveCatalogFormatTableITCaseBase.java index 9a7ff1e586a1..fc58ad59527c 100644 --- a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveCatalogFormatTableITCaseBase.java +++ b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveCatalogFormatTableITCaseBase.java @@ -44,7 +44,7 @@ import java.util.Map; import java.util.stream.Collectors; -import static org.apache.paimon.hive.HiveCatalogOptions.FORMAT_TABLE_ENABLED; +import static org.apache.paimon.options.CatalogOptions.FORMAT_TABLE_ENABLED; import static org.assertj.core.api.Assertions.assertThat; /** IT cases for using Paimon {@link HiveCatalog}. */ @@ -136,6 +136,28 @@ public void testPartitionTable() throws Exception { doTestFormatTable("partition_table"); } + @Test + public void testFlinkCreateCsvFormatTable() throws Exception { + tEnv.executeSql( + "CREATE TABLE flink_csv_table (a INT, b STRING) with ('type'='format-table', 'file.format'='csv')") + .await(); + doTestFormatTable("flink_csv_table"); + } + + @Test + public void testFlinkCreateFormatTableWithDelimiter() throws Exception { + tEnv.executeSql( + "CREATE TABLE flink_csv_table_delimiter (a INT, b STRING) with ('type'='format-table', 'file.format'='csv', 'field-delimiter'=';')"); + doTestFormatTable("flink_csv_table_delimiter"); + } + + @Test + public void testFlinkCreatePartitionTable() throws Exception { + tEnv.executeSql( + "CREATE TABLE flink_partition_table (a INT,b STRING) PARTITIONED BY (b) with ('type'='format-table', 'file.format'='csv')"); + doTestFormatTable("flink_partition_table"); + } + private void doTestFormatTable(String tableName) throws Exception { hiveShell.execute( String.format("INSERT INTO %s VALUES (100, 'Hive'), (200, 'Table')", tableName)); diff --git a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveCatalogITCaseBase.java b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveCatalogITCaseBase.java index aa3062ec7db3..2266a8484d9d 100644 --- a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveCatalogITCaseBase.java +++ b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveCatalogITCaseBase.java @@ -19,27 +19,26 @@ package org.apache.paimon.hive; import org.apache.paimon.catalog.Catalog; -import org.apache.paimon.catalog.CatalogLock; -import org.apache.paimon.catalog.CatalogLockFactory; import org.apache.paimon.catalog.Identifier; import org.apache.paimon.flink.FlinkCatalog; import org.apache.paimon.hive.annotation.Minio; import org.apache.paimon.hive.runner.PaimonEmbeddedHiveRunner; import org.apache.paimon.metastore.MetastoreClient; +import org.apache.paimon.operation.Lock; import org.apache.paimon.privilege.NoPrivilegeException; import org.apache.paimon.s3.MinioTestContainer; +import org.apache.paimon.table.CatalogEnvironment; import org.apache.paimon.table.FileStoreTable; import org.apache.paimon.table.Table; import org.apache.paimon.utils.IOUtils; +import org.apache.paimon.utils.TimeUtils; import com.klarna.hiverunner.HiveShell; import com.klarna.hiverunner.annotations.HiveSQL; import org.apache.flink.core.fs.FSDataInputStream; import org.apache.flink.core.fs.Path; -import org.apache.flink.streaming.api.environment.ExecutionCheckpointingOptions; import org.apache.flink.table.api.EnvironmentSettings; import org.apache.flink.table.api.TableEnvironment; -import org.apache.flink.table.api.TableException; import org.apache.flink.table.api.TableResult; import org.apache.flink.table.api.ValidationException; import org.apache.flink.table.api.config.ExecutionConfigOptions; @@ -140,7 +139,9 @@ private void registerHiveCatalog(String catalogName, Map catalog EnvironmentSettings.newInstance().inStreamingMode().build()); sEnv.getConfig() .getConfiguration() - .set(ExecutionCheckpointingOptions.CHECKPOINTING_INTERVAL, Duration.ofSeconds(1)); + .setString( + "execution.checkpointing.interval", + TimeUtils.formatWithHighestUnit(Duration.ofSeconds(1))); sEnv.getConfig().set(ExecutionConfigOptions.TABLE_EXEC_RESOURCE_DEFAULT_PARALLELISM, 1); tEnv.executeSql( @@ -203,7 +204,7 @@ public void testDbLocation() { @Test @LocationInProperties public void testDbLocationWithMetastoreLocationInProperties() - throws Catalog.DatabaseAlreadyExistException { + throws Catalog.DatabaseAlreadyExistException, Catalog.DatabaseNotExistException { String dbLocation = minioTestContainer.getS3UriForDefaultBucket() + "/" + UUID.randomUUID(); Catalog catalog = ((FlinkCatalog) tEnv.getCatalog(tEnv.getCurrentCatalog()).get()).catalog(); @@ -211,7 +212,7 @@ public void testDbLocationWithMetastoreLocationInProperties() properties.put("location", dbLocation); catalog.createDatabase("location_test_db", false, properties); - assertThat(catalog.databaseExists("location_test_db")); + catalog.getDatabase("location_test_db"); hiveShell.execute("USE location_test_db"); hiveShell.execute("CREATE TABLE location_test_db ( a INT, b INT )"); @@ -275,7 +276,8 @@ public void testTableOperations() throws Exception { .await(); tEnv.executeSql("CREATE TABLE s ( a INT, b STRING ) WITH ( 'file.format' = 'avro' )") .await(); - assertThat(collect("SHOW TABLES")).isEqualTo(Arrays.asList(Row.of("s"), Row.of("t"))); + assertThat(collect("SHOW TABLES")) + .containsExactlyInAnyOrder(Row.of("s"), Row.of("t"), Row.of("hive_table")); tEnv.executeSql( "CREATE TABLE IF NOT EXISTS s ( a INT, b STRING ) WITH ( 'file.format' = 'avro' )") @@ -294,17 +296,14 @@ public void testTableOperations() throws Exception { Path tablePath = new Path(path, "test_db.db/s"); assertThat(tablePath.getFileSystem().exists(tablePath)).isTrue(); tEnv.executeSql("DROP TABLE s").await(); - assertThat(collect("SHOW TABLES")).isEqualTo(Collections.singletonList(Row.of("t"))); + assertThat(collect("SHOW TABLES")) + .containsExactlyInAnyOrder(Row.of("t"), Row.of("hive_table")); assertThat(tablePath.getFileSystem().exists(tablePath)).isFalse(); tEnv.executeSql("DROP TABLE IF EXISTS s").await(); assertThatThrownBy(() -> tEnv.executeSql("DROP TABLE s").await()) .isInstanceOf(ValidationException.class) .hasMessage("Table with identifier 'my_hive.test_db.s' does not exist."); - assertThatThrownBy(() -> tEnv.executeSql("DROP TABLE hive_table").await()) - .isInstanceOf(ValidationException.class) - .hasMessage("Table with identifier 'my_hive.test_db.hive_table' does not exist."); - // alter table tEnv.executeSql("ALTER TABLE t SET ( 'manifest.target-file-size' = '16MB' )").await(); List actual = collect("SHOW CREATE TABLE t"); @@ -329,9 +328,9 @@ public void testTableOperations() throws Exception { tEnv.executeSql( "ALTER TABLE hive_table SET ( 'manifest.target-file-size' = '16MB' )") .await()) - .isInstanceOf(RuntimeException.class) + .rootCause() .hasMessage( - "Table `my_hive`.`test_db`.`hive_table` doesn't exist or is a temporary table."); + "Only support alter data table, but is: class org.apache.paimon.table.FormatTable$FormatTableImpl"); } @Test @@ -656,15 +655,6 @@ public void testFlinkWriteAndHiveRead() throws Exception { Arrays.asList( "true\t1\t1\t1\t1234567890123456789\t1.23\t3.14159\t1234.56\tABC\tv1\tHello, World!\t01\t010203\t2023-01-01\t2023-01-01 12:00:00.123\t[\"value1\",\"value2\",\"value3\"]\tvalue1\tvalue1\tvalue2\t{\"f0\":\"v1\",\"f1\":1}\tv1\t1", "false\t2\t2\t2\t234567890123456789\t2.34\t2.111111\t2345.67\tDEF\tv2\tApache Paimon\t04\t040506\t2023-02-01\t2023-02-01 12:00:00.456\t[\"value4\",\"value5\",\"value6\"]\tvalue4\tvalue11\tvalue22\t{\"f0\":\"v2\",\"f1\":2}\tv2\t2")); - - assertThatThrownBy( - () -> - tEnv.executeSql( - "INSERT INTO hive_table VALUES (1, 'Hi'), (2, 'Hello')") - .await()) - .isInstanceOf(TableException.class) - .hasMessage( - "Cannot find table '`my_hive`.`test_db`.`hive_table`' in any of the catalogs [default_catalog, my_hive], nor as a temporary table."); } @Test @@ -1128,11 +1118,12 @@ public void testAlterTable() throws Exception { } @Test - public void testHiveLock() throws InterruptedException { + public void testHiveLock() throws InterruptedException, Catalog.TableNotExistException { tEnv.executeSql("CREATE TABLE t (a INT)"); Catalog catalog = ((FlinkCatalog) tEnv.getCatalog(tEnv.getCurrentCatalog()).get()).catalog(); - CatalogLockFactory lockFactory = catalog.lockFactory().get(); + FileStoreTable table = (FileStoreTable) catalog.getTable(new Identifier("test_db", "t")); + CatalogEnvironment catalogEnv = table.catalogEnvironment(); AtomicInteger count = new AtomicInteger(0); List threads = new ArrayList<>(); @@ -1147,11 +1138,10 @@ public void testHiveLock() throws InterruptedException { Thread thread = new Thread( () -> { - CatalogLock lock = - lockFactory.createLock(catalog.lockContext().get()); + Lock lock = catalogEnv.lockFactory().create(); for (int j = 0; j < 10; j++) { try { - lock.runWithLock("test_db", "t", unsafeIncrement); + lock.runWithLock(unsafeIncrement); } catch (Exception e) { throw new RuntimeException(e); } @@ -1348,6 +1338,26 @@ public void testDropPartitionsToMetastore() throws Exception { "ptb=2a/pta=2", "ptb=2b/pta=2", "ptb=3a/pta=3", "ptb=3b/pta=3"); } + @Test + public void testCreatePartitionsToMetastore() throws Exception { + prepareTestAddPartitionsToMetastore(); + + // add partition + tEnv.executeSql( + "ALTER TABLE t ADD PARTITION (ptb = '1c', pta = 1) PARTITION (ptb = '1d', pta = 6)") + .await(); + assertThat(hiveShell.executeQuery("show partitions t")) + .containsExactlyInAnyOrder( + "ptb=1a/pta=1", + "ptb=1b/pta=1", + "ptb=1c/pta=1", + "ptb=1d/pta=6", + "ptb=2a/pta=2", + "ptb=2b/pta=2", + "ptb=3a/pta=3", + "ptb=3b/pta=3"); + } + @Test public void testAddPartitionsForTag() throws Exception { tEnv.executeSql( @@ -1891,6 +1901,49 @@ public void testExpiredPartitionsSyncToMetastore() throws Exception { .containsExactlyInAnyOrder("dt=9998-06-15", "dt=9999-06-15"); } + @Test + public void testView() throws Exception { + tEnv.executeSql("CREATE TABLE t ( a INT, b STRING ) WITH ( 'file.format' = 'avro' )") + .await(); + tEnv.executeSql("INSERT INTO t VALUES (1, 'Hi'), (2, 'Hello')").await(); + + // test flink view + tEnv.executeSql("CREATE VIEW flink_v AS SELECT a + 1, b FROM t").await(); + assertThat(collect("SELECT * FROM flink_v")) + .containsExactlyInAnyOrder(Row.of(2, "Hi"), Row.of(3, "Hello")); + assertThat(hiveShell.executeQuery("SELECT * FROM flink_v")) + .containsExactlyInAnyOrder("2\tHi", "3\tHello"); + + // test hive view + hiveShell.executeQuery("CREATE VIEW hive_v AS SELECT a + 1, b FROM t"); + assertThat(collect("SELECT * FROM hive_v")) + .containsExactlyInAnyOrder(Row.of(2, "Hi"), Row.of(3, "Hello")); + assertThat(hiveShell.executeQuery("SELECT * FROM hive_v")) + .containsExactlyInAnyOrder("2\tHi", "3\tHello"); + + assertThat(collect("SHOW VIEWS")) + .containsExactlyInAnyOrder(Row.of("flink_v"), Row.of("hive_v")); + + collect("DROP VIEW flink_v"); + collect("DROP VIEW hive_v"); + } + + @Test + public void renameView() throws Exception { + tEnv.executeSql("CREATE TABLE t ( a INT, b STRING ) WITH ( 'file.format' = 'avro' )") + .await(); + tEnv.executeSql("INSERT INTO t VALUES (1, 'Hi'), (2, 'Hello')").await(); + + tEnv.executeSql("CREATE VIEW flink_v AS SELECT a + 1, b FROM t").await(); + tEnv.executeSql("ALTER VIEW flink_v rename to flink_v_rename").await(); + assertThat(collect("SHOW VIEWS")).containsExactlyInAnyOrder(Row.of("flink_v_rename")); + + hiveShell.executeQuery("CREATE VIEW hive_v AS SELECT a + 1, b FROM t"); + tEnv.executeSql("ALTER VIEW hive_v rename to hive_v_rename").await(); + assertThat(collect("SHOW VIEWS")) + .containsExactlyInAnyOrder(Row.of("flink_v_rename"), Row.of("hive_v_rename")); + } + /** Prepare to update a paimon table with a custom path in the paimon file system. */ private void alterTableInFileSystem(TableEnvironment tEnv) throws Exception { tEnv.executeSql( @@ -1954,14 +2007,4 @@ protected List collect(String sql) throws Exception { } return result; } - - private List collectString(String sql) throws Exception { - List result = new ArrayList<>(); - try (CloseableIterator it = tEnv.executeSql(sql).collect()) { - while (it.hasNext()) { - result.add(it.next().toString()); - } - } - return result; - } } diff --git a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveTableSchemaTest.java b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveTableSchemaTest.java index 07cd00c8e67e..fe7aeac0833a 100644 --- a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveTableSchemaTest.java +++ b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveTableSchemaTest.java @@ -153,6 +153,54 @@ public void testMismatchedColumnNameAndType() throws Exception { .hasMessageContaining(expected); } + @Test + public void testSubsetColumnNameAndType() throws Exception { + createSchema(); + Properties properties = new Properties(); + List columns = Arrays.asList("a", "b"); + properties.setProperty("columns", String.join(",", columns)); + properties.setProperty( + "columns.types", + String.join( + ":", + Arrays.asList( + TypeInfoFactory.intTypeInfo.getTypeName(), + TypeInfoFactory.stringTypeInfo.getTypeName(), + TypeInfoFactory.getDecimalTypeInfo(6, 3).getTypeName()))); + properties.setProperty("columns.comments", "\0\0"); + properties.setProperty("location", tempDir.toString()); + List fields = HiveSchema.extract(null, properties).fieldNames(); + assertThat(fields).isEqualTo(columns); + } + + @Test + public void testSupersetColumnNameAndType() throws Exception { + createSchema(); + Properties properties = new Properties(); + properties.setProperty("columns", "a,b,c,d"); + properties.setProperty( + "columns.types", + String.join( + ":", + Arrays.asList( + TypeInfoFactory.intTypeInfo.getTypeName(), + TypeInfoFactory.stringTypeInfo.getTypeName(), + TypeInfoFactory.decimalTypeInfo.getTypeName(), + TypeInfoFactory.stringTypeInfo.getTypeName(), + TypeInfoFactory.getDecimalTypeInfo(6, 3).getTypeName()))); + properties.setProperty("columns.comments", "\0\0"); + properties.setProperty("location", tempDir.toString()); + String expected = + "Hive DDL is a superset of paimon schema! " + + "It is recommended not to write any column definition " + + "as Paimon external table can read schema from the specified location.\n" + + "There are 4 fields in Hive DDL: a, b, c, d\n" + + "There are 3 fields in Paimon schema: a, b, c\n"; + assertThatThrownBy(() -> HiveSchema.extract(null, properties)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining(expected); + } + @Test public void testTooFewColumns() throws Exception { createSchema(); @@ -162,16 +210,7 @@ public void testTooFewColumns() throws Exception { properties.setProperty("columns.types", TypeInfoFactory.intTypeInfo.getTypeName()); properties.setProperty("location", tempDir.toString()); properties.setProperty("columns.comments", ""); - - String expected = - "Hive DDL and paimon schema mismatched! " - + "It is recommended not to write any column definition " - + "as Paimon external table can read schema from the specified location.\n" - + "There are 1 fields in Hive DDL: a\n" - + "There are 3 fields in Paimon schema: a, b, c"; - assertThatExceptionOfType(IllegalArgumentException.class) - .isThrownBy(() -> HiveSchema.extract(null, properties)) - .withMessageContaining(expected); + assertThat(HiveSchema.extract(null, properties)).isInstanceOf(HiveSchema.class); } @Test @@ -194,7 +233,7 @@ public void testTooManyColumns() throws Exception { properties.setProperty("location", tempDir.toString()); String expected = - "Hive DDL and paimon schema mismatched! " + "Hive DDL is a superset of paimon schema! " + "It is recommended not to write any column definition " + "as Paimon external table can read schema from the specified location.\n" + "There are 5 fields in Hive DDL: a, b, c, d, e\n" diff --git a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveWriteITCase.java b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveWriteITCase.java index 57486ec30be9..c99eb9cd1f46 100644 --- a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveWriteITCase.java +++ b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveWriteITCase.java @@ -157,7 +157,7 @@ private String writeData(Table table, String path, List data) throw write.close(); commit.close(); - String tableName = "test_table_" + (UUID.randomUUID().toString().substring(0, 4)); + String tableName = "test_table_" + UUID.randomUUID().toString().replace('-', '_'); hiveShell.execute( String.join( "\n", diff --git a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/iceberg/IcebergHiveMetadataCommitterITCaseBase.java b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/iceberg/IcebergHiveMetadataCommitterITCaseBase.java new file mode 100644 index 000000000000..d0c64c5d3b7f --- /dev/null +++ b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/iceberg/IcebergHiveMetadataCommitterITCaseBase.java @@ -0,0 +1,198 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.iceberg; + +import org.apache.paimon.hive.runner.PaimonEmbeddedHiveRunner; + +import com.klarna.hiverunner.HiveShell; +import com.klarna.hiverunner.annotations.HiveSQL; +import org.apache.flink.table.api.EnvironmentSettings; +import org.apache.flink.table.api.TableEnvironment; +import org.apache.flink.table.api.TableResult; +import org.apache.flink.table.api.internal.TableEnvironmentImpl; +import org.apache.flink.types.Row; +import org.apache.flink.util.CloseableIterator; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.junit.runner.RunWith; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +/** IT cases for {@link IcebergHiveMetadataCommitter}. */ +@RunWith(PaimonEmbeddedHiveRunner.class) +public abstract class IcebergHiveMetadataCommitterITCaseBase { + + @Rule public TemporaryFolder folder = new TemporaryFolder(); + + @HiveSQL(files = {}) + protected static HiveShell hiveShell; + + private String path; + + @Before + public void before() throws Exception { + path = folder.newFolder().toURI().toString(); + } + + @After + public void after() { + hiveShell.execute("DROP DATABASE IF EXISTS test_db CASCADE"); + } + + @Test + public void testPrimaryKeyTable() throws Exception { + TableEnvironment tEnv = + TableEnvironmentImpl.create( + EnvironmentSettings.newInstance().inBatchMode().build()); + tEnv.executeSql( + "CREATE CATALOG my_paimon WITH ( 'type' = 'paimon', 'warehouse' = '" + + path + + "' )"); + tEnv.executeSql("CREATE DATABASE my_paimon.test_db"); + tEnv.executeSql( + "CREATE TABLE my_paimon.test_db.t ( pt INT, id INT, data STRING, PRIMARY KEY (pt, id) NOT ENFORCED ) " + + "PARTITIONED BY (pt) WITH " + + "( 'metadata.iceberg.storage' = 'hive-catalog', 'metadata.iceberg.uri' = '', 'file.format' = 'avro', " + // make sure all changes are visible in iceberg metadata + + " 'full-compaction.delta-commits' = '1' )"); + tEnv.executeSql( + "INSERT INTO my_paimon.test_db.t VALUES " + + "(1, 1, 'apple'), (1, 2, 'pear'), (2, 1, 'cat'), (2, 2, 'dog')") + .await(); + + tEnv.executeSql( + "CREATE CATALOG my_iceberg WITH " + + "( 'type' = 'iceberg', 'catalog-type' = 'hive', 'uri' = '', 'warehouse' = '" + + path + + "', 'cache-enabled' = 'false' )"); + Assert.assertEquals( + Arrays.asList(Row.of("pear", 2, 1), Row.of("dog", 2, 2)), + collect( + tEnv.executeSql( + "SELECT data, id, pt FROM my_iceberg.test_db.t WHERE id = 2 ORDER BY pt, id"))); + + tEnv.executeSql( + "INSERT INTO my_paimon.test_db.t VALUES " + + "(1, 1, 'cherry'), (2, 2, 'elephant')") + .await(); + Assert.assertEquals( + Arrays.asList( + Row.of(1, 1, "cherry"), + Row.of(1, 2, "pear"), + Row.of(2, 1, "cat"), + Row.of(2, 2, "elephant")), + collect(tEnv.executeSql("SELECT * FROM my_iceberg.test_db.t ORDER BY pt, id"))); + + Assert.assertTrue( + hiveShell + .executeQuery("DESC DATABASE EXTENDED test_db") + .toString() + .contains("iceberg/test_db")); + } + + @Test + public void testAppendOnlyTable() throws Exception { + TableEnvironment tEnv = + TableEnvironmentImpl.create( + EnvironmentSettings.newInstance().inBatchMode().build()); + tEnv.executeSql( + "CREATE CATALOG my_paimon WITH ( 'type' = 'paimon', 'warehouse' = '" + + path + + "' )"); + tEnv.executeSql("CREATE DATABASE my_paimon.test_db"); + tEnv.executeSql( + "CREATE TABLE my_paimon.test_db.t ( pt INT, id INT, data STRING ) PARTITIONED BY (pt) WITH " + + "( 'metadata.iceberg.storage' = 'hive-catalog', 'metadata.iceberg.uri' = '', 'file.format' = 'avro' )"); + tEnv.executeSql( + "INSERT INTO my_paimon.test_db.t VALUES " + + "(1, 1, 'apple'), (1, 2, 'pear'), (2, 1, 'cat'), (2, 2, 'dog')") + .await(); + + tEnv.executeSql( + "CREATE CATALOG my_iceberg WITH " + + "( 'type' = 'iceberg', 'catalog-type' = 'hive', 'uri' = '', 'warehouse' = '" + + path + + "', 'cache-enabled' = 'false' )"); + Assert.assertEquals( + Arrays.asList(Row.of("pear", 2, 1), Row.of("dog", 2, 2)), + collect( + tEnv.executeSql( + "SELECT data, id, pt FROM my_iceberg.test_db.t WHERE id = 2 ORDER BY pt, id"))); + + tEnv.executeSql( + "INSERT INTO my_paimon.test_db.t VALUES " + + "(1, 3, 'cherry'), (2, 3, 'elephant')") + .await(); + Assert.assertEquals( + Arrays.asList( + Row.of("pear", 2, 1), + Row.of("cherry", 3, 1), + Row.of("dog", 2, 2), + Row.of("elephant", 3, 2)), + collect( + tEnv.executeSql( + "SELECT data, id, pt FROM my_iceberg.test_db.t WHERE id > 1 ORDER BY pt, id"))); + } + + @Test + public void testCustomMetastoreClass() { + TableEnvironment tEnv = + TableEnvironmentImpl.create( + EnvironmentSettings.newInstance().inBatchMode().build()); + tEnv.executeSql( + "CREATE CATALOG my_paimon WITH ( 'type' = 'paimon', 'warehouse' = '" + + path + + "' )"); + tEnv.executeSql("CREATE DATABASE my_paimon.test_db"); + tEnv.executeSql( + String.format( + "CREATE TABLE my_paimon.test_db.t ( pt INT, id INT, data STRING ) PARTITIONED BY (pt) WITH " + + "( " + + "'metadata.iceberg.storage' = 'hive-catalog', " + + "'metadata.iceberg.uri' = '', " + + "'file.format' = 'avro', " + + "'metadata.iceberg.hive-client-class' = '%s')", + createFailHiveMetaStoreClient())); + Assert.assertThrows( + Exception.class, + () -> + tEnv.executeSql( + "INSERT INTO my_paimon.test_db.t VALUES " + + "(1, 1, 'apple'), (1, 2, 'pear'), (2, 1, 'cat'), (2, 2, 'dog')") + .await()); + } + + protected abstract String createFailHiveMetaStoreClient(); + + private List collect(TableResult result) throws Exception { + List rows = new ArrayList<>(); + try (CloseableIterator it = result.collect()) { + while (it.hasNext()) { + rows.add(it.next()); + } + } + return rows; + } +} diff --git a/paimon-hive/pom.xml b/paimon-hive/pom.xml index 5c306f4749b2..7d1d0f2c499c 100644 --- a/paimon-hive/pom.xml +++ b/paimon-hive/pom.xml @@ -49,6 +49,7 @@ under the License. 4.0.0 0.9.8 1.12.319 + 1.19 diff --git a/paimon-open-api/Makefile b/paimon-open-api/Makefile new file mode 100644 index 000000000000..c3264c83dbd0 --- /dev/null +++ b/paimon-open-api/Makefile @@ -0,0 +1,25 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# See: https://cwiki.apache.org/confluence/display/INFRA/git+-+.asf.yaml+features + + +install: + brew install yq + +generate: + @sh generate.sh diff --git a/paimon-open-api/README.md b/paimon-open-api/README.md new file mode 100644 index 000000000000..9d14a7cdd364 --- /dev/null +++ b/paimon-open-api/README.md @@ -0,0 +1,10 @@ +# Open API spec + +The `rest-catalog-open-api.yaml` defines the REST catalog interface. + +## Generate Open API Spec +```sh +make install +cd paimon-open-api +make generate +``` \ No newline at end of file diff --git a/paimon-open-api/generate.sh b/paimon-open-api/generate.sh new file mode 100755 index 000000000000..619b642ab760 --- /dev/null +++ b/paimon-open-api/generate.sh @@ -0,0 +1,49 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Start the application +cd .. +mvn spotless:apply +mvn clean install -DskipTests +cd ./paimon-open-api +mvn spring-boot:run & +SPRING_PID=$! +# Wait for the application to be ready +RETRY_COUNT=0 +MAX_RETRIES=10 +SLEEP_DURATION=5 + +until $(curl -s -o /dev/null -w "%{http_code}" http://localhost:8080/swagger-api-docs | grep -q "200"); do + ((RETRY_COUNT++)) + if [ $RETRY_COUNT -gt $MAX_RETRIES ]; then + echo "Failed to start the application after $MAX_RETRIES retries." + exit 1 + fi + echo "Application not ready yet. Retrying in $SLEEP_DURATION seconds..." + sleep $SLEEP_DURATION +done + +echo "Application is ready". + +# Generate the OpenAPI specification file +curl -s "http://localhost:8080/swagger-api-docs" | jq -M > ./rest-catalog-open-api.json +yq --prettyPrint -o=yaml ./rest-catalog-open-api.json > ./rest-catalog-open-api.yaml +rm -rf ./rest-catalog-open-api.json +mvn spotless:apply +# Stop the application +echo "Stopping application..." +kill $SPRING_PID \ No newline at end of file diff --git a/paimon-open-api/pom.xml b/paimon-open-api/pom.xml new file mode 100644 index 000000000000..942285243270 --- /dev/null +++ b/paimon-open-api/pom.xml @@ -0,0 +1,86 @@ + + + + 4.0.0 + + org.apache.paimon + paimon-parent + 1.0-SNAPSHOT + + + paimon-open-api + Paimon : Open API + + + 8 + 8 + UTF-8 + + + + org.springframework.boot + spring-boot-starter-web + 2.7.18 + + + ch.qos.logback + logback-classic + + + + + + + org.springdoc + springdoc-openapi-ui + 1.7.0 + + + org.apache.paimon + paimon-core + ${project.version} + + + io.swagger.core.v3 + swagger-annotations + 2.2.20 + + + + + + org.springframework.boot + spring-boot-maven-plugin + 2.7.6 + + + org.apache.maven.plugins + maven-compiler-plugin + + 8 + 8 + + + + + + \ No newline at end of file diff --git a/paimon-open-api/rest-catalog-open-api.yaml b/paimon-open-api/rest-catalog-open-api.yaml new file mode 100644 index 000000000000..9b69b3de2776 --- /dev/null +++ b/paimon-open-api/rest-catalog-open-api.yaml @@ -0,0 +1,220 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +--- +openapi: 3.0.1 +info: + title: RESTCatalog API + description: This API exposes endpoints to RESTCatalog. + license: + name: Apache 2.0 + url: https://www.apache.org/licenses/LICENSE-2.0.html + version: "1.0" +servers: + - url: http://localhost:8080 + description: Server URL in Development environment +paths: + /v1/{prefix}/databases: + get: + tags: + - database + summary: List Databases + operationId: listDatabases + parameters: + - name: prefix + in: path + required: true + schema: + type: string + responses: + "200": + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/ListDatabasesResponse' + "500": + description: Internal Server Error + post: + tags: + - database + summary: Create Databases + operationId: createDatabases + parameters: + - name: prefix + in: path + required: true + schema: + type: string + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/CreateDatabaseRequest' + responses: + "200": + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/CreateDatabaseResponse' + "409": + description: Resource has exist + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + "500": + description: Internal Server Error + /v1/{prefix}/databases/{database}: + get: + tags: + - database + summary: Get Database + operationId: getDatabases + parameters: + - name: prefix + in: path + required: true + schema: + type: string + - name: database + in: path + required: true + schema: + type: string + responses: + "200": + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/GetDatabaseResponse' + "404": + description: Resource not found + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + "500": + description: Internal Server Error + delete: + tags: + - database + summary: Drop Database + operationId: dropDatabases + parameters: + - name: prefix + in: path + required: true + schema: + type: string + - name: database + in: path + required: true + schema: + type: string + responses: + "404": + description: Resource not found + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + "500": + description: Internal Server Error + /v1/config: + get: + tags: + - config + summary: Get Config + operationId: getConfig + responses: + "200": + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/ConfigResponse' + "500": + description: Internal Server Error +components: + schemas: + CreateDatabaseRequest: + type: object + properties: + name: + type: string + ignoreIfExists: + type: boolean + options: + type: object + additionalProperties: + type: string + CreateDatabaseResponse: + type: object + properties: + name: + type: string + options: + type: object + additionalProperties: + type: string + ErrorResponse: + type: object + properties: + message: + type: string + code: + type: integer + format: int32 + stack: + type: array + items: + type: string + DatabaseName: + type: object + properties: + name: + type: string + ListDatabasesResponse: + type: object + properties: + databases: + type: array + items: + $ref: '#/components/schemas/DatabaseName' + GetDatabaseResponse: + type: object + properties: + name: + type: string + options: + type: object + additionalProperties: + type: string + ConfigResponse: + type: object + properties: + defaults: + type: object + additionalProperties: + type: string + overrides: + type: object + additionalProperties: + type: string diff --git a/paimon-open-api/src/main/java/org/apache/paimon/open/api/OpenApiApplication.java b/paimon-open-api/src/main/java/org/apache/paimon/open/api/OpenApiApplication.java new file mode 100644 index 000000000000..76ce4cbf83c6 --- /dev/null +++ b/paimon-open-api/src/main/java/org/apache/paimon/open/api/OpenApiApplication.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.open.api; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; + +/** OpenAPI application. */ +@SpringBootApplication +public class OpenApiApplication { + + public static void main(String[] args) { + SpringApplication.run(OpenApiApplication.class, args); + } +} diff --git a/paimon-open-api/src/main/java/org/apache/paimon/open/api/RESTCatalogController.java b/paimon-open-api/src/main/java/org/apache/paimon/open/api/RESTCatalogController.java new file mode 100644 index 000000000000..19f6f8cdf673 --- /dev/null +++ b/paimon-open-api/src/main/java/org/apache/paimon/open/api/RESTCatalogController.java @@ -0,0 +1,153 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.open.api; + +import org.apache.paimon.rest.ResourcePaths; +import org.apache.paimon.rest.requests.CreateDatabaseRequest; +import org.apache.paimon.rest.responses.ConfigResponse; +import org.apache.paimon.rest.responses.CreateDatabaseResponse; +import org.apache.paimon.rest.responses.DatabaseName; +import org.apache.paimon.rest.responses.ErrorResponse; +import org.apache.paimon.rest.responses.GetDatabaseResponse; +import org.apache.paimon.rest.responses.ListDatabasesResponse; + +import org.apache.paimon.shade.guava30.com.google.common.collect.ImmutableList; + +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.media.Content; +import io.swagger.v3.oas.annotations.media.Schema; +import io.swagger.v3.oas.annotations.parameters.RequestBody; +import io.swagger.v3.oas.annotations.responses.ApiResponse; +import io.swagger.v3.oas.annotations.responses.ApiResponses; +import org.springframework.web.bind.annotation.CrossOrigin; +import org.springframework.web.bind.annotation.DeleteMapping; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RestController; + +import java.util.HashMap; +import java.util.Map; + +/** * RESTCatalog management APIs. */ +@CrossOrigin(origins = "http://localhost:8081") +@RestController +public class RESTCatalogController { + + @Operation( + summary = "Get Config", + tags = {"config"}) + @ApiResponses({ + @ApiResponse( + responseCode = "200", + content = {@Content(schema = @Schema(implementation = ConfigResponse.class))}), + @ApiResponse( + responseCode = "500", + content = {@Content(schema = @Schema())}) + }) + @GetMapping(ResourcePaths.V1_CONFIG) + public ConfigResponse getConfig() { + Map defaults = new HashMap<>(); + Map overrides = new HashMap<>(); + return new ConfigResponse(defaults, overrides); + } + + @Operation( + summary = "List Databases", + tags = {"database"}) + @ApiResponses({ + @ApiResponse( + responseCode = "200", + content = { + @Content(schema = @Schema(implementation = ListDatabasesResponse.class)) + }), + @ApiResponse( + responseCode = "500", + content = {@Content(schema = @Schema())}) + }) + @GetMapping("/v1/{prefix}/databases") + public ListDatabasesResponse listDatabases(@PathVariable String prefix) { + return new ListDatabasesResponse(ImmutableList.of(new DatabaseName("account"))); + } + + @Operation( + summary = "Create Databases", + tags = {"database"}) + @ApiResponses({ + @ApiResponse( + responseCode = "200", + content = { + @Content(schema = @Schema(implementation = CreateDatabaseResponse.class)) + }), + @ApiResponse( + responseCode = "409", + description = "Resource has exist", + content = { + @Content( + schema = @Schema(implementation = ErrorResponse.class), + mediaType = "application/json") + }), + @ApiResponse( + responseCode = "500", + content = {@Content(schema = @Schema())}) + }) + @PostMapping("/v1/{prefix}/databases") + public CreateDatabaseResponse createDatabases( + @PathVariable String prefix, @RequestBody CreateDatabaseRequest request) { + Map properties = new HashMap<>(); + return new CreateDatabaseResponse("name", properties); + } + + @Operation( + summary = "Get Database", + tags = {"database"}) + @ApiResponses({ + @ApiResponse( + responseCode = "200", + content = {@Content(schema = @Schema(implementation = GetDatabaseResponse.class))}), + @ApiResponse( + responseCode = "404", + description = "Resource not found", + content = {@Content(schema = @Schema(implementation = ErrorResponse.class))}), + @ApiResponse( + responseCode = "500", + content = {@Content(schema = @Schema())}) + }) + @GetMapping("/v1/{prefix}/databases/{database}") + public GetDatabaseResponse getDatabases( + @PathVariable String prefix, @PathVariable String database) { + Map options = new HashMap<>(); + return new GetDatabaseResponse("name", options); + } + + @Operation( + summary = "Drop Database", + tags = {"database"}) + @ApiResponses({ + @ApiResponse( + responseCode = "404", + description = "Resource not found", + content = {@Content(schema = @Schema(implementation = ErrorResponse.class))}), + @ApiResponse( + responseCode = "500", + content = {@Content(schema = @Schema())}) + }) + @DeleteMapping("/v1/{prefix}/databases/{database}") + public void dropDatabases(@PathVariable String prefix, @PathVariable String database) {} +} diff --git a/paimon-open-api/src/main/java/org/apache/paimon/open/api/config/OpenAPIConfig.java b/paimon-open-api/src/main/java/org/apache/paimon/open/api/config/OpenAPIConfig.java new file mode 100644 index 000000000000..71ac066d4a70 --- /dev/null +++ b/paimon-open-api/src/main/java/org/apache/paimon/open/api/config/OpenAPIConfig.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.open.api.config; + +import io.swagger.v3.oas.models.OpenAPI; +import io.swagger.v3.oas.models.info.Info; +import io.swagger.v3.oas.models.info.License; +import io.swagger.v3.oas.models.responses.ApiResponses; +import io.swagger.v3.oas.models.servers.Server; +import org.springdoc.core.customizers.OpenApiCustomiser; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; + +/** Config for OpenAPI. */ +@Configuration +public class OpenAPIConfig { + + @Value("${openapi.url}") + private String devUrl; + + @Bean + public OpenAPI restCatalogOpenAPI() { + Server server = new Server(); + server.setUrl(devUrl); + server.setDescription("Server URL in Development environment"); + + License mitLicense = + new License() + .name("Apache 2.0") + .url("https://www.apache.org/licenses/LICENSE-2.0.html"); + + Info info = + new Info() + .title("RESTCatalog API") + .version("1.0") + .description("This API exposes endpoints to RESTCatalog.") + .license(mitLicense); + List servers = new ArrayList<>(); + servers.add(server); + return new OpenAPI().info(info).servers(servers); + } + + /** Sort response alphabetically. So the api generate will in same order everytime. */ + @Bean + public OpenApiCustomiser sortResponseAlphabetically() { + return openApi -> { + openApi.getPaths() + .values() + .forEach( + path -> + path.readOperations() + .forEach( + operation -> { + ApiResponses responses = + operation.getResponses(); + if (responses != null) { + ApiResponses sortedResponses = + new ApiResponses(); + List keys = + new ArrayList<>( + responses.keySet()); + keys.sort(Comparator.naturalOrder()); + + for (String key : keys) { + sortedResponses.addApiResponse( + key, responses.get(key)); + } + + operation.setResponses(sortedResponses); + } + })); + }; + } +} diff --git a/paimon-open-api/src/main/resources/application.properties b/paimon-open-api/src/main/resources/application.properties new file mode 100644 index 000000000000..1e7a987c9d40 --- /dev/null +++ b/paimon-open-api/src/main/resources/application.properties @@ -0,0 +1,26 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +springdoc.swagger-ui.path=/swagger-api +springdoc.api-docs.path=/swagger-api-docs +springdoc.swagger-ui.deepLinking=true +springdoc.swagger-ui.tryItOutEnabled=true +springdoc.swagger-ui.filter=true +springdoc.swagger-ui.tagsSorter=alpha +springdoc.swagger-ui.operations-sorter=alpha +# define response default media type +springdoc.default-produces-media-type=application/json +openapi.url=http://localhost:8080 diff --git a/paimon-service/paimon-service-client/src/main/java/org/apache/paimon/service/network/NetworkClient.java b/paimon-service/paimon-service-client/src/main/java/org/apache/paimon/service/network/NetworkClient.java index 2a16d53b97de..748d1d2f055e 100644 --- a/paimon-service/paimon-service-client/src/main/java/org/apache/paimon/service/network/NetworkClient.java +++ b/paimon-service/paimon-service-client/src/main/java/org/apache/paimon/service/network/NetworkClient.java @@ -143,30 +143,23 @@ public CompletableFuture sendRequest( new IllegalStateException(clientName + " is already shut down.")); } - final ServerConnection connection = - connections.computeIfAbsent( - serverAddress, - ignored -> { - final ServerConnection newConnection = - ServerConnection.createPendingConnection( - clientName, messageSerializer, stats); - bootstrap - .connect(serverAddress.getAddress(), serverAddress.getPort()) - .addListener( - (ChannelFutureListener) - newConnection::establishConnection); - - newConnection - .getCloseFuture() - .handle( - (ignoredA, ignoredB) -> - connections.remove( - serverAddress, newConnection)); - - return newConnection; - }); - - return connection.sendRequest(request); + ServerConnection serverConnection = connections.get(serverAddress); + if (serverConnection == null) { + final ServerConnection newConnection = + ServerConnection.createPendingConnection(clientName, messageSerializer, stats); + serverConnection = newConnection; + connections.put(serverAddress, newConnection); + bootstrap + .connect(serverAddress.getAddress(), serverAddress.getPort()) + .addListener((ChannelFutureListener) newConnection::establishConnection); + + newConnection + .getCloseFuture() + .handle( + (ignoredA, ignoredB) -> + connections.remove(serverAddress, newConnection)); + } + return serverConnection.sendRequest(request); } /** diff --git a/paimon-spark/paimon-spark-3.2/pom.xml b/paimon-spark/paimon-spark-3.2/pom.xml index 24e5198e9b2e..957319b47dab 100644 --- a/paimon-spark/paimon-spark-3.2/pom.xml +++ b/paimon-spark/paimon-spark-3.2/pom.xml @@ -38,250 +38,76 @@ under the License. org.apache.paimon - paimon-spark-common + paimon-spark3-common ${project.version} - org.scala-lang - scala-library - ${scala.version} - - - org.scala-lang - scala-reflect - ${scala.version} - - - org.scala-lang - scala-compiler - ${scala.version} + org.apache.paimon + paimon-spark-common_${scala.binary.version} + ${project.version} org.apache.spark - spark-sql_2.12 + spark-sql_${scala.binary.version} ${spark.version} - - - log4j - log4j - - - org.slf4j - slf4j-log4j12 - - - - org.apache.logging.log4j - log4j-slf4j2-impl - - - org.apache.orc - orc-core - - - org.apache.orc - orc-mapreduce - - - org.apache.parquet - parquet-column - - - org.apache.parquet - parquet-hadoop - - org.apache.spark - spark-core_2.12 + spark-core_${scala.binary.version} ${spark.version} - - - log4j - log4j - - - org.slf4j - slf4j-log4j12 - - - - org.apache.logging.log4j - log4j-slf4j2-impl - - - org.apache.orc - orc-core - - - org.apache.orc - orc-mapreduce - - - org.apache.parquet - parquet-column - - - com.google.protobuf - protobuf-java - - org.apache.paimon - paimon-spark-common + paimon-bundle + + + + + + org.apache.paimon + paimon-spark-ut ${project.version} - test-jar + tests test + org.apache.spark - spark-sql_2.12 + spark-sql_${scala.binary.version} ${spark.version} tests test - - - log4j - log4j - - - org.slf4j - slf4j-log4j12 - - - org.apache.orc - orc-core - - - org.apache.parquet - parquet-column - - - org.apache.parquet - parquet-hadoop - - + org.apache.spark - spark-core_2.12 + spark-catalyst_${scala.binary.version} ${spark.version} tests test - - - log4j - log4j - - - org.slf4j - slf4j-log4j12 - - - org.apache.logging.log4j - log4j-slf4j2-impl - - - org.apache.orc - orc-core - - + org.apache.spark - spark-catalyst_2.12 + spark-core_${scala.binary.version} ${spark.version} tests test - - - log4j - log4j - - - org.slf4j - slf4j-log4j12 - - - org.apache.orc - orc-core - - - - org.scalatest - scalatest_${scala.binary.version} - 3.1.0 - test - org.apache.spark - spark-hive_2.12 + spark-hive_${scala.binary.version} ${spark.version} test - - - log4j - log4j - - - org.slf4j - slf4j-log4j12 - - - org.apache.orc - orc-core - - - com.google.protobuf - protobuf-java - - - - - org.apache.paimon - paimon-hive-common - ${project.version} - test - - - org.apache.paimon - paimon-hive-common - ${project.version} - tests - test-jar - test - - - - net.alchim31.maven - scala-maven-plugin - ${scala-maven-plugin.version} - - - -nobootcp - -target:jvm-${target.java.version} - - false - - - - org.apache.maven.plugins - maven-compiler-plugin - - - org.apache.maven.plugins @@ -299,69 +125,20 @@ under the License. * com/github/luben/zstd/** + **/*libzstd-jni-*.so + **/*libzstd-jni-*.dll - org.apache.paimon:paimon-spark-common + org.apache.paimon:paimon-spark3-common - - net.alchim31.maven - scala-maven-plugin - ${scala-maven-plugin.version} - - - - scala-compile-first - process-resources - - add-source - compile - - - - - - scala-test-compile - process-test-resources - - testCompile - - - - - - org.scalatest - scalatest-maven-plugin - ${scalatest-maven-plugin.version} - - ${project.build.directory}/surefire-reports - . - -ea -Xmx4g -Xss4m -XX:MaxMetaspaceSize=2g -XX:ReservedCodeCacheSize=${CodeCacheSize} ${extraJavaTestArgs} -Dio.netty.tryReflectionSetAccessible=true - PaimonTestSuite.txt - - once - - - - test - - test - - - - diff --git a/paimon-spark/paimon-spark-3.2/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/RewritePaimonViewCommands.scala b/paimon-spark/paimon-spark-3.2/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/RewritePaimonViewCommands.scala new file mode 100644 index 000000000000..e759edd0c2c6 --- /dev/null +++ b/paimon-spark/paimon-spark-3.2/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/RewritePaimonViewCommands.scala @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.catalyst.parser.extensions + +import org.apache.paimon.spark.catalog.SupportView +import org.apache.paimon.spark.catalyst.plans.logical.{CreatePaimonView, DropPaimonView, ResolvedIdentifier, ShowPaimonViews} + +import org.apache.spark.sql.SparkSession +import org.apache.spark.sql.catalyst.analysis.{CTESubstitution, ResolvedNamespace, UnresolvedView} +import org.apache.spark.sql.catalyst.plans.logical._ +import org.apache.spark.sql.catalyst.rules.Rule +import org.apache.spark.sql.connector.catalog.{CatalogManager, LookupCatalog} + +case class RewritePaimonViewCommands(spark: SparkSession) + extends Rule[LogicalPlan] + with LookupCatalog { + + protected lazy val catalogManager: CatalogManager = spark.sessionState.catalogManager + + override def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperatorsUp { + + case CreateViewStatement( + ResolvedIdent(resolved), + userSpecifiedColumns, + comment, + properties, + Some(originalText), + child, + allowExisting, + replace, + _) => + CreatePaimonView( + child = resolved, + queryText = originalText, + query = CTESubstitution.apply(child), + columnAliases = userSpecifiedColumns.map(_._1), + columnComments = userSpecifiedColumns.map(_._2.orElse(Option.empty)), + comment = comment, + properties = properties, + allowExisting = allowExisting, + replace = replace + ) + + case DropView(ResolvedIdent(resolved), ifExists: Boolean) => + DropPaimonView(resolved, ifExists) + + case ShowViews(_, pattern, output) if catalogManager.currentCatalog.isInstanceOf[SupportView] => + ShowPaimonViews( + ResolvedNamespace(catalogManager.currentCatalog, catalogManager.currentNamespace), + pattern, + output) + } + + private object ResolvedIdent { + def unapply(unresolved: Any): Option[ResolvedIdentifier] = unresolved match { + case CatalogAndIdentifier(viewCatalog: SupportView, ident) => + Some(ResolvedIdentifier(viewCatalog, ident)) + case UnresolvedView(CatalogAndIdentifier(viewCatalog: SupportView, ident), _, _, _) => + Some(ResolvedIdentifier(viewCatalog, ident)) + case _ => + None + } + } +} diff --git a/paimon-spark/paimon-spark-3.2/src/test/scala/org/apache/paimon/spark/sql/PaimonViewTest.scala b/paimon-spark/paimon-spark-3.2/src/test/scala/org/apache/paimon/spark/sql/PaimonViewTest.scala new file mode 100644 index 000000000000..6ab8a2671b51 --- /dev/null +++ b/paimon-spark/paimon-spark-3.2/src/test/scala/org/apache/paimon/spark/sql/PaimonViewTest.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.sql + +class PaimonViewTest extends PaimonViewTestBase {} diff --git a/paimon-spark/paimon-spark-3.2/src/test/scala/org/apache/paimon/spark/sql/TagDdlTest.scala b/paimon-spark/paimon-spark-3.2/src/test/scala/org/apache/paimon/spark/sql/TagDdlTest.scala new file mode 100644 index 000000000000..2d199491dc0a --- /dev/null +++ b/paimon-spark/paimon-spark-3.2/src/test/scala/org/apache/paimon/spark/sql/TagDdlTest.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.sql + +class TagDdlTest extends PaimonTagDdlTestBase diff --git a/paimon-spark/paimon-spark-3.3/pom.xml b/paimon-spark/paimon-spark-3.3/pom.xml index 3950ff9f8aca..0a390d926789 100644 --- a/paimon-spark/paimon-spark-3.3/pom.xml +++ b/paimon-spark/paimon-spark-3.3/pom.xml @@ -38,245 +38,76 @@ under the License. org.apache.paimon - paimon-spark-common + paimon-spark3-common ${project.version} - org.scala-lang - scala-library - ${scala.version} - - - org.scala-lang - scala-reflect - ${scala.version} - - - org.scala-lang - scala-compiler - ${scala.version} + org.apache.paimon + paimon-spark-common_${scala.binary.version} + ${project.version} org.apache.spark - spark-sql_2.12 + spark-sql_${scala.binary.version} ${spark.version} - - - log4j - log4j - - - org.slf4j - slf4j-log4j12 - - - - org.apache.logging.log4j - log4j-slf4j2-impl - - - org.apache.orc - orc-core - - - org.apache.parquet - parquet-column - - - org.apache.parquet - parquet-hadoop - - org.apache.spark - spark-core_2.12 + spark-core_${scala.binary.version} ${spark.version} - - - log4j - log4j - - - org.slf4j - slf4j-log4j12 - - - - org.apache.logging.log4j - log4j-slf4j2-impl - - - org.apache.orc - orc-core - - - org.apache.orc - orc-mapreduce - - - org.apache.parquet - parquet-column - - - com.google.protobuf - protobuf-java - - org.apache.paimon - paimon-spark-common + paimon-bundle + + + + + + org.apache.paimon + paimon-spark-ut ${project.version} tests test + org.apache.spark - spark-sql_2.12 + spark-sql_${scala.binary.version} ${spark.version} tests test - - - log4j - log4j - - - org.slf4j - slf4j-log4j12 - - - org.apache.orc - orc-core - - - org.apache.parquet - parquet-column - - - org.apache.parquet - parquet-hadoop - - + org.apache.spark - spark-core_2.12 + spark-catalyst_${scala.binary.version} ${spark.version} tests test - - - log4j - log4j - - - org.slf4j - slf4j-log4j12 - - - org.apache.logging.log4j - log4j-slf4j2-impl - - - org.apache.orc - orc-core - - + org.apache.spark - spark-catalyst_2.12 + spark-core_${scala.binary.version} ${spark.version} tests test - - - log4j - log4j - - - org.slf4j - slf4j-log4j12 - - - org.apache.orc - orc-core - - - - - org.scalatest - scalatest_${scala.binary.version} - 3.1.0 - test + org.apache.spark - spark-hive_2.12 + spark-hive_${scala.binary.version} ${spark.version} test - - - log4j - log4j - - - org.slf4j - slf4j-log4j12 - - - org.apache.orc - orc-core - - - com.google.protobuf - protobuf-java - - - - - org.apache.paimon - paimon-hive-common - ${project.version} - test - - - org.apache.paimon - paimon-hive-common - ${project.version} - tests - test-jar - test - - - - net.alchim31.maven - scala-maven-plugin - ${scala-maven-plugin.version} - - - -nobootcp - -target:jvm-${target.java.version} - - false - - - - org.apache.maven.plugins - maven-compiler-plugin - - - org.apache.maven.plugins @@ -294,69 +125,20 @@ under the License. * com/github/luben/zstd/** + **/*libzstd-jni-*.so + **/*libzstd-jni-*.dll - org.apache.paimon:paimon-spark-common + org.apache.paimon:paimon-spark3-common - - net.alchim31.maven - scala-maven-plugin - ${scala-maven-plugin.version} - - - - scala-compile-first - process-resources - - add-source - compile - - - - - - scala-test-compile - process-test-resources - - testCompile - - - - - - org.scalatest - scalatest-maven-plugin - ${scalatest-maven-plugin.version} - - ${project.build.directory}/surefire-reports - . - -ea -Xmx4g -Xss4m -XX:MaxMetaspaceSize=2g -XX:ReservedCodeCacheSize=${CodeCacheSize} ${extraJavaTestArgs} -Dio.netty.tryReflectionSetAccessible=true - PaimonTestSuite.txt - - once - - - - test - - test - - - - diff --git a/paimon-spark/paimon-spark-3.3/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/RewritePaimonViewCommands.scala b/paimon-spark/paimon-spark-3.3/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/RewritePaimonViewCommands.scala new file mode 100644 index 000000000000..5d57cda2f34b --- /dev/null +++ b/paimon-spark/paimon-spark-3.3/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/RewritePaimonViewCommands.scala @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.catalyst.parser.extensions + +import org.apache.paimon.spark.catalog.SupportView +import org.apache.paimon.spark.catalyst.plans.logical.{CreatePaimonView, DropPaimonView, ResolvedIdentifier, ShowPaimonViews} + +import org.apache.spark.sql.SparkSession +import org.apache.spark.sql.catalyst.analysis.{CTESubstitution, ResolvedNamespace, UnresolvedDBObjectName, UnresolvedView} +import org.apache.spark.sql.catalyst.plans.logical._ +import org.apache.spark.sql.catalyst.rules.Rule +import org.apache.spark.sql.connector.catalog.{CatalogManager, LookupCatalog} + +case class RewritePaimonViewCommands(spark: SparkSession) + extends Rule[LogicalPlan] + with LookupCatalog { + + protected lazy val catalogManager: CatalogManager = spark.sessionState.catalogManager + + override def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperatorsUp { + + case CreateView( + ResolvedIdent(resolved), + userSpecifiedColumns, + comment, + properties, + Some(queryText), + query, + allowExisting, + replace) => + CreatePaimonView( + child = resolved, + queryText = queryText, + query = CTESubstitution.apply(query), + columnAliases = userSpecifiedColumns.map(_._1), + columnComments = userSpecifiedColumns.map(_._2.orElse(Option.empty)), + comment = comment, + properties = properties, + allowExisting = allowExisting, + replace = replace + ) + + case DropView(ResolvedIdent(resolved), ifExists: Boolean) => + DropPaimonView(resolved, ifExists) + + case ShowViews(_, pattern, output) if catalogManager.currentCatalog.isInstanceOf[SupportView] => + ShowPaimonViews( + ResolvedNamespace(catalogManager.currentCatalog, catalogManager.currentNamespace), + pattern, + output) + } + + private object ResolvedIdent { + def unapply(unresolved: Any): Option[ResolvedIdentifier] = unresolved match { + case UnresolvedDBObjectName(CatalogAndIdentifier(viewCatalog: SupportView, ident), _) => + Some(ResolvedIdentifier(viewCatalog, ident)) + case UnresolvedView(CatalogAndIdentifier(viewCatalog: SupportView, ident), _, _, _) => + Some(ResolvedIdentifier(viewCatalog, ident)) + case _ => + None + } + } +} diff --git a/paimon-spark/paimon-spark-3.3/src/test/scala/org/apache/paimon/spark/DataFrameWriteTest.scala b/paimon-spark/paimon-spark-3.3/src/test/scala/org/apache/paimon/spark/DataFrameWriteTest.scala index a3cecfc72e1d..cb449edb4ccb 100644 --- a/paimon-spark/paimon-spark-3.3/src/test/scala/org/apache/paimon/spark/DataFrameWriteTest.scala +++ b/paimon-spark/paimon-spark-3.3/src/test/scala/org/apache/paimon/spark/DataFrameWriteTest.scala @@ -18,10 +18,15 @@ package org.apache.paimon.spark +import org.apache.spark.SparkConf import org.junit.jupiter.api.Assertions class DataFrameWriteTest extends PaimonSparkTestBase { + override protected def sparkConf: SparkConf = { + super.sparkConf.set("spark.sql.catalog.paimon.cache-enabled", "false") + } + test("Paimon: DataFrameWrite.saveAsTable") { import testImplicits._ diff --git a/paimon-spark/paimon-spark-3.3/src/test/scala/org/apache/paimon/spark/sql/InsertOverwriteTest.scala b/paimon-spark/paimon-spark-3.3/src/test/scala/org/apache/paimon/spark/sql/InsertOverwriteTest.scala index 304b814b33d3..219d57c865c8 100644 --- a/paimon-spark/paimon-spark-3.3/src/test/scala/org/apache/paimon/spark/sql/InsertOverwriteTest.scala +++ b/paimon-spark/paimon-spark-3.3/src/test/scala/org/apache/paimon/spark/sql/InsertOverwriteTest.scala @@ -248,7 +248,7 @@ class InsertOverwriteTest extends PaimonSparkTestBase { spark.sql("SELECT * FROM T ORDER BY a, b"), Row(1, 3, "3") :: Row(2, 4, "4") :: Nil) - withSQLConf("spark.sql.sources.partitionOverwriteMode" -> "dynamic") { + withSparkSQLConf("spark.sql.sources.partitionOverwriteMode" -> "dynamic") { // dynamic overwrite the a=1 partition spark.sql("INSERT OVERWRITE T VALUES (1, 5, '5'), (1, 7, '7')") checkAnswer( @@ -289,7 +289,7 @@ class InsertOverwriteTest extends PaimonSparkTestBase { "ptv2", 22) :: Nil) - withSQLConf("spark.sql.sources.partitionOverwriteMode" -> "dynamic") { + withSparkSQLConf("spark.sql.sources.partitionOverwriteMode" -> "dynamic") { // dynamic overwrite the pt2=22 partition spark.sql( "INSERT OVERWRITE T PARTITION (pt2 = 22) VALUES (3, 'c2', 'ptv1'), (4, 'd2', 'ptv3')") diff --git a/paimon-spark/paimon-spark-3.3/src/test/scala/org/apache/paimon/spark/sql/PaimonViewTest.scala b/paimon-spark/paimon-spark-3.3/src/test/scala/org/apache/paimon/spark/sql/PaimonViewTest.scala new file mode 100644 index 000000000000..6ab8a2671b51 --- /dev/null +++ b/paimon-spark/paimon-spark-3.3/src/test/scala/org/apache/paimon/spark/sql/PaimonViewTest.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.sql + +class PaimonViewTest extends PaimonViewTestBase {} diff --git a/paimon-spark/paimon-spark-3.3/src/test/scala/org/apache/paimon/spark/sql/TagDdlTest.scala b/paimon-spark/paimon-spark-3.3/src/test/scala/org/apache/paimon/spark/sql/TagDdlTest.scala new file mode 100644 index 000000000000..2d199491dc0a --- /dev/null +++ b/paimon-spark/paimon-spark-3.3/src/test/scala/org/apache/paimon/spark/sql/TagDdlTest.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.sql + +class TagDdlTest extends PaimonTagDdlTestBase diff --git a/paimon-spark/paimon-spark-3.4/pom.xml b/paimon-spark/paimon-spark-3.4/pom.xml index 000acdc26c3c..0f4cb30e4f7f 100644 --- a/paimon-spark/paimon-spark-3.4/pom.xml +++ b/paimon-spark/paimon-spark-3.4/pom.xml @@ -38,245 +38,76 @@ under the License. org.apache.paimon - paimon-spark-common + paimon-spark3-common ${project.version} - org.scala-lang - scala-library - ${scala.version} - - - org.scala-lang - scala-reflect - ${scala.version} - - - org.scala-lang - scala-compiler - ${scala.version} + org.apache.paimon + paimon-spark-common_${scala.binary.version} + ${project.version} org.apache.spark - spark-sql_2.12 + spark-sql_${scala.binary.version} ${spark.version} - - - log4j - log4j - - - org.slf4j - slf4j-log4j12 - - - - org.apache.logging.log4j - log4j-slf4j2-impl - - - org.apache.orc - orc-core - - - org.apache.parquet - parquet-column - - - org.apache.parquet - parquet-hadoop - - org.apache.spark - spark-core_2.12 + spark-core_${scala.binary.version} ${spark.version} - - - log4j - log4j - - - org.slf4j - slf4j-log4j12 - - - - org.apache.logging.log4j - log4j-slf4j2-impl - - - org.apache.orc - orc-core - - - org.apache.orc - orc-mapreduce - - - org.apache.parquet - parquet-column - - - com.google.protobuf - protobuf-java - - org.apache.paimon - paimon-spark-common + paimon-bundle + + + + + + org.apache.paimon + paimon-spark-ut ${project.version} tests test + org.apache.spark - spark-sql_2.12 + spark-sql_${scala.binary.version} ${spark.version} tests test - - - log4j - log4j - - - org.slf4j - slf4j-log4j12 - - - org.apache.orc - orc-core - - - org.apache.parquet - parquet-column - - - org.apache.parquet - parquet-hadoop - - + org.apache.spark - spark-core_2.12 + spark-catalyst_${scala.binary.version} ${spark.version} tests test - - - log4j - log4j - - - org.slf4j - slf4j-log4j12 - - - org.apache.logging.log4j - log4j-slf4j2-impl - - - org.apache.orc - orc-core - - + org.apache.spark - spark-catalyst_2.12 + spark-core_${scala.binary.version} ${spark.version} tests test - - - log4j - log4j - - - org.slf4j - slf4j-log4j12 - - - org.apache.orc - orc-core - - - - - org.scalatest - scalatest_${scala.binary.version} - 3.1.0 - test + org.apache.spark - spark-hive_2.12 + spark-hive_${scala.binary.version} ${spark.version} test - - - log4j - log4j - - - org.slf4j - slf4j-log4j12 - - - org.apache.orc - orc-core - - - com.google.protobuf - protobuf-java - - - - - org.apache.paimon - paimon-hive-common - ${project.version} - test - - - org.apache.paimon - paimon-hive-common - ${project.version} - tests - test-jar - test - - - - net.alchim31.maven - scala-maven-plugin - ${scala-maven-plugin.version} - - - -nobootcp - -target:jvm-${target.java.version} - - false - - - - org.apache.maven.plugins - maven-compiler-plugin - - - org.apache.maven.plugins @@ -294,69 +125,20 @@ under the License. * com/github/luben/zstd/** + **/*libzstd-jni-*.so + **/*libzstd-jni-*.dll - org.apache.paimon:paimon-spark-common + org.apache.paimon:paimon-spark3-common - - net.alchim31.maven - scala-maven-plugin - ${scala-maven-plugin.version} - - - - scala-compile-first - process-resources - - add-source - compile - - - - - - scala-test-compile - process-test-resources - - testCompile - - - - - - org.scalatest - scalatest-maven-plugin - ${scalatest-maven-plugin.version} - - ${project.build.directory}/surefire-reports - . - -ea -Xmx4g -Xss4m -XX:MaxMetaspaceSize=2g -XX:ReservedCodeCacheSize=${CodeCacheSize} ${extraJavaTestArgs} -Dio.netty.tryReflectionSetAccessible=true - PaimonTestSuite.txt - - once - - - - test - - test - - - - diff --git a/paimon-spark/paimon-spark-3.4/src/test/scala/org/apache/paimon/spark/PaimonSinkTest.scala b/paimon-spark/paimon-spark-3.4/src/test/scala/org/apache/paimon/spark/PaimonSinkTest.scala index 18fb9e116ba4..ab4a9bcd9dbf 100644 --- a/paimon-spark/paimon-spark-3.4/src/test/scala/org/apache/paimon/spark/PaimonSinkTest.scala +++ b/paimon-spark/paimon-spark-3.4/src/test/scala/org/apache/paimon/spark/PaimonSinkTest.scala @@ -18,6 +18,7 @@ package org.apache.paimon.spark +import org.apache.spark.SparkConf import org.apache.spark.sql.{Dataset, Row} import org.apache.spark.sql.execution.streaming.MemoryStream import org.apache.spark.sql.functions.{col, mean, window} @@ -27,6 +28,10 @@ import java.sql.Date class PaimonSinkTest extends PaimonSparkTestBase with StreamTest { + override protected def sparkConf: SparkConf = { + super.sparkConf.set("spark.sql.catalog.paimon.cache-enabled", "false") + } + import testImplicits._ test("Paimon Sink: forEachBatch") { diff --git a/paimon-spark/paimon-spark-3.4/src/test/scala/org/apache/paimon/spark/sql/PaimonViewTest.scala b/paimon-spark/paimon-spark-3.4/src/test/scala/org/apache/paimon/spark/sql/PaimonViewTest.scala new file mode 100644 index 000000000000..6ab8a2671b51 --- /dev/null +++ b/paimon-spark/paimon-spark-3.4/src/test/scala/org/apache/paimon/spark/sql/PaimonViewTest.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.sql + +class PaimonViewTest extends PaimonViewTestBase {} diff --git a/paimon-spark/paimon-spark-3.4/src/test/scala/org/apache/paimon/spark/sql/TagDdlTest.scala b/paimon-spark/paimon-spark-3.4/src/test/scala/org/apache/paimon/spark/sql/TagDdlTest.scala new file mode 100644 index 000000000000..2d199491dc0a --- /dev/null +++ b/paimon-spark/paimon-spark-3.4/src/test/scala/org/apache/paimon/spark/sql/TagDdlTest.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.sql + +class TagDdlTest extends PaimonTagDdlTestBase diff --git a/paimon-spark/paimon-spark-3.5/pom.xml b/paimon-spark/paimon-spark-3.5/pom.xml index 0ba453068c07..1b9c96888908 100644 --- a/paimon-spark/paimon-spark-3.5/pom.xml +++ b/paimon-spark/paimon-spark-3.5/pom.xml @@ -38,237 +38,76 @@ under the License. org.apache.paimon - paimon-spark-common + paimon-spark3-common ${project.version} - org.scala-lang - scala-library - ${scala.version} - - - org.scala-lang - scala-reflect - ${scala.version} - - - org.scala-lang - scala-compiler - ${scala.version} + org.apache.paimon + paimon-spark-common_${scala.binary.version} + ${project.version} org.apache.spark - spark-sql_2.12 + spark-sql_${scala.binary.version} ${spark.version} - - - log4j - log4j - - - org.slf4j - slf4j-log4j12 - - - - org.apache.logging.log4j - log4j-slf4j2-impl - - - org.apache.orc - orc-core - - - org.apache.parquet - parquet-column - - org.apache.spark - spark-core_2.12 + spark-core_${scala.binary.version} ${spark.version} - - - log4j - log4j - - - org.slf4j - slf4j-log4j12 - - - - org.apache.logging.log4j - log4j-slf4j2-impl - - - org.apache.orc - orc-core - - - org.apache.orc - orc-mapreduce - - - org.apache.parquet - parquet-column - - - com.google.protobuf - protobuf-java - - org.apache.paimon - paimon-spark-common + paimon-bundle + + + + + + org.apache.paimon + paimon-spark-ut ${project.version} tests test + org.apache.spark - spark-sql_2.12 + spark-sql_${scala.binary.version} ${spark.version} tests test - - - log4j - log4j - - - org.slf4j - slf4j-log4j12 - - - org.apache.orc - orc-core - - - org.apache.parquet - parquet-column - - + org.apache.spark - spark-core_2.12 + spark-catalyst_${scala.binary.version} ${spark.version} tests test - - - log4j - log4j - - - org.slf4j - slf4j-log4j12 - - - org.apache.logging.log4j - log4j-slf4j2-impl - - - org.apache.orc - orc-core - - + org.apache.spark - spark-catalyst_2.12 + spark-core_${scala.binary.version} ${spark.version} tests test - - - log4j - log4j - - - org.slf4j - slf4j-log4j12 - - - org.apache.orc - orc-core - - - - - org.scalatest - scalatest_${scala.binary.version} - 3.1.0 - test + org.apache.spark - spark-hive_2.12 + spark-hive_${scala.binary.version} ${spark.version} test - - - log4j - log4j - - - org.slf4j - slf4j-log4j12 - - - org.apache.orc - orc-core - - - com.google.protobuf - protobuf-java - - - - - org.apache.paimon - paimon-hive-common - ${project.version} - test - - - org.apache.paimon - paimon-hive-common - ${project.version} - tests - test-jar - test - - - - net.alchim31.maven - scala-maven-plugin - ${scala-maven-plugin.version} - - - -nobootcp - -target:jvm-${target.java.version} - - false - - - - org.apache.maven.plugins - maven-compiler-plugin - - - org.apache.maven.plugins @@ -286,69 +125,20 @@ under the License. * com/github/luben/zstd/** + **/*libzstd-jni-*.so + **/*libzstd-jni-*.dll - org.apache.paimon:paimon-spark-common + org.apache.paimon:paimon-spark3-common - - net.alchim31.maven - scala-maven-plugin - ${scala-maven-plugin.version} - - - - scala-compile-first - process-resources - - add-source - compile - - - - - - scala-test-compile - process-test-resources - - testCompile - - - - - - org.scalatest - scalatest-maven-plugin - ${scalatest-maven-plugin.version} - - ${project.build.directory}/surefire-reports - . - -ea -Xmx4g -Xss4m -XX:MaxMetaspaceSize=2g -XX:ReservedCodeCacheSize=${CodeCacheSize} ${extraJavaTestArgs} -Dio.netty.tryReflectionSetAccessible=true - PaimonTestSuite.txt - - once - - - - test - - test - - - - diff --git a/paimon-spark/paimon-spark-3.5/src/test/scala/org/apache/paimon/spark/sql/PaimonViewTest.scala b/paimon-spark/paimon-spark-3.5/src/test/scala/org/apache/paimon/spark/sql/PaimonViewTest.scala new file mode 100644 index 000000000000..6ab8a2671b51 --- /dev/null +++ b/paimon-spark/paimon-spark-3.5/src/test/scala/org/apache/paimon/spark/sql/PaimonViewTest.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.sql + +class PaimonViewTest extends PaimonViewTestBase {} diff --git a/paimon-spark/paimon-spark-3.5/src/test/scala/org/apache/paimon/spark/sql/TagDdlTest.scala b/paimon-spark/paimon-spark-3.5/src/test/scala/org/apache/paimon/spark/sql/TagDdlTest.scala new file mode 100644 index 000000000000..92309d54167b --- /dev/null +++ b/paimon-spark/paimon-spark-3.5/src/test/scala/org/apache/paimon/spark/sql/TagDdlTest.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.sql + +class TagDdlTest extends PaimonTagDdlTestBase {} diff --git a/paimon-spark/paimon-spark-4.0/pom.xml b/paimon-spark/paimon-spark-4.0/pom.xml new file mode 100644 index 000000000000..8e7d166dc55b --- /dev/null +++ b/paimon-spark/paimon-spark-4.0/pom.xml @@ -0,0 +1,144 @@ + + + + 4.0.0 + + + org.apache.paimon + paimon-spark + 1.0-SNAPSHOT + + + paimon-spark-4.0 + Paimon : Spark : 4.0 + + + 4.0.0-preview2 + + + + + org.apache.paimon + paimon-spark4-common + ${project.version} + + + + org.apache.paimon + paimon-spark-common_${scala.binary.version} + ${project.version} + + + + org.apache.spark + spark-sql_${scala.binary.version} + ${spark.version} + + + + org.apache.spark + spark-core_${scala.binary.version} + ${spark.version} + + + + org.apache.paimon + paimon-bundle + + + + + + org.apache.paimon + paimon-spark-ut + ${project.version} + tests + test + + + + org.apache.spark + spark-sql_${scala.binary.version} + ${spark.version} + tests + test + + + + org.apache.spark + spark-catalyst_${scala.binary.version} + ${spark.version} + tests + test + + + + org.apache.spark + spark-core_${scala.binary.version} + ${spark.version} + tests + test + + + + org.apache.spark + spark-hive_${scala.binary.version} + ${spark.version} + test + + + + + + + org.apache.maven.plugins + maven-shade-plugin + + + shade-paimon + package + + shade + + + + + * + + com/github/luben/zstd/** + **/*libzstd-jni-*.so + **/*libzstd-jni-*.dll + + + + + + org.apache.paimon:paimon-spark4-common + + + + + + + + + diff --git a/paimon-spark/paimon-spark-4.0/src/main/scala/org/apache/paimon/spark/catalyst/optimizer/MergePaimonScalarSubqueries.scala b/paimon-spark/paimon-spark-4.0/src/main/scala/org/apache/paimon/spark/catalyst/optimizer/MergePaimonScalarSubqueries.scala new file mode 100644 index 000000000000..2144f77f3a6c --- /dev/null +++ b/paimon-spark/paimon-spark-4.0/src/main/scala/org/apache/paimon/spark/catalyst/optimizer/MergePaimonScalarSubqueries.scala @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.catalyst.optimizer + +import org.apache.paimon.spark.PaimonScan + +import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeMap, AttributeReference, ExprId, ScalarSubquery, SortOrder} +import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan +import org.apache.spark.sql.execution.datasources.v2.DataSourceV2ScanRelation + +object MergePaimonScalarSubqueries extends MergePaimonScalarSubqueriesBase { + + override def tryMergeDataSourceV2ScanRelation( + newV2ScanRelation: DataSourceV2ScanRelation, + cachedV2ScanRelation: DataSourceV2ScanRelation) + : Option[(LogicalPlan, AttributeMap[Attribute])] = { + (newV2ScanRelation, cachedV2ScanRelation) match { + case ( + DataSourceV2ScanRelation( + newRelation, + newScan: PaimonScan, + newOutput, + newPartitioning, + newOrdering), + DataSourceV2ScanRelation( + cachedRelation, + cachedScan: PaimonScan, + _, + cachedPartitioning, + cacheOrdering)) => + checkIdenticalPlans(newRelation, cachedRelation).flatMap { + outputMap => + if ( + samePartitioning(newPartitioning, cachedPartitioning, outputMap) && sameOrdering( + newOrdering, + cacheOrdering, + outputMap) + ) { + mergePaimonScan(newScan, cachedScan).map { + mergedScan => + val mergedAttributes = mergedScan + .readSchema() + .map(f => AttributeReference(f.name, f.dataType, f.nullable, f.metadata)()) + val cachedOutputNameMap = cachedRelation.output.map(a => a.name -> a).toMap + val mergedOutput = + mergedAttributes.map(a => cachedOutputNameMap.getOrElse(a.name, a)) + val newV2ScanRelation = DataSourceV2ScanRelation( + cachedRelation, + mergedScan, + mergedOutput, + cachedPartitioning) + + val mergedOutputNameMap = mergedOutput.map(a => a.name -> a).toMap + val newOutputMap = + AttributeMap(newOutput.map(a => a -> mergedOutputNameMap(a.name).toAttribute)) + + newV2ScanRelation -> newOutputMap + } + } else { + None + } + } + + case _ => None + } + } + + private def sameOrdering( + newOrdering: Option[Seq[SortOrder]], + cachedOrdering: Option[Seq[SortOrder]], + outputAttrMap: AttributeMap[Attribute]): Boolean = { + val mappedNewOrdering = newOrdering.map(_.map(mapAttributes(_, outputAttrMap))) + mappedNewOrdering.map(_.map(_.canonicalized)) == cachedOrdering.map(_.map(_.canonicalized)) + + } + + override protected def createScalarSubquery(plan: LogicalPlan, exprId: ExprId): ScalarSubquery = { + ScalarSubquery(plan, exprId = exprId) + } +} diff --git a/paimon-spark/paimon-spark-4.0/src/test/resources/hive-site.xml b/paimon-spark/paimon-spark-4.0/src/test/resources/hive-site.xml new file mode 100644 index 000000000000..bdf2bb090760 --- /dev/null +++ b/paimon-spark/paimon-spark-4.0/src/test/resources/hive-site.xml @@ -0,0 +1,56 @@ + + + + + hive.metastore.integral.jdo.pushdown + true + + + + hive.metastore.schema.verification + false + + + + hive.metastore.client.capability.check + false + + + + datanucleus.schema.autoCreateTables + true + + + + datanucleus.schema.autoCreateAll + true + + + + + datanucleus.connectionPoolingType + DBCP + + + + hive.metastore.uris + thrift://localhost:9090 + Thrift URI for the remote metastore. Used by metastore client to connect to remote metastore. + + \ No newline at end of file diff --git a/paimon-spark/paimon-spark-common/src/test/resources/log4j2-test.properties b/paimon-spark/paimon-spark-4.0/src/test/resources/log4j2-test.properties similarity index 100% rename from paimon-spark/paimon-spark-common/src/test/resources/log4j2-test.properties rename to paimon-spark/paimon-spark-4.0/src/test/resources/log4j2-test.properties diff --git a/paimon-spark/paimon-spark-4.0/src/test/scala/org/apache/paimon/spark/procedure/CompactProcedureTest.scala b/paimon-spark/paimon-spark-4.0/src/test/scala/org/apache/paimon/spark/procedure/CompactProcedureTest.scala new file mode 100644 index 000000000000..322d50a62127 --- /dev/null +++ b/paimon-spark/paimon-spark-4.0/src/test/scala/org/apache/paimon/spark/procedure/CompactProcedureTest.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.procedure + +class CompactProcedureTest extends CompactProcedureTestBase {} diff --git a/paimon-spark/paimon-spark-4.0/src/test/scala/org/apache/paimon/spark/procedure/ProcedureTest.scala b/paimon-spark/paimon-spark-4.0/src/test/scala/org/apache/paimon/spark/procedure/ProcedureTest.scala new file mode 100644 index 000000000000..d57846709877 --- /dev/null +++ b/paimon-spark/paimon-spark-4.0/src/test/scala/org/apache/paimon/spark/procedure/ProcedureTest.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.procedure + +class ProcedureTest extends ProcedureTestBase {} diff --git a/paimon-spark/paimon-spark-4.0/src/test/scala/org/apache/paimon/spark/sql/AnalyzeTableTest.scala b/paimon-spark/paimon-spark-4.0/src/test/scala/org/apache/paimon/spark/sql/AnalyzeTableTest.scala new file mode 100644 index 000000000000..255906d04bf2 --- /dev/null +++ b/paimon-spark/paimon-spark-4.0/src/test/scala/org/apache/paimon/spark/sql/AnalyzeTableTest.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.sql + +class AnalyzeTableTest extends AnalyzeTableTestBase {} diff --git a/paimon-spark/paimon-spark-4.0/src/test/scala/org/apache/paimon/spark/sql/DDLTest.scala b/paimon-spark/paimon-spark-4.0/src/test/scala/org/apache/paimon/spark/sql/DDLTest.scala new file mode 100644 index 000000000000..b729f57b33e7 --- /dev/null +++ b/paimon-spark/paimon-spark-4.0/src/test/scala/org/apache/paimon/spark/sql/DDLTest.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.sql + +class DDLTest extends DDLTestBase {} diff --git a/paimon-spark/paimon-spark-4.0/src/test/scala/org/apache/paimon/spark/sql/DDLWithHiveCatalogTest.scala b/paimon-spark/paimon-spark-4.0/src/test/scala/org/apache/paimon/spark/sql/DDLWithHiveCatalogTest.scala new file mode 100644 index 000000000000..a9ea3efc89ba --- /dev/null +++ b/paimon-spark/paimon-spark-4.0/src/test/scala/org/apache/paimon/spark/sql/DDLWithHiveCatalogTest.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.sql + +class DDLWithHiveCatalogTest extends DDLWithHiveCatalogTestBase {} diff --git a/paimon-spark/paimon-spark-4.0/src/test/scala/org/apache/paimon/spark/sql/DeleteFromTableTest.scala b/paimon-spark/paimon-spark-4.0/src/test/scala/org/apache/paimon/spark/sql/DeleteFromTableTest.scala new file mode 100644 index 000000000000..09554a1dbf8d --- /dev/null +++ b/paimon-spark/paimon-spark-4.0/src/test/scala/org/apache/paimon/spark/sql/DeleteFromTableTest.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.sql + +class DeleteFromTableTest extends DeleteFromTableTestBase {} diff --git a/paimon-spark/paimon-spark-4.0/src/test/scala/org/apache/paimon/spark/sql/InsertOverwriteTableTest.scala b/paimon-spark/paimon-spark-4.0/src/test/scala/org/apache/paimon/spark/sql/InsertOverwriteTableTest.scala new file mode 100644 index 000000000000..4f66584c303b --- /dev/null +++ b/paimon-spark/paimon-spark-4.0/src/test/scala/org/apache/paimon/spark/sql/InsertOverwriteTableTest.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.sql + +class InsertOverwriteTableTest extends InsertOverwriteTableTestBase {} diff --git a/paimon-spark/paimon-spark-4.0/src/test/scala/org/apache/paimon/spark/sql/MergeIntoTableTest.scala b/paimon-spark/paimon-spark-4.0/src/test/scala/org/apache/paimon/spark/sql/MergeIntoTableTest.scala new file mode 100644 index 000000000000..e1cfe3a3960f --- /dev/null +++ b/paimon-spark/paimon-spark-4.0/src/test/scala/org/apache/paimon/spark/sql/MergeIntoTableTest.scala @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.sql + +import org.apache.paimon.spark.{PaimonAppendBucketedTableTest, PaimonAppendNonBucketTableTest, PaimonPrimaryKeyBucketedTableTest, PaimonPrimaryKeyNonBucketTableTest} + +class MergeIntoPrimaryKeyBucketedTableTest + extends MergeIntoTableTestBase + with MergeIntoPrimaryKeyTableTest + with MergeIntoNotMatchedBySourceTest + with PaimonPrimaryKeyBucketedTableTest {} + +class MergeIntoPrimaryKeyNonBucketTableTest + extends MergeIntoTableTestBase + with MergeIntoPrimaryKeyTableTest + with MergeIntoNotMatchedBySourceTest + with PaimonPrimaryKeyNonBucketTableTest {} + +class MergeIntoAppendBucketedTableTest + extends MergeIntoTableTestBase + with MergeIntoNotMatchedBySourceTest + with PaimonAppendBucketedTableTest {} + +class MergeIntoAppendNonBucketedTableTest + extends MergeIntoTableTestBase + with MergeIntoNotMatchedBySourceTest + with PaimonAppendNonBucketTableTest {} diff --git a/paimon-spark/paimon-spark-4.0/src/test/scala/org/apache/paimon/spark/sql/PaimonCompositePartitionKeyTest.scala b/paimon-spark/paimon-spark-4.0/src/test/scala/org/apache/paimon/spark/sql/PaimonCompositePartitionKeyTest.scala new file mode 100644 index 000000000000..635185a9ed0e --- /dev/null +++ b/paimon-spark/paimon-spark-4.0/src/test/scala/org/apache/paimon/spark/sql/PaimonCompositePartitionKeyTest.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.sql + +class PaimonCompositePartitionKeyTest extends PaimonCompositePartitionKeyTestBase {} diff --git a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/ProcessRecordAttributesUtil.java b/paimon-spark/paimon-spark-4.0/src/test/scala/org/apache/paimon/spark/sql/PaimonOptimizationTest.scala similarity index 56% rename from paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/ProcessRecordAttributesUtil.java rename to paimon-spark/paimon-spark-4.0/src/test/scala/org/apache/paimon/spark/sql/PaimonOptimizationTest.scala index efe5e12b12d7..0a4dfb76959c 100644 --- a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/ProcessRecordAttributesUtil.java +++ b/paimon-spark/paimon-spark-4.0/src/test/scala/org/apache/paimon/spark/sql/PaimonOptimizationTest.scala @@ -16,16 +16,22 @@ * limitations under the License. */ -package org.apache.paimon.flink; +package org.apache.paimon.spark.sql -import org.apache.paimon.flink.sink.StoreSinkWrite; +import org.apache.paimon.spark.util.CTERelationRefUtils -import org.apache.flink.streaming.api.operators.Output; -import org.apache.flink.streaming.runtime.streamrecord.RecordAttributes; +import org.apache.spark.sql.catalyst.dsl.expressions._ +import org.apache.spark.sql.catalyst.expressions.{Attribute, GetStructField, NamedExpression, ScalarSubquery} -/** Placeholder class for new feature introduced since flink 1.19. Should never be used. */ -public class ProcessRecordAttributesUtil { - public static void processWithWrite(RecordAttributes recordAttributes, StoreSinkWrite write) {} +class PaimonOptimizationTest extends PaimonOptimizationTestBase { - public static void processWithOutput(RecordAttributes recordAttributes, Output output) {} + override def extractorExpression( + cteIndex: Int, + output: Seq[Attribute], + fieldIndex: Int): NamedExpression = { + GetStructField( + ScalarSubquery(CTERelationRefUtils.createCTERelationRef(cteIndex, _resolved = true, output)), + fieldIndex) + .as("scalarsubquery()") + } } diff --git a/paimon-spark/paimon-spark-4.0/src/test/scala/org/apache/paimon/spark/sql/PaimonViewTest.scala b/paimon-spark/paimon-spark-4.0/src/test/scala/org/apache/paimon/spark/sql/PaimonViewTest.scala new file mode 100644 index 000000000000..6ab8a2671b51 --- /dev/null +++ b/paimon-spark/paimon-spark-4.0/src/test/scala/org/apache/paimon/spark/sql/PaimonViewTest.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.sql + +class PaimonViewTest extends PaimonViewTestBase {} diff --git a/paimon-spark/paimon-spark-4.0/src/test/scala/org/apache/paimon/spark/sql/ShowColumnsTest.scala b/paimon-spark/paimon-spark-4.0/src/test/scala/org/apache/paimon/spark/sql/ShowColumnsTest.scala new file mode 100644 index 000000000000..6601dc2fca37 --- /dev/null +++ b/paimon-spark/paimon-spark-4.0/src/test/scala/org/apache/paimon/spark/sql/ShowColumnsTest.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.sql + +class ShowColumnsTest extends PaimonShowColumnsTestBase {} diff --git a/paimon-spark/paimon-spark-4.0/src/test/scala/org/apache/paimon/spark/sql/TagDdlTest.scala b/paimon-spark/paimon-spark-4.0/src/test/scala/org/apache/paimon/spark/sql/TagDdlTest.scala new file mode 100644 index 000000000000..92309d54167b --- /dev/null +++ b/paimon-spark/paimon-spark-4.0/src/test/scala/org/apache/paimon/spark/sql/TagDdlTest.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.sql + +class TagDdlTest extends PaimonTagDdlTestBase {} diff --git a/paimon-spark/paimon-spark-4.0/src/test/scala/org/apache/paimon/spark/sql/UpdateTableTest.scala b/paimon-spark/paimon-spark-4.0/src/test/scala/org/apache/paimon/spark/sql/UpdateTableTest.scala new file mode 100644 index 000000000000..194aab278c0e --- /dev/null +++ b/paimon-spark/paimon-spark-4.0/src/test/scala/org/apache/paimon/spark/sql/UpdateTableTest.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.sql + +class UpdateTableTest extends UpdateTableTestBase {} diff --git a/paimon-spark/paimon-spark-common/pom.xml b/paimon-spark/paimon-spark-common/pom.xml index e9ca8b0e1bee..052c4c4265fc 100644 --- a/paimon-spark/paimon-spark-common/pom.xml +++ b/paimon-spark/paimon-spark-common/pom.xml @@ -23,284 +23,47 @@ under the License. 4.0.0 - paimon-spark org.apache.paimon + paimon-spark 1.0-SNAPSHOT jar - paimon-spark-common - Paimon : Spark : Common + paimon-spark-common_${scala.binary.version} + Paimon : Spark : Common : ${scala.binary.version} - 3.5.3 + ${paimon-spark-common.spark.version} - - org.scala-lang - scala-library - ${scala.version} - - - org.scala-lang - scala-reflect - ${scala.version} - - - org.scala-lang - scala-compiler - ${scala.version} - - - - org.apache.spark - spark-avro_2.12 - ${spark.version} - test - - - - org.apache.spark - spark-sql_2.12 - ${spark.version} - - - log4j - log4j - - - org.slf4j - slf4j-log4j12 - - - - org.apache.logging.log4j - log4j-slf4j2-impl - - - org.apache.orc - orc-core - - - org.apache.orc - orc-mapreduce - - - org.apache.parquet - parquet-column - - - - org.apache.spark - spark-core_2.12 + spark-sql_${scala.binary.version} ${spark.version} - - - log4j - log4j - - - org.slf4j - slf4j-log4j12 - - - - org.apache.logging.log4j - log4j-slf4j2-impl - - - org.apache.orc - orc-core - - - org.apache.orc - orc-mapreduce - - - org.apache.parquet - parquet-column - - - com.google.protobuf - protobuf-java - - org.apache.spark - spark-hive_2.12 + spark-core_${scala.binary.version} ${spark.version} - test - - - log4j - log4j - - - org.slf4j - slf4j-log4j12 - - - org.apache.orc - orc-core - - - com.google.protobuf - protobuf-java - - - - org.apache.spark - spark-sql_2.12 - ${spark.version} - tests - test - - - log4j - log4j - - - org.slf4j - slf4j-log4j12 - - - org.apache.orc - orc-core - - - org.apache.parquet - parquet-column - - - - - org.apache.spark - spark-core_2.12 - ${spark.version} - tests - test - - - log4j - log4j - - - org.slf4j - slf4j-log4j12 - - - org.apache.logging.log4j - log4j-slf4j2-impl - - - org.apache.orc - orc-core - - - com.google.protobuf - protobuf-java - - - - - org.apache.spark - spark-catalyst_2.12 - ${spark.version} - tests - test - - - log4j - log4j - - - org.slf4j - slf4j-log4j12 - - - org.apache.orc - orc-core - - - org.antlr antlr4-runtime ${antlr4.version} - - org.scalatest - scalatest_${scala.binary.version} - 3.1.0 - test - - - org.apache.paimon - paimon-common - ${project.version} - test-jar - test - - - org.apache.paimon - paimon-hive-common - ${project.version} - test - + org.apache.paimon - paimon-hive-common - ${project.version} - tests - test-jar - test + paimon-bundle - - - - net.alchim31.maven - scala-maven-plugin - ${scala-maven-plugin.version} - - - -nobootcp - -target:jvm-${target.java.version} - - false - - - - org.apache.maven.plugins - maven-compiler-plugin - - - - - - org.apache.maven.plugins - maven-jar-plugin - - - prepare-test-jar - test-compile - - test-jar - - - - org.apache.maven.plugins maven-shade-plugin @@ -321,33 +84,7 @@ under the License. - - net.alchim31.maven - scala-maven-plugin - ${scala-maven-plugin.version} - - - - scala-compile-first - process-resources - - add-source - compile - - - - - scala-test-compile - process-test-resources - - testCompile - - - - org.antlr antlr4-maven-plugin @@ -365,30 +102,6 @@ under the License. src/main/antlr4 - - org.scalatest - scalatest-maven-plugin - ${scalatest-maven-plugin.version} - - ${project.build.directory}/surefire-reports - . - -ea -Xmx4g -Xss4m -XX:MaxMetaspaceSize=2g -XX:ReservedCodeCacheSize=${CodeCacheSize} ${extraJavaTestArgs} -Dio.netty.tryReflectionSetAccessible=true - PaimonTestSuite.txt - - once - - - - test - - test - - - - diff --git a/paimon-spark/paimon-spark-common/src/main/antlr4/org.apache.spark.sql.catalyst.parser.extensions/PaimonSqlExtensions.g4 b/paimon-spark/paimon-spark-common/src/main/antlr4/org.apache.spark.sql.catalyst.parser.extensions/PaimonSqlExtensions.g4 index 54e71b362fc3..207d9732160f 100644 --- a/paimon-spark/paimon-spark-common/src/main/antlr4/org.apache.spark.sql.catalyst.parser.extensions/PaimonSqlExtensions.g4 +++ b/paimon-spark/paimon-spark-common/src/main/antlr4/org.apache.spark.sql.catalyst.parser.extensions/PaimonSqlExtensions.g4 @@ -70,13 +70,40 @@ singleStatement statement : CALL multipartIdentifier '(' (callArgument (',' callArgument)*)? ')' #call - ; + | SHOW TAGS multipartIdentifier #showTags + | ALTER TABLE multipartIdentifier createReplaceTagClause #createOrReplaceTag + | ALTER TABLE multipartIdentifier DELETE TAG (IF EXISTS)? identifier #deleteTag + | ALTER TABLE multipartIdentifier RENAME TAG identifier TO identifier #renameTag + ; callArgument : expression #positionalArgument | identifier '=>' expression #namedArgument ; +createReplaceTagClause + : CREATE TAG (IF NOT EXISTS)? identifier tagOptions + | (CREATE OR)? REPLACE TAG identifier tagOptions + ; + +tagOptions + : (AS OF VERSION snapshotId)? (timeRetain)? + ; + +snapshotId + : number + ; + +timeRetain + : RETAIN number timeUnit + ; + +timeUnit + : DAYS + | HOURS + | MINUTES + ; + expression : constant | stringMap @@ -124,12 +151,34 @@ quotedIdentifier ; nonReserved - : CALL + : ALTER | AS | CALL | CREATE | DAYS | DELETE | EXISTS | HOURS | IF | NOT | OF | OR | TABLE + | REPLACE | RETAIN | VERSION | TAG | TRUE | FALSE | MAP ; +ALTER: 'ALTER'; +AS: 'AS'; CALL: 'CALL'; +CREATE: 'CREATE'; +DAYS: 'DAYS'; +DELETE: 'DELETE'; +EXISTS: 'EXISTS'; +HOURS: 'HOURS'; +IF : 'IF'; +MINUTES: 'MINUTES'; +NOT: 'NOT'; +OF: 'OF'; +OR: 'OR'; +RENAME: 'RENAME'; +REPLACE: 'REPLACE'; +RETAIN: 'RETAIN'; +SHOW: 'SHOW'; +TABLE: 'TABLE'; +TAG: 'TAG'; +TAGS: 'TAGS'; +TO: 'TO'; +VERSION: 'VERSION'; TRUE: 'TRUE'; FALSE: 'FALSE'; diff --git a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkInternalRow.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/AbstractSparkInternalRow.java similarity index 67% rename from paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkInternalRow.java rename to paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/AbstractSparkInternalRow.java index a73e97817504..28604a6d6293 100644 --- a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkInternalRow.java +++ b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/AbstractSparkInternalRow.java @@ -18,24 +18,15 @@ package org.apache.paimon.spark; -import org.apache.paimon.data.BinaryString; -import org.apache.paimon.data.InternalArray; -import org.apache.paimon.data.InternalMap; import org.apache.paimon.data.InternalRow; -import org.apache.paimon.data.Timestamp; -import org.apache.paimon.spark.util.shim.TypeUtils; +import org.apache.paimon.spark.data.SparkInternalRow; import org.apache.paimon.types.ArrayType; import org.apache.paimon.types.BigIntType; import org.apache.paimon.types.DataType; import org.apache.paimon.types.DataTypeChecks; -import org.apache.paimon.types.IntType; -import org.apache.paimon.types.MapType; -import org.apache.paimon.types.MultisetType; import org.apache.paimon.types.RowType; -import org.apache.spark.sql.catalyst.util.ArrayBasedMapData; import org.apache.spark.sql.catalyst.util.ArrayData; -import org.apache.spark.sql.catalyst.util.DateTimeUtils; import org.apache.spark.sql.catalyst.util.MapData; import org.apache.spark.sql.types.BinaryType; import org.apache.spark.sql.types.BooleanType; @@ -61,19 +52,23 @@ import java.util.Objects; +import static org.apache.paimon.spark.DataConverter.fromPaimon; import static org.apache.paimon.utils.InternalRowUtils.copyInternalRow; -/** Spark {@link org.apache.spark.sql.catalyst.InternalRow} to wrap {@link InternalRow}. */ -public class SparkInternalRow extends org.apache.spark.sql.catalyst.InternalRow { +/** + * An abstract {@link SparkInternalRow} that overwrite all the common methods in spark3 and spark4. + */ +public abstract class AbstractSparkInternalRow extends SparkInternalRow { - private final RowType rowType; + protected RowType rowType; - private InternalRow row; + protected InternalRow row; - public SparkInternalRow(RowType rowType) { + public AbstractSparkInternalRow(RowType rowType) { this.rowType = rowType; } + @Override public SparkInternalRow replace(InternalRow row) { this.row = row; return this; @@ -96,7 +91,7 @@ public void update(int i, Object value) { @Override public org.apache.spark.sql.catalyst.InternalRow copy() { - return new SparkInternalRow(rowType).replace(copyInternalRow(row, rowType)); + return SparkInternalRow.create(rowType).replace(copyInternalRow(row, rowType)); } @Override @@ -255,7 +250,7 @@ public boolean equals(Object o) { if (o == null || getClass() != o.getClass()) { return false; } - SparkInternalRow that = (SparkInternalRow) o; + AbstractSparkInternalRow that = (AbstractSparkInternalRow) o; return Objects.equals(rowType, that.rowType) && Objects.equals(row, that.row); } @@ -263,78 +258,4 @@ public boolean equals(Object o) { public int hashCode() { return Objects.hash(rowType, row); } - - // ================== static methods ========================================= - - public static Object fromPaimon(Object o, DataType type) { - if (o == null) { - return null; - } - switch (type.getTypeRoot()) { - case TIMESTAMP_WITHOUT_TIME_ZONE: - case TIMESTAMP_WITH_LOCAL_TIME_ZONE: - return fromPaimon((Timestamp) o); - case CHAR: - case VARCHAR: - return fromPaimon((BinaryString) o); - case DECIMAL: - return fromPaimon((org.apache.paimon.data.Decimal) o); - case ARRAY: - return fromPaimon((InternalArray) o, (ArrayType) type); - case MAP: - case MULTISET: - return fromPaimon((InternalMap) o, type); - case ROW: - return fromPaimon((InternalRow) o, (RowType) type); - default: - return o; - } - } - - public static UTF8String fromPaimon(BinaryString string) { - return UTF8String.fromBytes(string.toBytes()); - } - - public static Decimal fromPaimon(org.apache.paimon.data.Decimal decimal) { - return Decimal.apply(decimal.toBigDecimal()); - } - - public static org.apache.spark.sql.catalyst.InternalRow fromPaimon( - InternalRow row, RowType rowType) { - return new SparkInternalRow(rowType).replace(row); - } - - public static long fromPaimon(Timestamp timestamp) { - if (TypeUtils.treatPaimonTimestampTypeAsSparkTimestampType()) { - return DateTimeUtils.fromJavaTimestamp(timestamp.toSQLTimestamp()); - } else { - return timestamp.toMicros(); - } - } - - public static ArrayData fromPaimon(InternalArray array, ArrayType arrayType) { - return fromPaimonArrayElementType(array, arrayType.getElementType()); - } - - private static ArrayData fromPaimonArrayElementType(InternalArray array, DataType elementType) { - return new SparkArrayData(elementType).replace(array); - } - - public static MapData fromPaimon(InternalMap map, DataType mapType) { - DataType keyType; - DataType valueType; - if (mapType instanceof MapType) { - keyType = ((MapType) mapType).getKeyType(); - valueType = ((MapType) mapType).getValueType(); - } else if (mapType instanceof MultisetType) { - keyType = ((MultisetType) mapType).getElementType(); - valueType = new IntType(); - } else { - throw new UnsupportedOperationException("Unsupported type: " + mapType); - } - - return new ArrayBasedMapData( - fromPaimonArrayElementType(map.keyArray(), keyType), - fromPaimonArrayElementType(map.valueArray(), valueType)); - } } diff --git a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/DataConverter.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/DataConverter.java new file mode 100644 index 000000000000..0b5ea899476e --- /dev/null +++ b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/DataConverter.java @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark; + +import org.apache.paimon.data.BinaryString; +import org.apache.paimon.data.InternalArray; +import org.apache.paimon.data.InternalMap; +import org.apache.paimon.data.InternalRow; +import org.apache.paimon.data.Timestamp; +import org.apache.paimon.spark.data.SparkArrayData; +import org.apache.paimon.spark.data.SparkInternalRow; +import org.apache.paimon.spark.util.shim.TypeUtils; +import org.apache.paimon.types.ArrayType; +import org.apache.paimon.types.DataType; +import org.apache.paimon.types.IntType; +import org.apache.paimon.types.MapType; +import org.apache.paimon.types.MultisetType; +import org.apache.paimon.types.RowType; + +import org.apache.spark.sql.catalyst.util.ArrayBasedMapData; +import org.apache.spark.sql.catalyst.util.ArrayData; +import org.apache.spark.sql.catalyst.util.DateTimeUtils; +import org.apache.spark.sql.catalyst.util.MapData; +import org.apache.spark.sql.types.Decimal; +import org.apache.spark.unsafe.types.UTF8String; + +/** A data converter that convert Paimon data to Spark Data. */ +public class DataConverter { + + public static Object fromPaimon(Object o, DataType type) { + if (o == null) { + return null; + } + switch (type.getTypeRoot()) { + case TIMESTAMP_WITHOUT_TIME_ZONE: + case TIMESTAMP_WITH_LOCAL_TIME_ZONE: + return fromPaimon((Timestamp) o); + case CHAR: + case VARCHAR: + return fromPaimon((BinaryString) o); + case DECIMAL: + return fromPaimon((org.apache.paimon.data.Decimal) o); + case ARRAY: + return fromPaimon((InternalArray) o, (ArrayType) type); + case MAP: + case MULTISET: + return fromPaimon((InternalMap) o, type); + case ROW: + return fromPaimon((InternalRow) o, (RowType) type); + default: + return o; + } + } + + public static UTF8String fromPaimon(BinaryString string) { + return UTF8String.fromBytes(string.toBytes()); + } + + public static Decimal fromPaimon(org.apache.paimon.data.Decimal decimal) { + return Decimal.apply(decimal.toBigDecimal()); + } + + public static org.apache.spark.sql.catalyst.InternalRow fromPaimon( + InternalRow row, RowType rowType) { + return SparkInternalRow.create(rowType).replace(row); + } + + public static long fromPaimon(Timestamp timestamp) { + if (TypeUtils.treatPaimonTimestampTypeAsSparkTimestampType()) { + return DateTimeUtils.fromJavaTimestamp(timestamp.toSQLTimestamp()); + } else { + return timestamp.toMicros(); + } + } + + public static ArrayData fromPaimon(InternalArray array, ArrayType arrayType) { + return fromPaimonArrayElementType(array, arrayType.getElementType()); + } + + private static ArrayData fromPaimonArrayElementType(InternalArray array, DataType elementType) { + return SparkArrayData.create(elementType).replace(array); + } + + public static MapData fromPaimon(InternalMap map, DataType mapType) { + DataType keyType; + DataType valueType; + if (mapType instanceof MapType) { + keyType = ((MapType) mapType).getKeyType(); + valueType = ((MapType) mapType).getValueType(); + } else if (mapType instanceof MultisetType) { + keyType = ((MultisetType) mapType).getElementType(); + valueType = new IntType(); + } else { + throw new UnsupportedOperationException("Unsupported type: " + mapType); + } + + return new ArrayBasedMapData( + fromPaimonArrayElementType(map.keyArray(), keyType), + fromPaimonArrayElementType(map.valueArray(), valueType)); + } +} diff --git a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkArrayData.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkArrayData.java deleted file mode 100644 index 0e7428eabde7..000000000000 --- a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkArrayData.java +++ /dev/null @@ -1,172 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.paimon.spark; - -import org.apache.paimon.data.InternalArray; -import org.apache.paimon.types.ArrayType; -import org.apache.paimon.types.BigIntType; -import org.apache.paimon.types.DataType; -import org.apache.paimon.types.DataTypeChecks; -import org.apache.paimon.types.RowType; -import org.apache.paimon.utils.InternalRowUtils; - -import org.apache.spark.sql.catalyst.InternalRow; -import org.apache.spark.sql.catalyst.expressions.SpecializedGettersReader; -import org.apache.spark.sql.catalyst.util.ArrayData; -import org.apache.spark.sql.catalyst.util.MapData; -import org.apache.spark.sql.types.Decimal; -import org.apache.spark.unsafe.types.CalendarInterval; -import org.apache.spark.unsafe.types.UTF8String; - -import static org.apache.paimon.spark.SparkInternalRow.fromPaimon; -import static org.apache.paimon.utils.InternalRowUtils.copyArray; - -/** Spark {@link ArrayData} to wrap Paimon {@link InternalArray}. */ -public class SparkArrayData extends ArrayData { - - private final DataType elementType; - - private InternalArray array; - - public SparkArrayData(DataType elementType) { - this.elementType = elementType; - } - - public SparkArrayData replace(InternalArray array) { - this.array = array; - return this; - } - - @Override - public int numElements() { - return array.size(); - } - - @Override - public ArrayData copy() { - return new SparkArrayData(elementType).replace(copyArray(array, elementType)); - } - - @Override - public Object[] array() { - Object[] objects = new Object[numElements()]; - for (int i = 0; i < objects.length; i++) { - objects[i] = fromPaimon(InternalRowUtils.get(array, i, elementType), elementType); - } - return objects; - } - - @Override - public void setNullAt(int i) { - throw new UnsupportedOperationException(); - } - - @Override - public void update(int i, Object value) { - throw new UnsupportedOperationException(); - } - - @Override - public boolean isNullAt(int ordinal) { - return array.isNullAt(ordinal); - } - - @Override - public boolean getBoolean(int ordinal) { - return array.getBoolean(ordinal); - } - - @Override - public byte getByte(int ordinal) { - return array.getByte(ordinal); - } - - @Override - public short getShort(int ordinal) { - return array.getShort(ordinal); - } - - @Override - public int getInt(int ordinal) { - return array.getInt(ordinal); - } - - @Override - public long getLong(int ordinal) { - if (elementType instanceof BigIntType) { - return array.getLong(ordinal); - } - - return getTimestampMicros(ordinal); - } - - private long getTimestampMicros(int ordinal) { - return fromPaimon(array.getTimestamp(ordinal, DataTypeChecks.getPrecision(elementType))); - } - - @Override - public float getFloat(int ordinal) { - return array.getFloat(ordinal); - } - - @Override - public double getDouble(int ordinal) { - return array.getDouble(ordinal); - } - - @Override - public Decimal getDecimal(int ordinal, int precision, int scale) { - return fromPaimon(array.getDecimal(ordinal, precision, scale)); - } - - @Override - public UTF8String getUTF8String(int ordinal) { - return fromPaimon(array.getString(ordinal)); - } - - @Override - public byte[] getBinary(int ordinal) { - return array.getBinary(ordinal); - } - - @Override - public CalendarInterval getInterval(int ordinal) { - throw new UnsupportedOperationException(); - } - - @Override - public InternalRow getStruct(int ordinal, int numFields) { - return fromPaimon(array.getRow(ordinal, numFields), (RowType) elementType); - } - - @Override - public ArrayData getArray(int ordinal) { - return fromPaimon(array.getArray(ordinal), (ArrayType) elementType); - } - - @Override - public MapData getMap(int ordinal) { - return fromPaimon(array.getMap(ordinal), elementType); - } - - @Override - public Object get(int ordinal, org.apache.spark.sql.types.DataType dataType) { - return SpecializedGettersReader.read(this, ordinal, dataType, true, true); - } -} diff --git a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkCatalog.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkCatalog.java index 0c827714a4e2..d6318c723fe0 100644 --- a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkCatalog.java +++ b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkCatalog.java @@ -27,7 +27,9 @@ import org.apache.paimon.schema.SchemaChange; import org.apache.paimon.spark.catalog.SparkBaseCatalog; import org.apache.paimon.spark.catalog.SupportFunction; +import org.apache.paimon.spark.catalog.SupportView; import org.apache.paimon.table.FormatTable; +import org.apache.paimon.table.FormatTableOptions; import org.apache.spark.sql.SparkSession; import org.apache.spark.sql.catalyst.analysis.NamespaceAlreadyExistsException; @@ -45,6 +47,7 @@ import org.apache.spark.sql.execution.datasources.csv.CSVFileFormat; import org.apache.spark.sql.execution.datasources.orc.OrcFileFormat; import org.apache.spark.sql.execution.datasources.parquet.ParquetFileFormat; +import org.apache.spark.sql.execution.datasources.v2.FileTable; import org.apache.spark.sql.execution.datasources.v2.csv.CSVTable; import org.apache.spark.sql.execution.datasources.v2.orc.OrcTable; import org.apache.spark.sql.execution.datasources.v2.parquet.ParquetTable; @@ -70,10 +73,11 @@ import static org.apache.paimon.spark.SparkCatalogOptions.DEFAULT_DATABASE; import static org.apache.paimon.spark.SparkTypeUtils.toPaimonType; import static org.apache.paimon.spark.util.OptionUtils.copyWithSQLConf; -import static org.apache.paimon.utils.Preconditions.checkArgument; +import static org.apache.paimon.spark.utils.CatalogUtils.checkNamespace; +import static org.apache.paimon.spark.utils.CatalogUtils.toIdentifier; /** Spark {@link TableCatalog} for paimon. */ -public class SparkCatalog extends SparkBaseCatalog implements SupportFunction { +public class SparkCatalog extends SparkBaseCatalog implements SupportFunction, SupportView { private static final Logger LOG = LoggerFactory.getLogger(SparkCatalog.class); @@ -101,7 +105,9 @@ public void initialize(String name, CaseInsensitiveStringMap options) { this.catalog = CatalogFactory.createCatalog(catalogContext); this.defaultDatabase = options.getOrDefault(DEFAULT_DATABASE.key(), DEFAULT_DATABASE.defaultValue()); - if (!catalog.databaseExists(defaultNamespace()[0])) { + try { + catalog.getDatabase(defaultNamespace()[0]); + } catch (Catalog.DatabaseNotExistException e) { try { createNamespace(defaultNamespace(), new HashMap<>()); } catch (NamespaceAlreadyExistsException ignored) { @@ -122,10 +128,7 @@ public String[] defaultNamespace() { @Override public void createNamespace(String[] namespace, Map metadata) throws NamespaceAlreadyExistsException { - checkArgument( - isValidateNamespace(namespace), - "Namespace %s is not valid", - Arrays.toString(namespace)); + checkNamespace(namespace); try { catalog.createDatabase(namespace[0], false, metadata); } catch (Catalog.DatabaseAlreadyExistException e) { @@ -148,25 +151,22 @@ public String[][] listNamespaces(String[] namespace) throws NoSuchNamespaceExcep if (namespace.length == 0) { return listNamespaces(); } - if (!isValidateNamespace(namespace)) { - throw new NoSuchNamespaceException(namespace); - } - if (catalog.databaseExists(namespace[0])) { + checkNamespace(namespace); + try { + catalog.getDatabase(namespace[0]); return new String[0][]; + } catch (Catalog.DatabaseNotExistException e) { + throw new NoSuchNamespaceException(namespace); } - throw new NoSuchNamespaceException(namespace); } @Override public Map loadNamespaceMetadata(String[] namespace) throws NoSuchNamespaceException { - checkArgument( - isValidateNamespace(namespace), - "Namespace %s is not valid", - Arrays.toString(namespace)); + checkNamespace(namespace); String dataBaseName = namespace[0]; try { - return catalog.loadDatabaseProperties(dataBaseName); + return catalog.getDatabase(dataBaseName).options(); } catch (Catalog.DatabaseNotExistException e) { throw new NoSuchNamespaceException(namespace); } @@ -201,10 +201,7 @@ public boolean dropNamespace(String[] namespace) throws NoSuchNamespaceException */ public boolean dropNamespace(String[] namespace, boolean cascade) throws NoSuchNamespaceException { - checkArgument( - isValidateNamespace(namespace), - "Namespace %s is not valid", - Arrays.toString(namespace)); + checkNamespace(namespace); try { catalog.dropDatabase(namespace[0], false, cascade); return true; @@ -218,10 +215,7 @@ public boolean dropNamespace(String[] namespace, boolean cascade) @Override public Identifier[] listTables(String[] namespace) throws NoSuchNamespaceException { - checkArgument( - isValidateNamespace(namespace), - "Missing database in namespace: %s", - Arrays.toString(namespace)); + checkNamespace(namespace); try { return catalog.listTables(namespace[0]).stream() .map(table -> Identifier.of(namespace, table)) @@ -233,10 +227,7 @@ public Identifier[] listTables(String[] namespace) throws NoSuchNamespaceExcepti @Override public void invalidateTable(Identifier ident) { - try { - catalog.invalidateTable(toIdentifier(ident)); - } catch (NoSuchTableException ignored) { - } + catalog.invalidateTable(toIdentifier(ident)); } @Override @@ -283,15 +274,6 @@ public SparkTable loadTable(Identifier ident, long timestamp) throws NoSuchTable } } - @Override - public boolean tableExists(Identifier ident) { - try { - return catalog.tableExists(toIdentifier(ident)); - } catch (NoSuchTableException e) { - return false; - } - } - @Override public org.apache.spark.sql.connector.catalog.Table alterTable( Identifier ident, TableChange... changes) throws NoSuchTableException { @@ -315,26 +297,8 @@ public org.apache.spark.sql.connector.catalog.Table createTable( Map properties) throws TableAlreadyExistsException, NoSuchNamespaceException { try { - String provider = properties.get(TableCatalog.PROP_PROVIDER); - if ((!usePaimon(provider)) - && SparkSource.FORMAT_NAMES().contains(provider.toLowerCase())) { - Map newProperties = new HashMap<>(properties); - newProperties.put(TYPE.key(), FORMAT_TABLE.toString()); - newProperties.put(FILE_FORMAT.key(), provider.toLowerCase()); - catalog.createTable( - toIdentifier(ident), - toInitialSchema(schema, partitions, newProperties), - false); - } else { - checkArgument( - usePaimon(provider), - "SparkCatalog can only create paimon table, but current provider is %s", - provider); - catalog.createTable( - toIdentifier(ident), - toInitialSchema(schema, partitions, properties), - false); - } + catalog.createTable( + toIdentifier(ident), toInitialSchema(schema, partitions, properties), false); return loadTable(ident); } catch (Catalog.TableAlreadyExistException e) { throw new TableAlreadyExistsException(ident); @@ -350,7 +314,7 @@ public boolean dropTable(Identifier ident) { try { catalog.dropTable(toIdentifier(ident), false); return true; - } catch (Catalog.TableNotExistException | NoSuchTableException e) { + } catch (Catalog.TableNotExistException e) { return false; } } @@ -374,26 +338,22 @@ private SchemaChange toSchemaChange(TableChange change) { } } else if (change instanceof TableChange.AddColumn) { TableChange.AddColumn add = (TableChange.AddColumn) change; - validateAlterNestedField(add.fieldNames()); SchemaChange.Move move = getMove(add.position(), add.fieldNames()); return SchemaChange.addColumn( - add.fieldNames()[0], + add.fieldNames(), toPaimonType(add.dataType()).copy(add.isNullable()), add.comment(), move); } else if (change instanceof TableChange.RenameColumn) { TableChange.RenameColumn rename = (TableChange.RenameColumn) change; - validateAlterNestedField(rename.fieldNames()); - return SchemaChange.renameColumn(rename.fieldNames()[0], rename.newName()); + return SchemaChange.renameColumn(rename.fieldNames(), rename.newName()); } else if (change instanceof TableChange.DeleteColumn) { TableChange.DeleteColumn delete = (TableChange.DeleteColumn) change; - validateAlterNestedField(delete.fieldNames()); - return SchemaChange.dropColumn(delete.fieldNames()[0]); + return SchemaChange.dropColumn(delete.fieldNames()); } else if (change instanceof TableChange.UpdateColumnType) { TableChange.UpdateColumnType update = (TableChange.UpdateColumnType) change; - validateAlterNestedField(update.fieldNames()); return SchemaChange.updateColumnType( - update.fieldNames()[0], toPaimonType(update.newDataType()), true); + update.fieldNames(), toPaimonType(update.newDataType()), true); } else if (change instanceof TableChange.UpdateColumnNullability) { TableChange.UpdateColumnNullability update = (TableChange.UpdateColumnNullability) change; @@ -427,11 +387,18 @@ private static SchemaChange.Move getMove( private Schema toInitialSchema( StructType schema, Transform[] partitions, Map properties) { Map normalizedProperties = new HashMap<>(properties); - if (!normalizedProperties.containsKey(TableCatalog.PROP_PROVIDER)) { - normalizedProperties.put(TableCatalog.PROP_PROVIDER, SparkSource.NAME()); + String provider = properties.get(TableCatalog.PROP_PROVIDER); + if (!usePaimon(provider) && SparkSource.FORMAT_NAMES().contains(provider.toLowerCase())) { + normalizedProperties.put(TYPE.key(), FORMAT_TABLE.toString()); + normalizedProperties.put(FILE_FORMAT.key(), provider.toLowerCase()); } + normalizedProperties.remove(TableCatalog.PROP_PROVIDER); normalizedProperties.remove(PRIMARY_KEY_IDENTIFIER); normalizedProperties.remove(TableCatalog.PROP_COMMENT); + if (normalizedProperties.containsKey(TableCatalog.PROP_LOCATION)) { + String path = normalizedProperties.remove(TableCatalog.PROP_LOCATION); + normalizedProperties.put(CoreOptions.PATH.key(), path); + } String pkAsString = properties.get(PRIMARY_KEY_IDENTIFIER); List primaryKeys = pkAsString == null @@ -455,23 +422,12 @@ private Schema toInitialSchema( return schemaBuilder.build(); } - private void validateAlterNestedField(String[] fieldNames) { - if (fieldNames.length > 1) { - throw new UnsupportedOperationException( - "Alter nested column is not supported: " + Arrays.toString(fieldNames)); - } - } - private void validateAlterProperty(String alterKey) { if (PRIMARY_KEY_IDENTIFIER.equals(alterKey)) { throw new UnsupportedOperationException("Alter primary key is not supported"); } } - private boolean isValidateNamespace(String[] namespace) { - return namespace.length == 1; - } - @Override public void renameTable(Identifier oldIdent, Identifier newIdent) throws NoSuchTableException, TableAlreadyExistsException { @@ -486,65 +442,63 @@ public void renameTable(Identifier oldIdent, Identifier newIdent) // --------------------- tools ------------------------------------------ - protected org.apache.paimon.catalog.Identifier toIdentifier(Identifier ident) - throws NoSuchTableException { - if (!isValidateNamespace(ident.namespace())) { - throw new NoSuchTableException(ident); - } - - return new org.apache.paimon.catalog.Identifier(ident.namespace()[0], ident.name()); - } - protected org.apache.spark.sql.connector.catalog.Table loadSparkTable( Identifier ident, Map extraOptions) throws NoSuchTableException { try { org.apache.paimon.table.Table paimonTable = catalog.getTable(toIdentifier(ident)); if (paimonTable instanceof FormatTable) { - FormatTable formatTable = (FormatTable) paimonTable; - StructType schema = SparkTypeUtils.fromPaimonRowType(formatTable.rowType()); - List pathList = new ArrayList<>(); - pathList.add(formatTable.location()); - CaseInsensitiveStringMap dsOptions = - new CaseInsensitiveStringMap(formatTable.options()); - if (formatTable.format() == FormatTable.Format.CSV) { - return new CSVTable( - ident.name(), - SparkSession.active(), - dsOptions, - scala.collection.JavaConverters.asScalaBuffer(pathList).toSeq(), - scala.Option.apply(schema), - CSVFileFormat.class); - } else if (formatTable.format() == FormatTable.Format.ORC) { - return new OrcTable( - ident.name(), - SparkSession.active(), - dsOptions, - scala.collection.JavaConverters.asScalaBuffer(pathList).toSeq(), - scala.Option.apply(schema), - OrcFileFormat.class); - } else if (formatTable.format() == FormatTable.Format.PARQUET) { - return new ParquetTable( - ident.name(), - SparkSession.active(), - dsOptions, - scala.collection.JavaConverters.asScalaBuffer(pathList).toSeq(), - scala.Option.apply(schema), - ParquetFileFormat.class); - } else { - throw new UnsupportedOperationException( - "Unsupported format table " - + ident.name() - + " format " - + formatTable.format().name()); - } + return convertToFileTable(ident, (FormatTable) paimonTable); } else { - return new SparkTable(copyWithSQLConf(paimonTable, extraOptions)); + return new SparkTable( + copyWithSQLConf( + paimonTable, catalogName, toIdentifier(ident), extraOptions)); } } catch (Catalog.TableNotExistException e) { throw new NoSuchTableException(ident); } } + private static FileTable convertToFileTable(Identifier ident, FormatTable formatTable) { + StructType schema = SparkTypeUtils.fromPaimonRowType(formatTable.rowType()); + List pathList = new ArrayList<>(); + pathList.add(formatTable.location()); + Options options = Options.fromMap(formatTable.options()); + CaseInsensitiveStringMap dsOptions = new CaseInsensitiveStringMap(options.toMap()); + if (formatTable.format() == FormatTable.Format.CSV) { + options.set("sep", options.get(FormatTableOptions.FIELD_DELIMITER)); + dsOptions = new CaseInsensitiveStringMap(options.toMap()); + return new CSVTable( + ident.name(), + SparkSession.active(), + dsOptions, + scala.collection.JavaConverters.asScalaBuffer(pathList).toSeq(), + scala.Option.apply(schema), + CSVFileFormat.class); + } else if (formatTable.format() == FormatTable.Format.ORC) { + return new OrcTable( + ident.name(), + SparkSession.active(), + dsOptions, + scala.collection.JavaConverters.asScalaBuffer(pathList).toSeq(), + scala.Option.apply(schema), + OrcFileFormat.class); + } else if (formatTable.format() == FormatTable.Format.PARQUET) { + return new ParquetTable( + ident.name(), + SparkSession.active(), + dsOptions, + scala.collection.JavaConverters.asScalaBuffer(pathList).toSeq(), + scala.Option.apply(schema), + ParquetFileFormat.class); + } else { + throw new UnsupportedOperationException( + "Unsupported format table " + + ident.name() + + " format " + + formatTable.format().name()); + } + } + protected List convertPartitionTransforms(Transform[] transforms) { List partitionColNames = new ArrayList<>(transforms.length); for (Transform transform : transforms) { diff --git a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkGenericCatalog.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkGenericCatalog.java index 12407f2614ff..6b7b17b1b1a5 100644 --- a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkGenericCatalog.java +++ b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkGenericCatalog.java @@ -52,6 +52,7 @@ import org.apache.spark.sql.internal.SQLConf; import org.apache.spark.sql.internal.SessionState; import org.apache.spark.sql.internal.StaticSQLConf; +import org.apache.spark.sql.paimon.shims.SparkShimLoader; import org.apache.spark.sql.types.StructType; import org.apache.spark.sql.util.CaseInsensitiveStringMap; import org.slf4j.Logger; @@ -185,7 +186,7 @@ public Table loadTable(Identifier ident, long timestamp) throws NoSuchTableExcep @Override public void invalidateTable(Identifier ident) { // We do not need to check whether the table exists and whether - // it is an Paimon table to reduce remote service requests. + // it is a Paimon table to reduce remote service requests. sparkCatalog.invalidateTable(ident); asTableCatalog().invalidateTable(ident); } @@ -202,7 +203,8 @@ public Table createTable( return sparkCatalog.createTable(ident, schema, partitions, properties); } else { // delegate to the session catalog - return asTableCatalog().createTable(ident, schema, partitions, properties); + return SparkShimLoader.getSparkShim() + .createTable(asTableCatalog(), ident, schema, partitions, properties); } } @@ -286,12 +288,6 @@ private CaseInsensitiveStringMap autoFillConfigurations( Map newOptions = new HashMap<>(options.asCaseSensitiveMap()); fillAliyunConfigurations(newOptions, hadoopConf); fillCommonConfigurations(newOptions, sqlConf); - - // if spark is case-insensitive, set allow upper case to catalog - if (!sqlConf.caseSensitiveAnalysis()) { - newOptions.put(ALLOW_UPPER_CASE.key(), "true"); - } - return new CaseInsensitiveStringMap(newOptions); } @@ -311,12 +307,16 @@ private void fillCommonConfigurations(Map options, SQLConf sqlCo String warehouse = sqlConf.warehousePath(); options.put(WAREHOUSE.key(), warehouse); } + if (!options.containsKey(METASTORE.key())) { String metastore = sqlConf.getConf(StaticSQLConf.CATALOG_IMPLEMENTATION()); if (HiveCatalogOptions.IDENTIFIER.equals(metastore)) { options.put(METASTORE.key(), metastore); } } + + options.put(CatalogOptions.FORMAT_TABLE_ENABLED.key(), "false"); + String sessionCatalogDefaultDatabase = SQLConfUtils.defaultDatabase(sqlConf); if (options.containsKey(DEFAULT_DATABASE.key())) { String userDefineDefaultDatabase = options.get(DEFAULT_DATABASE.key()); @@ -330,6 +330,11 @@ private void fillCommonConfigurations(Map options, SQLConf sqlCo } else { options.put(DEFAULT_DATABASE.key(), sessionCatalogDefaultDatabase); } + + // if spark is case-insensitive, set allow upper case to catalog + if (!sqlConf.caseSensitiveAnalysis()) { + options.put(ALLOW_UPPER_CASE.key(), "true"); + } } @Override diff --git a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkProcedures.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkProcedures.java index 1d5bd9df2f86..f5052ea25f95 100644 --- a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkProcedures.java +++ b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkProcedures.java @@ -18,6 +18,7 @@ package org.apache.paimon.spark; +import org.apache.paimon.spark.procedure.CompactManifestProcedure; import org.apache.paimon.spark.procedure.CompactProcedure; import org.apache.paimon.spark.procedure.CreateBranchProcedure; import org.apache.paimon.spark.procedure.CreateTagFromTimestampProcedure; @@ -26,6 +27,7 @@ import org.apache.paimon.spark.procedure.DeleteTagProcedure; import org.apache.paimon.spark.procedure.ExpirePartitionsProcedure; import org.apache.paimon.spark.procedure.ExpireSnapshotsProcedure; +import org.apache.paimon.spark.procedure.ExpireTagsProcedure; import org.apache.paimon.spark.procedure.FastForwardProcedure; import org.apache.paimon.spark.procedure.MarkPartitionDoneProcedure; import org.apache.paimon.spark.procedure.MigrateDatabaseProcedure; @@ -33,16 +35,22 @@ import org.apache.paimon.spark.procedure.MigrateTableProcedure; import org.apache.paimon.spark.procedure.Procedure; import org.apache.paimon.spark.procedure.ProcedureBuilder; +import org.apache.paimon.spark.procedure.PurgeFilesProcedure; +import org.apache.paimon.spark.procedure.RefreshObjectTableProcedure; import org.apache.paimon.spark.procedure.RemoveOrphanFilesProcedure; import org.apache.paimon.spark.procedure.RenameTagProcedure; import org.apache.paimon.spark.procedure.RepairProcedure; +import org.apache.paimon.spark.procedure.ReplaceTagProcedure; import org.apache.paimon.spark.procedure.ResetConsumerProcedure; import org.apache.paimon.spark.procedure.RollbackProcedure; +import org.apache.paimon.spark.procedure.RollbackToTimestampProcedure; +import org.apache.paimon.spark.procedure.RollbackToWatermarkProcedure; import org.apache.paimon.shade.guava30.com.google.common.collect.ImmutableMap; import java.util.Locale; import java.util.Map; +import java.util.Set; import java.util.function.Supplier; /** The {@link Procedure}s including all the stored procedures. */ @@ -57,15 +65,24 @@ public static ProcedureBuilder newBuilder(String name) { return builderSupplier != null ? builderSupplier.get() : null; } + public static Set names() { + return BUILDERS.keySet(); + } + private static Map> initProcedureBuilders() { ImmutableMap.Builder> procedureBuilders = ImmutableMap.builder(); procedureBuilders.put("rollback", RollbackProcedure::builder); + procedureBuilders.put("rollback_to_timestamp", RollbackToTimestampProcedure::builder); + procedureBuilders.put("rollback_to_watermark", RollbackToWatermarkProcedure::builder); + procedureBuilders.put("purge_files", PurgeFilesProcedure::builder); procedureBuilders.put("create_tag", CreateTagProcedure::builder); + procedureBuilders.put("replace_tag", ReplaceTagProcedure::builder); procedureBuilders.put("rename_tag", RenameTagProcedure::builder); procedureBuilders.put( "create_tag_from_timestamp", CreateTagFromTimestampProcedure::builder); procedureBuilders.put("delete_tag", DeleteTagProcedure::builder); + procedureBuilders.put("expire_tags", ExpireTagsProcedure::builder); procedureBuilders.put("create_branch", CreateBranchProcedure::builder); procedureBuilders.put("delete_branch", DeleteBranchProcedure::builder); procedureBuilders.put("compact", CompactProcedure::builder); @@ -79,6 +96,8 @@ private static Map> initProcedureBuilders() { procedureBuilders.put("fast_forward", FastForwardProcedure::builder); procedureBuilders.put("reset_consumer", ResetConsumerProcedure::builder); procedureBuilders.put("mark_partition_done", MarkPartitionDoneProcedure::builder); + procedureBuilders.put("compact_manifest", CompactManifestProcedure::builder); + procedureBuilders.put("refresh_object_table", RefreshObjectTableProcedure::builder); return procedureBuilders.build(); } } diff --git a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkTypeUtils.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkTypeUtils.java index 8bba676200ce..f6643f758406 100644 --- a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkTypeUtils.java +++ b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkTypeUtils.java @@ -81,6 +81,10 @@ public static DataType fromPaimonType(org.apache.paimon.types.DataType type) { return type.accept(PaimonToSparkTypeVisitor.INSTANCE); } + public static org.apache.paimon.types.RowType toPaimonRowType(StructType type) { + return (RowType) toPaimonType(type); + } + public static org.apache.paimon.types.DataType toPaimonType(DataType dataType) { return SparkToPaimonTypeVisitor.visit(dataType); } diff --git a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/catalog/SupportFunction.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/catalog/SupportFunction.java index 91a6d7b4a2e6..772a2f4ed53d 100644 --- a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/catalog/SupportFunction.java +++ b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/catalog/SupportFunction.java @@ -29,8 +29,6 @@ import java.util.Arrays; -import scala.Option; - import static org.apache.paimon.catalog.Catalog.SYSTEM_DATABASE_NAME; /** Catalog methods for working with Functions. */ @@ -56,8 +54,7 @@ default Identifier[] listFunctions(String[] namespace) throws NoSuchNamespaceExc return new Identifier[0]; } - throw new NoSuchNamespaceException( - "Namespace " + Arrays.toString(namespace) + " is not valid", Option.empty()); + throw new RuntimeException("Namespace " + Arrays.toString(namespace) + " is not valid"); } @Override @@ -69,7 +66,6 @@ default UnboundFunction loadFunction(Identifier ident) throws NoSuchFunctionExce } } - throw new NoSuchFunctionException( - "Function " + ident + " is not a paimon function", Option.empty()); + throw new RuntimeException("Function " + ident + " is not a paimon function"); } } diff --git a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/catalog/SupportView.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/catalog/SupportView.java new file mode 100644 index 000000000000..b8ce86e89286 --- /dev/null +++ b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/catalog/SupportView.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.catalog; + +import org.apache.paimon.catalog.Catalog; +import org.apache.paimon.spark.SparkTypeUtils; +import org.apache.paimon.view.View; +import org.apache.paimon.view.ViewImpl; + +import org.apache.spark.sql.catalyst.analysis.NoSuchNamespaceException; +import org.apache.spark.sql.connector.catalog.Identifier; +import org.apache.spark.sql.types.StructType; + +import java.util.List; +import java.util.Map; + +import static org.apache.paimon.spark.utils.CatalogUtils.checkNamespace; +import static org.apache.paimon.spark.utils.CatalogUtils.toIdentifier; + +/** Catalog methods for working with Views. */ +public interface SupportView extends WithPaimonCatalog { + + default List listViews(String[] namespace) throws NoSuchNamespaceException { + try { + checkNamespace(namespace); + return paimonCatalog().listViews(namespace[0]); + } catch (Catalog.DatabaseNotExistException e) { + throw new NoSuchNamespaceException(namespace); + } + } + + default View loadView(Identifier ident) throws Catalog.ViewNotExistException { + return paimonCatalog().getView(toIdentifier(ident)); + } + + default void createView( + Identifier ident, + StructType schema, + String queryText, + String comment, + Map properties, + Boolean ignoreIfExists) + throws NoSuchNamespaceException { + org.apache.paimon.catalog.Identifier paimonIdent = toIdentifier(ident); + try { + paimonCatalog() + .createView( + paimonIdent, + new ViewImpl( + paimonIdent, + SparkTypeUtils.toPaimonRowType(schema), + queryText, + comment, + properties), + ignoreIfExists); + } catch (Catalog.ViewAlreadyExistException e) { + throw new RuntimeException("view already exists: " + ident, e); + } catch (Catalog.DatabaseNotExistException e) { + throw new NoSuchNamespaceException(ident.namespace()); + } + } + + default void dropView(Identifier ident, Boolean ignoreIfExists) { + try { + paimonCatalog().dropView(toIdentifier(ident), ignoreIfExists); + } catch (Catalog.ViewNotExistException e) { + throw new RuntimeException("view not exists: " + ident, e); + } + } +} diff --git a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/CompactManifestProcedure.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/CompactManifestProcedure.java new file mode 100644 index 000000000000..dd064d892c3d --- /dev/null +++ b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/CompactManifestProcedure.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.procedure; + +import org.apache.paimon.operation.FileStoreCommit; +import org.apache.paimon.table.FileStoreTable; + +import org.apache.spark.sql.catalyst.InternalRow; +import org.apache.spark.sql.connector.catalog.Identifier; +import org.apache.spark.sql.connector.catalog.TableCatalog; +import org.apache.spark.sql.types.DataTypes; +import org.apache.spark.sql.types.Metadata; +import org.apache.spark.sql.types.StructField; +import org.apache.spark.sql.types.StructType; + +import static org.apache.spark.sql.types.DataTypes.StringType; + +/** + * Compact manifest procedure. Usage: + * + * 
    
+ *  CALL sys.compact_manifest(table => 'tableId')
+ *
    + */ +public class CompactManifestProcedure extends BaseProcedure { + + private static final ProcedureParameter[] PARAMETERS = + new ProcedureParameter[] {ProcedureParameter.required("table", StringType)}; + + private static final StructType OUTPUT_TYPE = + new StructType( + new StructField[] { + new StructField("result", DataTypes.BooleanType, true, Metadata.empty()) + }); + + protected CompactManifestProcedure(TableCatalog tableCatalog) { + super(tableCatalog); + } + + @Override + public ProcedureParameter[] parameters() { + return PARAMETERS; + } + + @Override + public StructType outputType() { + return OUTPUT_TYPE; + } + + @Override + public InternalRow[] call(InternalRow args) { + + Identifier tableIdent = toIdentifier(args.getString(0), PARAMETERS[0].name()); + FileStoreTable table = (FileStoreTable) loadSparkTable(tableIdent).getTable(); + + try (FileStoreCommit commit = + table.store() + .newCommit(table.coreOptions().createCommitUser()) + .ignoreEmptyCommit(false)) { + commit.compactManifest(); + } + + return new InternalRow[] {newInternalRow(true)}; + } + + @Override + public String description() { + return "This procedure execute compact action on paimon table."; + } + + public static ProcedureBuilder builder() { + return new Builder() { + @Override + public CompactManifestProcedure doBuild() { + return new CompactManifestProcedure(tableCatalog()); + } + }; + } +} diff --git a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/CompactProcedure.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/CompactProcedure.java index 71cf04cf5ef5..4a43e39c31ba 100644 --- a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/CompactProcedure.java +++ b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/CompactProcedure.java @@ -107,6 +107,7 @@ public class CompactProcedure extends BaseProcedure { new ProcedureParameter[] { ProcedureParameter.required("table", StringType), ProcedureParameter.optional("partitions", StringType), + ProcedureParameter.optional("compact_strategy", StringType), ProcedureParameter.optional("order_strategy", StringType), ProcedureParameter.optional("order_by", StringType), ProcedureParameter.optional("where", StringType), @@ -120,6 +121,9 @@ public class CompactProcedure extends BaseProcedure { new StructField("result", DataTypes.BooleanType, true, Metadata.empty()) }); + private static final String MINOR = "minor"; + private static final String FULL = "full"; + protected CompactProcedure(TableCatalog tableCatalog) { super(tableCatalog); } @@ -138,15 +142,17 @@ public StructType outputType() { public InternalRow[] call(InternalRow args) { Identifier tableIdent = toIdentifier(args.getString(0), PARAMETERS[0].name()); String partitions = blank(args, 1) ? null : args.getString(1); - String sortType = blank(args, 2) ? TableSorter.OrderType.NONE.name() : args.getString(2); + // make full compact strategy as default. + String compactStrategy = blank(args, 2) ? FULL : args.getString(2); + String sortType = blank(args, 3) ? TableSorter.OrderType.NONE.name() : args.getString(3); List sortColumns = - blank(args, 3) + blank(args, 4) ? Collections.emptyList() - : Arrays.asList(args.getString(3).split(",")); - String where = blank(args, 4) ? null : args.getString(4); - String options = args.isNullAt(5) ? null : args.getString(5); + : Arrays.asList(args.getString(4).split(",")); + String where = blank(args, 5) ? null : args.getString(5); + String options = args.isNullAt(6) ? null : args.getString(6); Duration partitionIdleTime = - blank(args, 6) ? null : TimeUtils.parseDuration(args.getString(6)); + blank(args, 7) ? null : TimeUtils.parseDuration(args.getString(7)); if (TableSorter.OrderType.NONE.name().equals(sortType) && !sortColumns.isEmpty()) { throw new IllegalArgumentException( "order_strategy \"none\" cannot work with order_by columns."); @@ -155,6 +161,14 @@ public InternalRow[] call(InternalRow args) { throw new IllegalArgumentException( "sort compact do not support 'partition_idle_time'."); } + + if (!(compactStrategy.equalsIgnoreCase(FULL) || compactStrategy.equalsIgnoreCase(MINOR))) { + throw new IllegalArgumentException( + String.format( + "The compact strategy only supports 'full' or 'minor', but '%s' is configured.", + compactStrategy)); + } + checkArgument( partitions == null || where == null, "partitions and where cannot be used together."); @@ -192,6 +206,7 @@ public InternalRow[] call(InternalRow args) { newInternalRow( execute( (FileStoreTable) table, + compactStrategy, sortType, sortColumns, relation, @@ -212,6 +227,7 @@ private boolean blank(InternalRow args, int index) { private boolean execute( FileStoreTable table, + String compactStrategy, String sortType, List sortColumns, DataSourceV2Relation relation, @@ -219,6 +235,7 @@ private boolean execute( @Nullable Duration partitionIdleTime) { BucketMode bucketMode = table.bucketMode(); TableSorter.OrderType orderType = TableSorter.OrderType.of(sortType); + boolean fullCompact = compactStrategy.equalsIgnoreCase(FULL); Predicate filter = condition == null ? null @@ -233,7 +250,8 @@ private boolean execute( switch (bucketMode) { case HASH_FIXED: case HASH_DYNAMIC: - compactAwareBucketTable(table, filter, partitionIdleTime, javaSparkContext); + compactAwareBucketTable( + table, fullCompact, filter, partitionIdleTime, javaSparkContext); break; case BUCKET_UNAWARE: compactUnAwareBucketTable(table, filter, partitionIdleTime, javaSparkContext); @@ -259,6 +277,7 @@ private boolean execute( private void compactAwareBucketTable( FileStoreTable table, + boolean fullCompact, @Nullable Predicate filter, @Nullable Duration partitionIdleTime, JavaSparkContext javaSparkContext) { @@ -304,7 +323,7 @@ private void compactAwareBucketTable( SerializationUtils.deserializeBinaryRow( pair.getLeft()), pair.getRight(), - true); + fullCompact); } CommitMessageSerializer serializer = new CommitMessageSerializer(); diff --git a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/CreateOrReplaceTagBaseProcedure.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/CreateOrReplaceTagBaseProcedure.java new file mode 100644 index 000000000000..ed264140b797 --- /dev/null +++ b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/CreateOrReplaceTagBaseProcedure.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.procedure; + +import org.apache.paimon.table.Table; +import org.apache.paimon.utils.TimeUtils; + +import org.apache.spark.sql.catalyst.InternalRow; +import org.apache.spark.sql.connector.catalog.Identifier; +import org.apache.spark.sql.connector.catalog.TableCatalog; +import org.apache.spark.sql.types.DataTypes; +import org.apache.spark.sql.types.Metadata; +import org.apache.spark.sql.types.StructField; +import org.apache.spark.sql.types.StructType; + +import java.time.Duration; + +import static org.apache.spark.sql.types.DataTypes.LongType; +import static org.apache.spark.sql.types.DataTypes.StringType; + +/** A base procedure to create or replace a tag. */ +public abstract class CreateOrReplaceTagBaseProcedure extends BaseProcedure { + + private static final ProcedureParameter[] PARAMETERS = + new ProcedureParameter[] { + ProcedureParameter.required("table", StringType), + ProcedureParameter.required("tag", StringType), + ProcedureParameter.optional("snapshot", LongType), + ProcedureParameter.optional("time_retained", StringType) + }; + + private static final StructType OUTPUT_TYPE = + new StructType( + new StructField[] { + new StructField("result", DataTypes.BooleanType, true, Metadata.empty()) + }); + + protected CreateOrReplaceTagBaseProcedure(TableCatalog tableCatalog) { + super(tableCatalog); + } + + @Override + public ProcedureParameter[] parameters() { + return PARAMETERS; + } + + @Override + public StructType outputType() { + return OUTPUT_TYPE; + } + + @Override + public InternalRow[] call(InternalRow args) { + Identifier tableIdent = toIdentifier(args.getString(0), PARAMETERS[0].name()); + String tag = args.getString(1); + Long snapshot = args.isNullAt(2) ? null : args.getLong(2); + Duration timeRetained = + args.isNullAt(3) ? null : TimeUtils.parseDuration(args.getString(3)); + + return modifyPaimonTable( + tableIdent, + table -> { + createOrReplaceTag(table, tag, snapshot, timeRetained); + InternalRow outputRow = newInternalRow(true); + return new InternalRow[] {outputRow}; + }); + } + + abstract void createOrReplaceTag( + Table table, String tagName, Long snapshotId, Duration timeRetained); +} diff --git a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/CreateTagProcedure.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/CreateTagProcedure.java index b3f863c5e305..157743f9892e 100644 --- a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/CreateTagProcedure.java +++ b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/CreateTagProcedure.java @@ -18,71 +18,26 @@ package org.apache.paimon.spark.procedure; -import org.apache.paimon.utils.TimeUtils; +import org.apache.paimon.table.Table; -import org.apache.spark.sql.catalyst.InternalRow; -import org.apache.spark.sql.connector.catalog.Identifier; import org.apache.spark.sql.connector.catalog.TableCatalog; -import org.apache.spark.sql.types.DataTypes; -import org.apache.spark.sql.types.Metadata; -import org.apache.spark.sql.types.StructField; -import org.apache.spark.sql.types.StructType; import java.time.Duration; -import static org.apache.spark.sql.types.DataTypes.LongType; -import static org.apache.spark.sql.types.DataTypes.StringType; - /** A procedure to create a tag. */ -public class CreateTagProcedure extends BaseProcedure { - - private static final ProcedureParameter[] PARAMETERS = - new ProcedureParameter[] { - ProcedureParameter.required("table", StringType), - ProcedureParameter.required("tag", StringType), - ProcedureParameter.optional("snapshot", LongType), - ProcedureParameter.optional("time_retained", StringType) - }; - - private static final StructType OUTPUT_TYPE = - new StructType( - new StructField[] { - new StructField("result", DataTypes.BooleanType, true, Metadata.empty()) - }); +public class CreateTagProcedure extends CreateOrReplaceTagBaseProcedure { - protected CreateTagProcedure(TableCatalog tableCatalog) { + private CreateTagProcedure(TableCatalog tableCatalog) { super(tableCatalog); } @Override - public ProcedureParameter[] parameters() { - return PARAMETERS; - } - - @Override - public StructType outputType() { - return OUTPUT_TYPE; - } - - @Override - public InternalRow[] call(InternalRow args) { - Identifier tableIdent = toIdentifier(args.getString(0), PARAMETERS[0].name()); - String tag = args.getString(1); - Long snapshot = args.isNullAt(2) ? null : args.getLong(2); - Duration timeRetained = - args.isNullAt(3) ? null : TimeUtils.parseDuration(args.getString(3)); - - return modifyPaimonTable( - tableIdent, - table -> { - if (snapshot == null) { - table.createTag(tag, timeRetained); - } else { - table.createTag(tag, snapshot, timeRetained); - } - InternalRow outputRow = newInternalRow(true); - return new InternalRow[] {outputRow}; - }); + void createOrReplaceTag(Table table, String tagName, Long snapshotId, Duration timeRetained) { + if (snapshotId == null) { + table.createTag(tagName, timeRetained); + } else { + table.createTag(tagName, snapshotId, timeRetained); + } } public static ProcedureBuilder builder() { diff --git a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/DeleteBranchProcedure.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/DeleteBranchProcedure.java index e398eee0261f..4a01c33d6af1 100644 --- a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/DeleteBranchProcedure.java +++ b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/DeleteBranchProcedure.java @@ -18,6 +18,8 @@ package org.apache.paimon.spark.procedure; +import org.apache.paimon.spark.catalog.WithPaimonCatalog; + import org.apache.spark.sql.catalyst.InternalRow; import org.apache.spark.sql.connector.catalog.Identifier; import org.apache.spark.sql.connector.catalog.TableCatalog; @@ -61,13 +63,20 @@ public StructType outputType() { public InternalRow[] call(InternalRow args) { Identifier tableIdent = toIdentifier(args.getString(0), PARAMETERS[0].name()); String branchStr = args.getString(1); - return modifyPaimonTable( - tableIdent, - table -> { - table.deleteBranches(branchStr); - InternalRow outputRow = newInternalRow(true); - return new InternalRow[] {outputRow}; - }); + InternalRow[] result = + modifyPaimonTable( + tableIdent, + table -> { + table.deleteBranches(branchStr); + InternalRow outputRow = newInternalRow(true); + return new InternalRow[] {outputRow}; + }); + ((WithPaimonCatalog) tableCatalog()) + .paimonCatalog() + .invalidateTable( + new org.apache.paimon.catalog.Identifier( + tableIdent.namespace()[0], tableIdent.name(), branchStr)); + return result; } public static ProcedureBuilder builder() { diff --git a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/ExpirePartitionsProcedure.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/ExpirePartitionsProcedure.java index 7b388227e5a4..e3a53d2bd2ef 100644 --- a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/ExpirePartitionsProcedure.java +++ b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/ExpirePartitionsProcedure.java @@ -107,9 +107,10 @@ public InternalRow[] call(InternalRow args) { .catalogEnvironment() .metastoreClientFactory()) .map(MetastoreClient.Factory::create) - .orElse(null)); + .orElse(null), + fileStore.options().partitionExpireMaxNum()); if (maxExpires != null) { - partitionExpire.withMaxExpires(maxExpires); + partitionExpire.withMaxExpireNum(maxExpires); } List> expired = partitionExpire.expire(Long.MAX_VALUE); return expired == null || expired.isEmpty() diff --git a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/ExpireTagsProcedure.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/ExpireTagsProcedure.java new file mode 100644 index 000000000000..d75ca5ee0aac --- /dev/null +++ b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/ExpireTagsProcedure.java @@ -0,0 +1,109 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.procedure; + +import org.apache.paimon.table.FileStoreTable; +import org.apache.paimon.tag.TagTimeExpire; +import org.apache.paimon.utils.DateTimeUtils; + +import org.apache.spark.sql.catalyst.InternalRow; +import org.apache.spark.sql.connector.catalog.Identifier; +import org.apache.spark.sql.connector.catalog.TableCatalog; +import org.apache.spark.sql.types.Metadata; +import org.apache.spark.sql.types.StructField; +import org.apache.spark.sql.types.StructType; +import org.apache.spark.unsafe.types.UTF8String; + +import java.time.LocalDateTime; +import java.util.List; +import java.util.TimeZone; + +import static org.apache.spark.sql.types.DataTypes.StringType; + +/** A procedure to expire tags by time. */ +public class ExpireTagsProcedure extends BaseProcedure { + + private static final ProcedureParameter[] PARAMETERS = + new ProcedureParameter[] { + ProcedureParameter.required("table", StringType), + ProcedureParameter.optional("older_than", StringType) + }; + + private static final StructType OUTPUT_TYPE = + new StructType( + new StructField[] { + new StructField("expired_tags", StringType, false, Metadata.empty()) + }); + + protected ExpireTagsProcedure(TableCatalog tableCatalog) { + super(tableCatalog); + } + + @Override + public ProcedureParameter[] parameters() { + return PARAMETERS; + } + + @Override + public StructType outputType() { + return OUTPUT_TYPE; + } + + @Override + public InternalRow[] call(InternalRow args) { + Identifier tableIdent = toIdentifier(args.getString(0), PARAMETERS[0].name()); + String olderThanStr = args.isNullAt(1) ? null : args.getString(1); + return modifyPaimonTable( + tableIdent, + table -> { + FileStoreTable fileStoreTable = (FileStoreTable) table; + TagTimeExpire tagTimeExpire = + fileStoreTable.store().newTagCreationManager().getTagTimeExpire(); + if (olderThanStr != null) { + LocalDateTime olderThanTime = + DateTimeUtils.parseTimestampData( + olderThanStr, 3, TimeZone.getDefault()) + .toLocalDateTime(); + tagTimeExpire.withOlderThanTime(olderThanTime); + } + List expired = tagTimeExpire.expire(); + return expired.isEmpty() + ? new InternalRow[] { + newInternalRow(UTF8String.fromString("No expired tags.")) + } + : expired.stream() + .map(x -> newInternalRow(UTF8String.fromString(x))) + .toArray(InternalRow[]::new); + }); + } + + public static ProcedureBuilder builder() { + return new BaseProcedure.Builder() { + @Override + public ExpireTagsProcedure doBuild() { + return new ExpireTagsProcedure(tableCatalog()); + } + }; + } + + @Override + public String description() { + return "ExpireTagsProcedure"; + } +} diff --git a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/MigrateFileProcedure.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/MigrateFileProcedure.java index 32f89d47b1a0..95d55df01178 100644 --- a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/MigrateFileProcedure.java +++ b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/MigrateFileProcedure.java @@ -88,7 +88,9 @@ public InternalRow[] call(InternalRow args) { Catalog paimonCatalog = ((WithPaimonCatalog) tableCatalog()).paimonCatalog(); - if (!(paimonCatalog.tableExists(targetTableId))) { + try { + paimonCatalog.getTable(targetTableId); + } catch (Catalog.TableNotExistException e) { throw new IllegalArgumentException( "Target paimon table does not exist: " + targetTable); } diff --git a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/PurgeFilesProcedure.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/PurgeFilesProcedure.java new file mode 100644 index 000000000000..8a7aec6e1410 --- /dev/null +++ b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/PurgeFilesProcedure.java @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.procedure; + +import org.apache.paimon.fs.FileIO; +import org.apache.paimon.fs.Path; +import org.apache.paimon.table.FileStoreTable; + +import org.apache.spark.sql.catalyst.InternalRow; +import org.apache.spark.sql.connector.catalog.Identifier; +import org.apache.spark.sql.connector.catalog.TableCatalog; +import org.apache.spark.sql.types.Metadata; +import org.apache.spark.sql.types.StructField; +import org.apache.spark.sql.types.StructType; +import org.apache.spark.unsafe.types.UTF8String; + +import java.io.IOException; +import java.util.Arrays; + +import static org.apache.spark.sql.types.DataTypes.StringType; + +/** A procedure to purge files for a table. */ +public class PurgeFilesProcedure extends BaseProcedure { + + private static final ProcedureParameter[] PARAMETERS = + new ProcedureParameter[] {ProcedureParameter.required("table", StringType)}; + + private static final StructType OUTPUT_TYPE = + new StructType( + new StructField[] { + new StructField("result", StringType, true, Metadata.empty()) + }); + + private PurgeFilesProcedure(TableCatalog tableCatalog) { + super(tableCatalog); + } + + @Override + public ProcedureParameter[] parameters() { + return PARAMETERS; + } + + @Override + public StructType outputType() { + return OUTPUT_TYPE; + } + + @Override + public InternalRow[] call(InternalRow args) { + Identifier tableIdent = toIdentifier(args.getString(0), PARAMETERS[0].name()); + + return modifyPaimonTable( + tableIdent, + table -> { + FileStoreTable fileStoreTable = (FileStoreTable) table; + FileIO fileIO = fileStoreTable.fileIO(); + Path tablePath = fileStoreTable.snapshotManager().tablePath(); + try { + Arrays.stream(fileIO.listStatus(tablePath)) + .filter(f -> !f.getPath().getName().contains("schema")) + .forEach( + fileStatus -> { + try { + fileIO.delete(fileStatus.getPath(), true); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + } catch (IOException e) { + throw new RuntimeException(e); + } + + InternalRow outputRow = + newInternalRow( + UTF8String.fromString( + String.format( + "Success purge files with table: %s.", + fileStoreTable.name()))); + return new InternalRow[] {outputRow}; + }); + } + + public static ProcedureBuilder builder() { + return new BaseProcedure.Builder() { + @Override + public PurgeFilesProcedure doBuild() { + return new PurgeFilesProcedure(tableCatalog()); + } + }; + } + + @Override + public String description() { + return "PurgeFilesProcedure"; + } +} diff --git a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/RefreshObjectTableProcedure.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/RefreshObjectTableProcedure.java new file mode 100644 index 000000000000..c6b6fdab4723 --- /dev/null +++ b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/RefreshObjectTableProcedure.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.procedure; + +import org.apache.paimon.table.object.ObjectTable; + +import org.apache.spark.sql.catalyst.InternalRow; +import org.apache.spark.sql.connector.catalog.Identifier; +import org.apache.spark.sql.connector.catalog.TableCatalog; +import org.apache.spark.sql.types.DataTypes; +import org.apache.spark.sql.types.Metadata; +import org.apache.spark.sql.types.StructField; +import org.apache.spark.sql.types.StructType; + +import static org.apache.spark.sql.types.DataTypes.StringType; + +/** Spark procedure to refresh Object Table. */ +public class RefreshObjectTableProcedure extends BaseProcedure { + + private static final ProcedureParameter[] PARAMETERS = + new ProcedureParameter[] {ProcedureParameter.required("table", StringType)}; + + private static final StructType OUTPUT_TYPE = + new StructType( + new StructField[] { + new StructField("file_number", DataTypes.LongType, false, Metadata.empty()) + }); + + protected RefreshObjectTableProcedure(TableCatalog tableCatalog) { + super(tableCatalog); + } + + @Override + public ProcedureParameter[] parameters() { + return PARAMETERS; + } + + @Override + public StructType outputType() { + return OUTPUT_TYPE; + } + + @Override + public InternalRow[] call(InternalRow args) { + Identifier tableIdent = toIdentifier(args.getString(0), PARAMETERS[0].name()); + return modifyPaimonTable( + tableIdent, + table -> { + ObjectTable objectTable = (ObjectTable) table; + long fileNumber = objectTable.refresh(); + InternalRow outputRow = newInternalRow(fileNumber); + return new InternalRow[] {outputRow}; + }); + } + + public static ProcedureBuilder builder() { + return new Builder() { + @Override + public RefreshObjectTableProcedure doBuild() { + return new RefreshObjectTableProcedure(tableCatalog()); + } + }; + } + + @Override + public String description() { + return "RefreshObjectTableProcedure"; + } +} diff --git a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/RemoveOrphanFilesProcedure.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/RemoveOrphanFilesProcedure.java index 293e84ca14bd..a929641106c6 100644 --- a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/RemoveOrphanFilesProcedure.java +++ b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/RemoveOrphanFilesProcedure.java @@ -19,6 +19,7 @@ package org.apache.paimon.spark.procedure; import org.apache.paimon.catalog.Catalog; +import org.apache.paimon.operation.CleanOrphanFilesResult; import org.apache.paimon.operation.LocalOrphanFilesClean; import org.apache.paimon.operation.OrphanFilesClean; import org.apache.paimon.spark.catalog.WithPaimonCatalog; @@ -66,7 +67,9 @@ public class RemoveOrphanFilesProcedure extends BaseProcedure { private static final StructType OUTPUT_TYPE = new StructType( new StructField[] { - new StructField("result", LongType, true, Metadata.empty()) + new StructField("deletedFileCount", LongType, true, Metadata.empty()), + new StructField( + "deletedFileTotalLenInBytes", LongType, true, Metadata.empty()) }); private RemoveOrphanFilesProcedure(TableCatalog tableCatalog) { @@ -104,11 +107,11 @@ public InternalRow[] call(InternalRow args) { Catalog catalog = ((WithPaimonCatalog) tableCatalog()).paimonCatalog(); String mode = args.isNullAt(4) ? "DISTRIBUTED" : args.getString(4); - long deletedFiles; + CleanOrphanFilesResult cleanOrphanFilesResult; try { switch (mode.toUpperCase(Locale.ROOT)) { case "LOCAL": - deletedFiles = + cleanOrphanFilesResult = LocalOrphanFilesClean.executeDatabaseOrphanFiles( catalog, identifier.getDatabaseName(), @@ -120,7 +123,7 @@ public InternalRow[] call(InternalRow args) { args.isNullAt(3) ? null : args.getInt(3)); break; case "DISTRIBUTED": - deletedFiles = + cleanOrphanFilesResult = SparkOrphanFilesClean.executeDatabaseOrphanFiles( catalog, identifier.getDatabaseName(), @@ -137,7 +140,12 @@ public InternalRow[] call(InternalRow args) { + mode + ". Only 'DISTRIBUTED' and 'LOCAL' are supported."); } - return new InternalRow[] {newInternalRow(deletedFiles)}; + + return new InternalRow[] { + newInternalRow( + cleanOrphanFilesResult.getDeletedFileCount(), + cleanOrphanFilesResult.getDeletedFileTotalLenInBytes()) + }; } catch (Exception e) { throw new RuntimeException(e); } diff --git a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/ReplaceTagProcedure.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/ReplaceTagProcedure.java new file mode 100644 index 000000000000..205fca5ee69e --- /dev/null +++ b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/ReplaceTagProcedure.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.procedure; + +import org.apache.paimon.table.Table; + +import org.apache.spark.sql.connector.catalog.TableCatalog; + +import java.time.Duration; + +/** A procedure to replace a tag. */ +public class ReplaceTagProcedure extends CreateOrReplaceTagBaseProcedure { + + private ReplaceTagProcedure(TableCatalog tableCatalog) { + super(tableCatalog); + } + + @Override + void createOrReplaceTag(Table table, String tagName, Long snapshotId, Duration timeRetained) { + table.replaceTag(tagName, snapshotId, timeRetained); + } + + public static ProcedureBuilder builder() { + return new BaseProcedure.Builder() { + @Override + public ReplaceTagProcedure doBuild() { + return new ReplaceTagProcedure(tableCatalog()); + } + }; + } + + @Override + public String description() { + return "ReplaceTagProcedure"; + } +} diff --git a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/ResetConsumerProcedure.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/ResetConsumerProcedure.java index a13227e95dc7..0f7fabd05d13 100644 --- a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/ResetConsumerProcedure.java +++ b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/ResetConsumerProcedure.java @@ -90,6 +90,7 @@ public InternalRow[] call(InternalRow args) { if (nextSnapshotId == null) { consumerManager.deleteConsumer(consumerId); } else { + fileStoreTable.snapshotManager().snapshot(nextSnapshotId); consumerManager.resetConsumer(consumerId, new Consumer(nextSnapshotId)); } diff --git a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/RollbackToTimestampProcedure.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/RollbackToTimestampProcedure.java new file mode 100644 index 000000000000..a01f08b3fc7d --- /dev/null +++ b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/RollbackToTimestampProcedure.java @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.procedure; + +import org.apache.paimon.Snapshot; +import org.apache.paimon.table.FileStoreTable; +import org.apache.paimon.utils.Preconditions; + +import org.apache.spark.sql.catalyst.InternalRow; +import org.apache.spark.sql.connector.catalog.Identifier; +import org.apache.spark.sql.connector.catalog.TableCatalog; +import org.apache.spark.sql.types.Metadata; +import org.apache.spark.sql.types.StructField; +import org.apache.spark.sql.types.StructType; +import org.apache.spark.unsafe.types.UTF8String; + +import static org.apache.spark.sql.types.DataTypes.LongType; +import static org.apache.spark.sql.types.DataTypes.StringType; + +/** A procedure to rollback to a timestamp. */ +public class RollbackToTimestampProcedure extends BaseProcedure { + + private static final ProcedureParameter[] PARAMETERS = + new ProcedureParameter[] { + ProcedureParameter.required("table", StringType), + // timestamp value + ProcedureParameter.required("timestamp", LongType) + }; + + private static final StructType OUTPUT_TYPE = + new StructType( + new StructField[] { + new StructField("result", StringType, true, Metadata.empty()) + }); + + private RollbackToTimestampProcedure(TableCatalog tableCatalog) { + super(tableCatalog); + } + + @Override + public ProcedureParameter[] parameters() { + return PARAMETERS; + } + + @Override + public StructType outputType() { + return OUTPUT_TYPE; + } + + @Override + public InternalRow[] call(InternalRow args) { + Identifier tableIdent = toIdentifier(args.getString(0), PARAMETERS[0].name()); + Long timestamp = args.getLong(1); + + return modifyPaimonTable( + tableIdent, + table -> { + FileStoreTable fileStoreTable = (FileStoreTable) table; + Snapshot snapshot = + fileStoreTable.snapshotManager().earlierOrEqualTimeMills(timestamp); + Preconditions.checkNotNull( + snapshot, + String.format("count not find snapshot earlier than %s", timestamp)); + long snapshotId = snapshot.id(); + fileStoreTable.rollbackTo(snapshotId); + InternalRow outputRow = + newInternalRow( + UTF8String.fromString( + String.format( + "Success roll back to snapshot: %s .", + snapshotId))); + return new InternalRow[] {outputRow}; + }); + } + + public static ProcedureBuilder builder() { + return new BaseProcedure.Builder() { + @Override + public RollbackToTimestampProcedure doBuild() { + return new RollbackToTimestampProcedure(tableCatalog()); + } + }; + } + + @Override + public String description() { + return "RollbackToTimestampProcedure"; + } +} diff --git a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/RollbackToWatermarkProcedure.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/RollbackToWatermarkProcedure.java new file mode 100644 index 000000000000..09185f02c919 --- /dev/null +++ b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/RollbackToWatermarkProcedure.java @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.procedure; + +import org.apache.paimon.Snapshot; +import org.apache.paimon.table.FileStoreTable; +import org.apache.paimon.utils.Preconditions; + +import org.apache.spark.sql.catalyst.InternalRow; +import org.apache.spark.sql.connector.catalog.Identifier; +import org.apache.spark.sql.connector.catalog.TableCatalog; +import org.apache.spark.sql.types.Metadata; +import org.apache.spark.sql.types.StructField; +import org.apache.spark.sql.types.StructType; +import org.apache.spark.unsafe.types.UTF8String; + +import static org.apache.spark.sql.types.DataTypes.LongType; +import static org.apache.spark.sql.types.DataTypes.StringType; + +/** A procedure to rollback to a watermark. */ +public class RollbackToWatermarkProcedure extends BaseProcedure { + + private static final ProcedureParameter[] PARAMETERS = + new ProcedureParameter[] { + ProcedureParameter.required("table", StringType), + // watermark value + ProcedureParameter.required("watermark", LongType) + }; + + private static final StructType OUTPUT_TYPE = + new StructType( + new StructField[] { + new StructField("result", StringType, true, Metadata.empty()) + }); + + private RollbackToWatermarkProcedure(TableCatalog tableCatalog) { + super(tableCatalog); + } + + @Override + public ProcedureParameter[] parameters() { + return PARAMETERS; + } + + @Override + public StructType outputType() { + return OUTPUT_TYPE; + } + + @Override + public InternalRow[] call(InternalRow args) { + Identifier tableIdent = toIdentifier(args.getString(0), PARAMETERS[0].name()); + Long watermark = args.getLong(1); + + return modifyPaimonTable( + tableIdent, + table -> { + FileStoreTable fileStoreTable = (FileStoreTable) table; + Snapshot snapshot = + fileStoreTable.snapshotManager().earlierOrEqualWatermark(watermark); + Preconditions.checkNotNull( + snapshot, + String.format("count not find snapshot earlier than %s", watermark)); + long snapshotId = snapshot.id(); + fileStoreTable.rollbackTo(snapshotId); + InternalRow outputRow = + newInternalRow( + UTF8String.fromString( + String.format( + "Success roll back to snapshot: %s .", + snapshotId))); + return new InternalRow[] {outputRow}; + }); + } + + public static ProcedureBuilder builder() { + return new BaseProcedure.Builder() { + @Override + public RollbackToWatermarkProcedure doBuild() { + return new RollbackToWatermarkProcedure(tableCatalog()); + } + }; + } + + @Override + public String description() { + return "RollbackToWatermarkProcedure"; + } +} diff --git a/paimon-hive/paimon-hive-connector-3.1/src/main/java/org/apache/paimon/hive/LocalZonedTimestampTypeUtils.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/utils/CatalogUtils.java similarity index 53% rename from paimon-hive/paimon-hive-connector-3.1/src/main/java/org/apache/paimon/hive/LocalZonedTimestampTypeUtils.java rename to paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/utils/CatalogUtils.java index b7c5d26ae657..fca9df210e70 100644 --- a/paimon-hive/paimon-hive-connector-3.1/src/main/java/org/apache/paimon/hive/LocalZonedTimestampTypeUtils.java +++ b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/utils/CatalogUtils.java @@ -16,25 +16,26 @@ * limitations under the License. */ -package org.apache.paimon.hive; +package org.apache.paimon.spark.utils; -import org.apache.paimon.types.LocalZonedTimestampType; +import org.apache.spark.sql.connector.catalog.Identifier; -import org.apache.hadoop.hive.serde2.typeinfo.TimestampLocalTZTypeInfo; -import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; -import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; +import java.util.Arrays; -/** - * Utils to convert between Hive {@link TimestampLocalTZTypeInfo} and Paimon {@link - * LocalZonedTimestampType}. - */ -public class LocalZonedTimestampTypeUtils { +import static org.apache.paimon.utils.Preconditions.checkArgument; + +/** Utils of catalog. */ +public class CatalogUtils { - public static boolean isLocalZonedTimestampType(TypeInfo hiveTypeInfo) { - return hiveTypeInfo instanceof TimestampLocalTZTypeInfo; + public static void checkNamespace(String[] namespace) { + checkArgument( + namespace.length == 1, + "Paimon only support single namespace, but got %s", + Arrays.toString(namespace)); } - public static TypeInfo toHiveType(LocalZonedTimestampType paimonType) { - return TypeInfoFactory.timestampLocalTZTypeInfo; + public static org.apache.paimon.catalog.Identifier toIdentifier(Identifier ident) { + checkNamespace(ident.namespace()); + return new org.apache.paimon.catalog.Identifier(ident.namespace()[0], ident.name()); } } diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/ColumnPruningAndPushDown.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/ColumnPruningAndPushDown.scala index 95c8f4b3a9a8..f29c146b775a 100644 --- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/ColumnPruningAndPushDown.scala +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/ColumnPruningAndPushDown.scala @@ -62,7 +62,7 @@ trait ColumnPruningAndPushDown extends Scan with Logging { _readBuilder.withFilter(pushedPredicate) } pushDownLimit.foreach(_readBuilder.withLimit) - _readBuilder + _readBuilder.dropStats() } final def metadataColumns: Seq[PaimonMetadataColumn] = { diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonPartitionManagement.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonPartitionManagement.scala index 113ad4c3b7a8..840f1341a69d 100644 --- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonPartitionManagement.scala +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonPartitionManagement.scala @@ -18,7 +18,10 @@ package org.apache.paimon.spark +import org.apache.paimon.CoreOptions +import org.apache.paimon.metastore.MetastoreClient import org.apache.paimon.operation.FileStoreCommit +import org.apache.paimon.spark.data.SparkInternalRow import org.apache.paimon.table.FileStoreTable import org.apache.paimon.table.sink.BatchWriteBuilder import org.apache.paimon.types.RowType @@ -41,7 +44,8 @@ trait PaimonPartitionManagement extends SupportsAtomicPartitionManagement { override lazy val partitionSchema: StructType = SparkTypeUtils.fromPaimonRowType(partitionRowType) - override def dropPartitions(internalRows: Array[InternalRow]): Boolean = { + private def toPaimonPartitions( + rows: Array[InternalRow]): Array[java.util.LinkedHashMap[String, String]] = { table match { case fileStoreTable: FileStoreTable => val rowConverter = CatalystTypeConverters @@ -49,19 +53,38 @@ trait PaimonPartitionManagement extends SupportsAtomicPartitionManagement { val rowDataPartitionComputer = new InternalRowPartitionComputer( fileStoreTable.coreOptions().partitionDefaultName(), partitionRowType, - table.partitionKeys().asScala.toArray) + table.partitionKeys().asScala.toArray, + CoreOptions.fromMap(table.options()).legacyPartitionName) - val partitions = internalRows.map { + rows.map { r => rowDataPartitionComputer .generatePartValues(new SparkRow(partitionRowType, rowConverter(r).asInstanceOf[Row])) - .asInstanceOf[JMap[String, String]] } + case _ => + throw new UnsupportedOperationException("Only FileStoreTable supports partitions.") + } + } + + override def dropPartitions(rows: Array[InternalRow]): Boolean = { + table match { + case fileStoreTable: FileStoreTable => + val partitions = toPaimonPartitions(rows).map(_.asInstanceOf[JMap[String, String]]) val commit: FileStoreCommit = fileStoreTable.store.newCommit(UUID.randomUUID.toString) + var metastoreClient: MetastoreClient = null + val clientFactory = fileStoreTable.catalogEnvironment().metastoreClientFactory try { commit.dropPartitions(partitions.toSeq.asJava, BatchWriteBuilder.COMMIT_IDENTIFIER) + // sync to metastore with delete partitions + if (clientFactory != null && fileStoreTable.coreOptions().partitionedTableInMetastore()) { + metastoreClient = clientFactory.create() + toPaimonPartitions(rows).foreach(metastoreClient.deletePartition) + } } finally { commit.close() + if (metastoreClient != null) { + metastoreClient.close() + } } true @@ -77,7 +100,7 @@ trait PaimonPartitionManagement extends SupportsAtomicPartitionManagement { } override def loadPartitionMetadata(ident: InternalRow): JMap[String, String] = { - throw new UnsupportedOperationException("Load partition is not supported") + Map.empty[String, String].asJava } override def listPartitionIdentifiers( @@ -94,7 +117,7 @@ trait PaimonPartitionManagement extends SupportsAtomicPartitionManagement { s"the partition schema '${partitionSchema.sql}'." ) table.newReadBuilder.newScan.listPartitions.asScala - .map(binaryRow => SparkInternalRow.fromPaimon(binaryRow, partitionRowType)) + .map(binaryRow => DataConverter.fromPaimon(binaryRow, partitionRowType)) .filter( sparkInternalRow => { partitionCols.zipWithIndex @@ -112,8 +135,26 @@ trait PaimonPartitionManagement extends SupportsAtomicPartitionManagement { } override def createPartitions( - internalRows: Array[InternalRow], + rows: Array[InternalRow], maps: Array[JMap[String, String]]): Unit = { - throw new UnsupportedOperationException("Create partition is not supported") + table match { + case fileStoreTable: FileStoreTable => + val partitions = toPaimonPartitions(rows) + val metastoreFactory = fileStoreTable.catalogEnvironment().metastoreClientFactory() + if (metastoreFactory == null) { + throw new UnsupportedOperationException( + "The table must have metastore to create partition.") + } + val metastoreClient: MetastoreClient = metastoreFactory.create + try { + if (fileStoreTable.coreOptions().partitionedTableInMetastore()) { + partitions.foreach(metastoreClient.addPartition) + } + } finally { + metastoreClient.close() + } + case _ => + throw new UnsupportedOperationException("Only FileStoreTable supports create partitions.") + } } } diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonPartitionReader.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonPartitionReader.scala index fa9072df3149..526178e28ec3 100644 --- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonPartitionReader.scala +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonPartitionReader.scala @@ -20,6 +20,7 @@ package org.apache.paimon.spark import org.apache.paimon.data.{InternalRow => PaimonInternalRow} import org.apache.paimon.reader.RecordReader +import org.apache.paimon.spark.data.SparkInternalRow import org.apache.paimon.spark.schema.PaimonMetadataColumn import org.apache.paimon.table.source.{DataSplit, Split} diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonPartitionReaderFactory.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonPartitionReaderFactory.scala index 94de0bec3b50..59b07a794481 100644 --- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonPartitionReaderFactory.scala +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonPartitionReaderFactory.scala @@ -18,10 +18,11 @@ package org.apache.paimon.spark -import org.apache.paimon.data +import org.apache.paimon.data.{InternalRow => PaimonInternalRow} import org.apache.paimon.disk.IOManager import org.apache.paimon.reader.RecordReader import org.apache.paimon.spark.SparkUtils.createIOManager +import org.apache.paimon.spark.data.SparkInternalRow import org.apache.paimon.spark.schema.PaimonMetadataColumn import org.apache.paimon.table.source.{ReadBuilder, Split} import org.apache.paimon.types.RowType @@ -45,13 +46,13 @@ case class PaimonPartitionReaderFactory( val dataFields = new JList(readBuilder.readType().getFields) dataFields.addAll(metadataColumns.map(_.toPaimonDataField).asJava) val rowType = new RowType(dataFields) - new SparkInternalRow(rowType) + SparkInternalRow.create(rowType) } override def createReader(partition: InputPartition): PartitionReader[InternalRow] = { partition match { case paimonInputPartition: PaimonInputPartition => - val readFunc: Split => RecordReader[data.InternalRow] = + val readFunc: Split => RecordReader[PaimonInternalRow] = (split: Split) => readBuilder.newRead().withIOManager(ioManager).createReader(split) PaimonPartitionReader(readFunc, paimonInputPartition, row, metadataColumns) case _ => diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonScanBuilder.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonScanBuilder.scala index 8f4bc27bf3f9..0393a1cd1578 100644 --- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonScanBuilder.scala +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonScanBuilder.scala @@ -21,6 +21,7 @@ package org.apache.paimon.spark import org.apache.paimon.predicate.PredicateBuilder import org.apache.paimon.spark.aggregate.LocalAggregator import org.apache.paimon.table.Table +import org.apache.paimon.table.source.DataSplit import org.apache.spark.sql.connector.expressions.aggregate.Aggregation import org.apache.spark.sql.connector.read.{Scan, SupportsPushDownAggregates, SupportsPushDownLimit} @@ -34,15 +35,14 @@ class PaimonScanBuilder(table: Table) private var localScan: Option[Scan] = None override def pushLimit(limit: Int): Boolean = { - if (table.primaryKeys().isEmpty) { - pushDownLimit = Some(limit) - } - // just make a best effort to push down limit + // It is safe, since we will do nothing if it is the primary table and the split is not `rawConvertible` + pushDownLimit = Some(limit) + // just make the best effort to push down limit false } override def supportCompletePushDown(aggregation: Aggregation): Boolean = { - // for now we only support complete push down, so there is no difference with `pushAggregation` + // for now, we only support complete push down, so there is no difference with `pushAggregation` pushAggregation(aggregation) } @@ -67,8 +67,11 @@ class PaimonScanBuilder(table: Table) val pushedPartitionPredicate = PredicateBuilder.and(pushedPredicates.map(_._2): _*) readBuilder.withFilter(pushedPartitionPredicate) } - val scan = readBuilder.newScan() - scan.listPartitionEntries.asScala.foreach(aggregator.update) + val dataSplits = readBuilder.newScan().plan().splits().asScala.map(_.asInstanceOf[DataSplit]) + if (!dataSplits.forall(_.mergedRowCountAvailable())) { + return false + } + dataSplits.foreach(aggregator.update) localScan = Some( PaimonLocalScan( aggregator.result(), diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonStatistics.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonStatistics.scala index 15a62f266c56..8dd464933032 100644 --- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonStatistics.scala +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonStatistics.scala @@ -18,6 +18,7 @@ package org.apache.paimon.spark +import org.apache.paimon.spark.data.SparkInternalRow import org.apache.paimon.stats.ColStats import org.apache.paimon.types.{DataField, DataType, RowType} @@ -118,8 +119,10 @@ object PaimonColumnStats { def apply(dateType: DataType, paimonColStats: ColStats[_]): PaimonColumnStats = { PaimonColumnStats( paimonColStats.nullCount, - Optional.ofNullable(SparkInternalRow.fromPaimon(paimonColStats.min().orElse(null), dateType)), - Optional.ofNullable(SparkInternalRow.fromPaimon(paimonColStats.max().orElse(null), dateType)), + Optional.ofNullable( + DataConverter + .fromPaimon(paimonColStats.min().orElse(null), dateType)), + Optional.ofNullable(DataConverter.fromPaimon(paimonColStats.max().orElse(null), dateType)), paimonColStats.distinctCount, paimonColStats.avgLen, paimonColStats.maxLen @@ -129,12 +132,12 @@ object PaimonColumnStats { def apply(v1ColStats: ColumnStat): PaimonColumnStats = { import PaimonImplicits._ PaimonColumnStats( - if (v1ColStats.nullCount.isDefined) OptionalLong.of(v1ColStats.nullCount.get.longValue()) + if (v1ColStats.nullCount.isDefined) OptionalLong.of(v1ColStats.nullCount.get.longValue) else OptionalLong.empty(), v1ColStats.min, v1ColStats.max, if (v1ColStats.distinctCount.isDefined) - OptionalLong.of(v1ColStats.distinctCount.get.longValue()) + OptionalLong.of(v1ColStats.distinctCount.get.longValue) else OptionalLong.empty(), if (v1ColStats.avgLen.isDefined) OptionalLong.of(v1ColStats.avgLen.get.longValue()) else OptionalLong.empty(), diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/SaveMode.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/SaveMode.scala index c4dae5e3ddd8..bcd6f68ab1a1 100644 --- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/SaveMode.scala +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/SaveMode.scala @@ -23,15 +23,15 @@ import org.apache.spark.sql.sources.{AlwaysTrue, Filter} sealed private[spark] trait SaveMode extends Serializable -object InsertInto extends SaveMode +case object InsertInto extends SaveMode case class Overwrite(filters: Option[Filter]) extends SaveMode -object DynamicOverWrite extends SaveMode +case object DynamicOverWrite extends SaveMode -object ErrorIfExists extends SaveMode +case object ErrorIfExists extends SaveMode -object Ignore extends SaveMode +case object Ignore extends SaveMode object SaveMode { def transform(saveMode: SparkSaveMode): SaveMode = { diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/ScanHelper.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/ScanHelper.scala index a5d4f10bac33..b5b56ba1d509 100644 --- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/ScanHelper.scala +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/ScanHelper.scala @@ -76,7 +76,7 @@ trait ScanHelper extends Logging { def closeDataSplit(): Unit = { if (currentSplit.nonEmpty && currentDataFiles.nonEmpty) { val newSplit = - copyDataSplit(currentSplit.get, currentDataFiles, currentDeletionFiles) + copyDataSplit(currentSplit.get, currentDataFiles.toSeq, currentDeletionFiles.toSeq) currentSplits += newSplit } currentDataFiles.clear() diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/SparkSource.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/SparkSource.scala index 67ab1312fa4e..d80d7350a655 100644 --- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/SparkSource.scala +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/SparkSource.scala @@ -18,11 +18,13 @@ package org.apache.paimon.spark -import org.apache.paimon.catalog.CatalogContext +import org.apache.paimon.CoreOptions +import org.apache.paimon.catalog.{CatalogContext, CatalogUtils, Identifier} import org.apache.paimon.options.Options +import org.apache.paimon.spark.SparkSource.NAME import org.apache.paimon.spark.commands.WriteIntoPaimonTable import org.apache.paimon.spark.sources.PaimonSink -import org.apache.paimon.spark.util.OptionUtils.mergeSQLConf +import org.apache.paimon.spark.util.OptionUtils.{extractCatalogName, mergeSQLConfWithIdentifier} import org.apache.paimon.table.{DataTable, FileStoreTable, FileStoreTableFactory} import org.apache.paimon.table.system.AuditLogTable @@ -80,9 +82,15 @@ class SparkSource } private def loadTable(options: JMap[String, String]): DataTable = { + val path = CoreOptions.path(options) val catalogContext = CatalogContext.create( - Options.fromMap(mergeSQLConf(options)), - SparkSession.active.sessionState.newHadoopConf()) + Options.fromMap( + mergeSQLConfWithIdentifier( + options, + extractCatalogName().getOrElse(NAME), + Identifier.create(CatalogUtils.database(path), CatalogUtils.table(path)))), + SparkSession.active.sessionState.newHadoopConf() + ) val table = FileStoreTableFactory.create(catalogContext) if (Options.fromMap(options).get(SparkConnectorOptions.READ_CHANGELOG)) { new AuditLogTable(table) @@ -110,7 +118,7 @@ object SparkSource { val NAME = "paimon" - val FORMAT_NAMES = Seq("csv", "orc", "parquet") + val FORMAT_NAMES: Seq[String] = Seq("csv", "orc", "parquet") def toBaseRelation(table: FileStoreTable, _sqlContext: SQLContext): BaseRelation = { new BaseRelation { diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/SparkTable.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/SparkTable.scala index e45d15aa9017..b9a90d8b5bef 100644 --- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/SparkTable.scala +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/SparkTable.scala @@ -64,6 +64,9 @@ case class SparkTable(table: Table) if (table.comment.isPresent) { properties.put(TableCatalog.PROP_COMMENT, table.comment.get) } + if (properties.containsKey(CoreOptions.PATH.key())) { + properties.put(TableCatalog.PROP_LOCATION, properties.get(CoreOptions.PATH.key())) + } properties case _ => Collections.emptyMap() } @@ -107,4 +110,8 @@ case class SparkTable(table: Table) throw new RuntimeException("Only FileStoreTable can be written.") } } + + override def toString: String = { + s"${table.getClass.getSimpleName}[${table.fullName()}]" + } } diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/SparkWrite.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/SparkWrite.scala index 7e7919592eb7..fd43decfc6ad 100644 --- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/SparkWrite.scala +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/SparkWrite.scala @@ -35,4 +35,8 @@ class SparkWrite(val table: FileStoreTable, saveMode: SaveMode, options: Options WriteIntoPaimonTable(table, saveMode, data, options).run(data.sparkSession) } } + + override def toString: String = { + s"table: ${table.fullName()}, saveMode: $saveMode, options: ${options.toMap}" + } } diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/aggregate/LocalAggregator.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/aggregate/LocalAggregator.scala index cd9718cf44eb..8988e7218d1f 100644 --- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/aggregate/LocalAggregator.scala +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/aggregate/LocalAggregator.scala @@ -19,9 +19,10 @@ package org.apache.paimon.spark.aggregate import org.apache.paimon.data.BinaryRow -import org.apache.paimon.manifest.PartitionEntry -import org.apache.paimon.spark.{SparkInternalRow, SparkTypeUtils} +import org.apache.paimon.spark.SparkTypeUtils +import org.apache.paimon.spark.data.SparkInternalRow import org.apache.paimon.table.{DataTable, Table} +import org.apache.paimon.table.source.DataSplit import org.apache.paimon.utils.{InternalRowUtils, ProjectedRow} import org.apache.spark.sql.catalyst.InternalRow @@ -77,13 +78,7 @@ class LocalAggregator(table: Table) { } def pushAggregation(aggregation: Aggregation): Boolean = { - if ( - !table.isInstanceOf[DataTable] || - !table.primaryKeys.isEmpty - ) { - return false - } - if (table.asInstanceOf[DataTable].coreOptions.deletionVectorsEnabled) { + if (!table.isInstanceOf[DataTable]) { return false } @@ -104,15 +99,15 @@ class LocalAggregator(table: Table) { ProjectedRow.from(requiredGroupByIndexMapping.toArray).replaceRow(partitionRow) // `ProjectedRow` does not support `hashCode`, so do a deep copy val genericRow = InternalRowUtils.copyInternalRow(projectedRow, partitionType) - new SparkInternalRow(partitionType).replace(genericRow) + SparkInternalRow.create(partitionType).replace(genericRow) } - def update(partitionEntry: PartitionEntry): Unit = { + def update(dataSplit: DataSplit): Unit = { assert(isInitialized) - val groupByRow = requiredGroupByRow(partitionEntry.partition()) + val groupByRow = requiredGroupByRow(dataSplit.partition()) val aggFuncEvaluator = groupByEvaluatorMap.getOrElseUpdate(groupByRow, aggFuncEvaluatorGetter()) - aggFuncEvaluator.foreach(_.update(partitionEntry)) + aggFuncEvaluator.foreach(_.update(dataSplit)) } def result(): Array[InternalRow] = { @@ -146,7 +141,7 @@ class LocalAggregator(table: Table) { } trait AggFuncEvaluator[T] { - def update(partitionEntry: PartitionEntry): Unit + def update(dataSplit: DataSplit): Unit def result(): T def resultType: DataType def prettyName: String @@ -155,8 +150,8 @@ trait AggFuncEvaluator[T] { class CountStarEvaluator extends AggFuncEvaluator[Long] { private var _result: Long = 0L - override def update(partitionEntry: PartitionEntry): Unit = { - _result += partitionEntry.recordCount() + override def update(dataSplit: DataSplit): Unit = { + _result += dataSplit.mergedRowCount() } override def result(): Long = _result diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/analysis/PaimonAnalysis.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/analysis/PaimonAnalysis.scala index 98d3c03aacbb..f567d925ea57 100644 --- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/analysis/PaimonAnalysis.scala +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/analysis/PaimonAnalysis.scala @@ -148,7 +148,8 @@ class PaimonAnalysis(session: SparkSession) extends Rule[LogicalPlan] { case (s1: StructType, s2: StructType) => s1.zip(s2).forall { case (d1, d2) => schemaCompatible(d1.dataType, d2.dataType) } case (a1: ArrayType, a2: ArrayType) => - a1.containsNull == a2.containsNull && schemaCompatible(a1.elementType, a2.elementType) + // todo: support array type nullable evaluation + schemaCompatible(a1.elementType, a2.elementType) case (m1: MapType, m2: MapType) => m1.valueContainsNull == m2.valueContainsNull && schemaCompatible(m1.keyType, m2.keyType) && diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/analysis/PaimonMergeIntoBase.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/analysis/PaimonMergeIntoBase.scala index ba6108395a7c..f2530b50c04c 100644 --- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/analysis/PaimonMergeIntoBase.scala +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/analysis/PaimonMergeIntoBase.scala @@ -53,7 +53,7 @@ trait PaimonMergeIntoBase merge.notMatchedActions.flatMap(_.condition).foreach(checkCondition) val updateActions = merge.matchedActions.collect { case a: UpdateAction => a } - val primaryKeys = v2Table.getTable.primaryKeys().asScala + val primaryKeys = v2Table.getTable.primaryKeys().asScala.toSeq if (primaryKeys.nonEmpty) { checkUpdateActionValidity( AttributeSet(targetOutput), diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/analysis/PaimonResolvePartitionSpec.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/analysis/PaimonResolvePartitionSpec.scala new file mode 100644 index 000000000000..5d6a5a063c06 --- /dev/null +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/analysis/PaimonResolvePartitionSpec.scala @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.catalyst.analysis + +import org.apache.spark.sql.PaimonUtils.{normalizePartitionSpec, requireExactMatchedPartitionSpec} +import org.apache.spark.sql.catalyst.InternalRow +import org.apache.spark.sql.catalyst.analysis.{PartitionSpec, ResolvedPartitionSpec, UnresolvedPartitionSpec} +import org.apache.spark.sql.catalyst.analysis.ResolvePartitionSpec.conf +import org.apache.spark.sql.catalyst.catalog.CatalogTypes.TablePartitionSpec +import org.apache.spark.sql.catalyst.expressions.{Cast, Literal} +import org.apache.spark.sql.catalyst.util.CharVarcharUtils +import org.apache.spark.sql.connector.catalog.{Identifier, TableCatalog} +import org.apache.spark.sql.execution.datasources.v2.DataSourceV2Implicits._ +import org.apache.spark.sql.types.{StringType, StructField, StructType} + +object PaimonResolvePartitionSpec { + + def resolve( + catalog: TableCatalog, + tableIndent: Identifier, + partitionSpec: PartitionSpec): ResolvedPartitionSpec = { + val table = catalog.loadTable(tableIndent).asPartitionable + partitionSpec match { + case u: UnresolvedPartitionSpec => + val partitionSchema = table.partitionSchema() + resolvePartitionSpec(table.name(), u, partitionSchema, allowPartitionSpec = false) + case o => o.asInstanceOf[ResolvedPartitionSpec] + } + } + + private def resolvePartitionSpec( + tableName: String, + partSpec: UnresolvedPartitionSpec, + partSchema: StructType, + allowPartitionSpec: Boolean): ResolvedPartitionSpec = { + val normalizedSpec = normalizePartitionSpec(partSpec.spec, partSchema, tableName, conf.resolver) + if (!allowPartitionSpec) { + requireExactMatchedPartitionSpec(tableName, normalizedSpec, partSchema.fieldNames) + } + val partitionNames = normalizedSpec.keySet + val requestedFields = partSchema.filter(field => partitionNames.contains(field.name)) + ResolvedPartitionSpec( + requestedFields.map(_.name), + convertToPartIdent(normalizedSpec, requestedFields), + partSpec.location) + } + + def convertToPartIdent( + partitionSpec: TablePartitionSpec, + schema: Seq[StructField]): InternalRow = { + val partValues = schema.map { + part => + val raw = partitionSpec.get(part.name).orNull + val dt = CharVarcharUtils.replaceCharVarcharWithString(part.dataType) + Cast(Literal.create(raw, StringType), dt, Some(conf.sessionLocalTimeZone)).eval() + } + InternalRow.fromSeq(partValues) + } +} diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/analysis/PaimonUpdateTable.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/analysis/PaimonUpdateTable.scala index 123c67a2fc20..ad3912ddb70d 100644 --- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/analysis/PaimonUpdateTable.scala +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/analysis/PaimonUpdateTable.scala @@ -41,7 +41,7 @@ object PaimonUpdateTable table.getTable match { case paimonTable: FileStoreTable => - val primaryKeys = paimonTable.primaryKeys().asScala + val primaryKeys = paimonTable.primaryKeys().asScala.toSeq if (!validUpdateAssignment(u.table.outputSet, primaryKeys, assignments)) { throw new RuntimeException("Can't update the primary key column.") } diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/analysis/PaimonViewResolver.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/analysis/PaimonViewResolver.scala new file mode 100644 index 000000000000..3da0ddab6417 --- /dev/null +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/analysis/PaimonViewResolver.scala @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.catalyst.analysis + +import org.apache.paimon.catalog.Catalog.ViewNotExistException +import org.apache.paimon.spark.SparkTypeUtils +import org.apache.paimon.spark.catalog.SupportView +import org.apache.paimon.view.View + +import org.apache.spark.sql.SparkSession +import org.apache.spark.sql.catalyst.analysis.{GetColumnByOrdinal, UnresolvedRelation, UnresolvedTableOrView} +import org.apache.spark.sql.catalyst.expressions.{Alias, Attribute, UpCast} +import org.apache.spark.sql.catalyst.parser.ParseException +import org.apache.spark.sql.catalyst.parser.extensions.{CurrentOrigin, Origin} +import org.apache.spark.sql.catalyst.plans.logical.{LeafNode, LogicalPlan, Project, SubqueryAlias} +import org.apache.spark.sql.catalyst.rules.Rule +import org.apache.spark.sql.connector.catalog.{Identifier, PaimonLookupCatalog} + +case class PaimonViewResolver(spark: SparkSession) + extends Rule[LogicalPlan] + with PaimonLookupCatalog { + + protected lazy val catalogManager = spark.sessionState.catalogManager + + override def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperators { + case u @ UnresolvedRelation(parts @ CatalogAndIdentifier(catalog: SupportView, ident), _, _) => + try { + val view = catalog.loadView(ident) + createViewRelation(parts, view) + } catch { + case _: ViewNotExistException => + u + } + + case u @ UnresolvedTableOrView(CatalogAndIdentifier(catalog: SupportView, ident), _, _) => + try { + catalog.loadView(ident) + ResolvedPaimonView(catalog, ident) + } catch { + case _: ViewNotExistException => + u + } + } + + private def createViewRelation(nameParts: Seq[String], view: View): LogicalPlan = { + val parsedPlan = parseViewText(nameParts.toArray.mkString("."), view.query) + + val aliases = SparkTypeUtils.fromPaimonRowType(view.rowType()).fields.zipWithIndex.map { + case (expected, pos) => + val attr = GetColumnByOrdinal(pos, expected.dataType) + Alias(UpCast(attr, expected.dataType), expected.name)(explicitMetadata = + Some(expected.metadata)) + } + + SubqueryAlias(nameParts, Project(aliases, parsedPlan)) + } + + private def parseViewText(name: String, viewText: String): LogicalPlan = { + val origin = Origin( + objectType = Some("VIEW"), + objectName = Some(name) + ) + try { + CurrentOrigin.withOrigin(origin) { + try { + spark.sessionState.sqlParser.parseQuery(viewText) + } catch { + // For compatibility with Spark 3.2 and below + case _: NoSuchMethodError => + spark.sessionState.sqlParser.parsePlan(viewText) + } + } + } catch { + case _: ParseException => + throw new RuntimeException("Failed to parse view text: " + viewText) + } + } +} + +case class ResolvedPaimonView(catalog: SupportView, identifier: Identifier) extends LeafNode { + override def output: Seq[Attribute] = Nil +} diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/analysis/expressions/ExpressionHelper.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/analysis/expressions/ExpressionHelper.scala index c008819fb0cc..d4010ea33811 100644 --- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/analysis/expressions/ExpressionHelper.scala +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/analysis/expressions/ExpressionHelper.scala @@ -23,12 +23,13 @@ import org.apache.paimon.spark.SparkFilterConverter import org.apache.paimon.spark.catalyst.Compatibility import org.apache.paimon.types.RowType +import org.apache.spark.sql.{Column, SparkSession} import org.apache.spark.sql.PaimonUtils.{normalizeExprs, translateFilter} -import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.analysis.Resolver import org.apache.spark.sql.catalyst.expressions.{Alias, And, Attribute, Cast, Expression, GetStructField, Literal, PredicateHelper, SubqueryExpression} import org.apache.spark.sql.catalyst.plans.logical.{Filter, LogicalPlan} import org.apache.spark.sql.internal.SQLConf +import org.apache.spark.sql.paimon.shims.SparkShimLoader import org.apache.spark.sql.types.{DataType, NullType} /** An expression helper. */ @@ -36,6 +37,14 @@ trait ExpressionHelper extends PredicateHelper { import ExpressionHelper._ + def toColumn(expr: Expression): Column = { + SparkShimLoader.getSparkShim.column(expr) + } + + def toExpression(spark: SparkSession, col: Column): Expression = { + SparkShimLoader.getSparkShim.convertToExpression(spark, col) + } + protected def resolveExpression( spark: SparkSession)(expr: Expression, plan: LogicalPlan): Expression = { if (expr.resolved) { diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/optimizer/EvalSubqueriesForDeleteTable.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/optimizer/EvalSubqueriesForDeleteTable.scala index 5d264370adcd..4cf9284f97f6 100644 --- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/optimizer/EvalSubqueriesForDeleteTable.scala +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/optimizer/EvalSubqueriesForDeleteTable.scala @@ -49,7 +49,10 @@ object EvalSubqueriesForDeleteTable extends Rule[LogicalPlan] with ExpressionHel plan.transformDown { case d @ DeleteFromPaimonTableCommand(_, table, condition) if SubqueryExpression.hasSubquery(condition) && - isPredicatePartitionColumnsOnly(condition, table.partitionKeys().asScala, resolver) => + isPredicatePartitionColumnsOnly( + condition, + table.partitionKeys().asScala.toSeq, + resolver) => try { d.copy(condition = evalSubquery(condition)) } catch { diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/optimizer/MergePaimonScalarSubqueriesBase.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/optimizer/MergePaimonScalarSubqueriesBase.scala index eca8c9cdfced..3428ed89f004 100644 --- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/optimizer/MergePaimonScalarSubqueriesBase.scala +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/optimizer/MergePaimonScalarSubqueriesBase.scala @@ -28,6 +28,7 @@ import org.apache.spark.sql.catalyst.rules.Rule import org.apache.spark.sql.catalyst.trees.TreePattern.{SCALAR_SUBQUERY, SCALAR_SUBQUERY_REFERENCE, TreePattern} import org.apache.spark.sql.execution.datasources.v2.DataSourceV2ScanRelation import org.apache.spark.sql.internal.SQLConf +import org.apache.spark.sql.paimon.shims.SparkShimLoader import org.apache.spark.sql.types.{DataType, StructType} import scala.collection.mutable.ArrayBuffer @@ -95,7 +96,7 @@ trait MergePaimonScalarSubqueriesBase extends Rule[LogicalPlan] with PredicateHe val newPlan = removeReferences(planWithReferences, cache) val subqueryCTEs = cache.filter(_.merged).map(_.plan.asInstanceOf[CTERelationDef]) if (subqueryCTEs.nonEmpty) { - WithCTE(newPlan, subqueryCTEs) + WithCTE(newPlan, subqueryCTEs.toSeq) } else { newPlan } @@ -335,22 +336,24 @@ trait MergePaimonScalarSubqueriesBase extends Rule[LogicalPlan] with PredicateHe // Only allow aggregates of the same implementation because merging different implementations // could cause performance regression. private def supportedAggregateMerge(newPlan: Aggregate, cachedPlan: Aggregate) = { - val newPlanAggregateExpressions = newPlan.aggregateExpressions.flatMap(_.collect { - case a: AggregateExpression => a - }) - val cachedPlanAggregateExpressions = cachedPlan.aggregateExpressions.flatMap(_.collect { - case a: AggregateExpression => a - }) - val newPlanSupportsHashAggregate = Aggregate.supportsHashAggregate( - newPlanAggregateExpressions.flatMap(_.aggregateFunction.aggBufferAttributes)) - val cachedPlanSupportsHashAggregate = Aggregate.supportsHashAggregate( - cachedPlanAggregateExpressions.flatMap(_.aggregateFunction.aggBufferAttributes)) + val aggregateExpressionsSeq = Seq(newPlan, cachedPlan).map { + plan => plan.aggregateExpressions.flatMap(_.collect { case a: AggregateExpression => a }) + } + val groupByExpressionSeq = Seq(newPlan, cachedPlan).map(_.groupingExpressions) + + val Seq(newPlanSupportsHashAggregate, cachedPlanSupportsHashAggregate) = + aggregateExpressionsSeq.zip(groupByExpressionSeq).map { + case (aggregateExpressions, groupByExpressions) => + SparkShimLoader.getSparkShim.supportsHashAggregate( + aggregateExpressions.flatMap(_.aggregateFunction.aggBufferAttributes), + groupByExpressions) + } + newPlanSupportsHashAggregate && cachedPlanSupportsHashAggregate || newPlanSupportsHashAggregate == cachedPlanSupportsHashAggregate && { - val newPlanSupportsObjectHashAggregate = - Aggregate.supportsObjectHashAggregate(newPlanAggregateExpressions) - val cachedPlanSupportsObjectHashAggregate = - Aggregate.supportsObjectHashAggregate(cachedPlanAggregateExpressions) + val Seq(newPlanSupportsObjectHashAggregate, cachedPlanSupportsObjectHashAggregate) = + aggregateExpressionsSeq.map( + aggregateExpressions => Aggregate.supportsObjectHashAggregate(aggregateExpressions)) newPlanSupportsObjectHashAggregate && cachedPlanSupportsObjectHashAggregate || newPlanSupportsObjectHashAggregate == cachedPlanSupportsObjectHashAggregate } diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/plans/logical/CreateOrReplaceTagCommand.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/plans/logical/CreateOrReplaceTagCommand.scala new file mode 100644 index 000000000000..0830fc9ed3d6 --- /dev/null +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/plans/logical/CreateOrReplaceTagCommand.scala @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.catalyst.plans.logical + +import org.apache.paimon.spark.leafnode.PaimonLeafCommand + +import org.apache.spark.sql.catalyst.expressions.Attribute + +case class CreateOrReplaceTagCommand( + table: Seq[String], + tagName: String, + tagOptions: TagOptions, + create: Boolean, + replace: Boolean, + ifNotExists: Boolean) + extends PaimonLeafCommand { + + override def output: Seq[Attribute] = Nil + + override def simpleString(maxFields: Int): String = { + s"Create tag: $tagName for table: $table" + } +} diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/plans/logical/DeleteTagCommand.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/plans/logical/DeleteTagCommand.scala new file mode 100644 index 000000000000..072ed6b09f39 --- /dev/null +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/plans/logical/DeleteTagCommand.scala @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.catalyst.plans.logical + +import org.apache.paimon.spark.leafnode.PaimonLeafCommand + +import org.apache.spark.sql.catalyst.expressions.Attribute + +case class DeleteTagCommand(table: Seq[String], tagStr: String, ifExists: Boolean) + extends PaimonLeafCommand { + + override def output: Seq[Attribute] = Nil + + override def simpleString(maxFields: Int): String = { + s"Delete tag: $tagStr for table: $table" + } +} diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/plans/logical/PaimonTableValuedFunctions.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/plans/logical/PaimonTableValuedFunctions.scala index 4d63c2a8d2be..6edbf533cbbc 100644 --- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/plans/logical/PaimonTableValuedFunctions.scala +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/plans/logical/PaimonTableValuedFunctions.scala @@ -19,8 +19,6 @@ package org.apache.paimon.spark.catalyst.plans.logical import org.apache.paimon.CoreOptions -import org.apache.paimon.spark.SparkCatalog -import org.apache.paimon.spark.catalog.Catalogs import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.FunctionIdentifier @@ -28,7 +26,7 @@ import org.apache.spark.sql.catalyst.analysis.FunctionRegistryBase import org.apache.spark.sql.catalyst.analysis.TableFunctionRegistry.TableFunctionBuilder import org.apache.spark.sql.catalyst.expressions.{Attribute, Expression, ExpressionInfo} import org.apache.spark.sql.catalyst.plans.logical.{LeafNode, LogicalPlan} -import org.apache.spark.sql.connector.catalog.Identifier +import org.apache.spark.sql.connector.catalog.{Identifier, TableCatalog} import org.apache.spark.sql.execution.datasources.v2.DataSourceV2Relation import org.apache.spark.sql.util.CaseInsensitiveStringMap @@ -40,7 +38,7 @@ object PaimonTableValuedFunctions { val supportedFnNames: Seq[String] = Seq(INCREMENTAL_QUERY) - type TableFunctionDescription = (FunctionIdentifier, ExpressionInfo, TableFunctionBuilder) + private type TableFunctionDescription = (FunctionIdentifier, ExpressionInfo, TableFunctionBuilder) def getTableValueFunctionInjection(fnName: String): TableFunctionDescription = { val (info, builder) = fnName match { @@ -60,13 +58,7 @@ object PaimonTableValuedFunctions { val sessionState = spark.sessionState val catalogManager = sessionState.catalogManager - - val sparkCatalog = new SparkCatalog() - val currentCatalog = catalogManager.currentCatalog.name() - sparkCatalog.initialize( - currentCatalog, - Catalogs.catalogOptions(currentCatalog, spark.sessionState.conf)) - + val sparkCatalog = catalogManager.currentCatalog.asInstanceOf[TableCatalog] val tableId = sessionState.sqlParser.parseTableIdentifier(args.head.eval().toString) val namespace = tableId.database.map(Array(_)).getOrElse(catalogManager.currentNamespace) val ident = Identifier.of(namespace, tableId.table) diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/plans/logical/PaimonViewCommand.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/plans/logical/PaimonViewCommand.scala new file mode 100644 index 000000000000..24b27bb0e6cc --- /dev/null +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/plans/logical/PaimonViewCommand.scala @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.catalyst.plans.logical + +import org.apache.paimon.spark.leafnode.{PaimonBinaryCommand, PaimonUnaryCommand} + +import org.apache.spark.sql.catalyst.expressions.Attribute +import org.apache.spark.sql.catalyst.plans.logical.{LeafNode, LogicalPlan, ShowViews, Statistics} +import org.apache.spark.sql.connector.catalog.{CatalogPlugin, Identifier} + +case class CreatePaimonView( + child: LogicalPlan, + queryText: String, + query: LogicalPlan, + columnAliases: Seq[String], + columnComments: Seq[Option[String]], + queryColumnNames: Seq[String] = Seq.empty, + comment: Option[String], + properties: Map[String, String], + allowExisting: Boolean, + replace: Boolean) + extends PaimonBinaryCommand { + + override def left: LogicalPlan = child + + override def right: LogicalPlan = query + + override protected def withNewChildrenInternal( + newLeft: LogicalPlan, + newRight: LogicalPlan): LogicalPlan = + copy(child = newLeft, query = newRight) +} + +case class DropPaimonView(child: LogicalPlan, ifExists: Boolean) extends PaimonUnaryCommand { + + override protected def withNewChildInternal(newChild: LogicalPlan): DropPaimonView = + copy(child = newChild) +} + +case class ShowPaimonViews( + namespace: LogicalPlan, + pattern: Option[String], + override val output: Seq[Attribute] = ShowViews.getOutputAttrs) + extends PaimonUnaryCommand { + + override def child: LogicalPlan = namespace + + override protected def withNewChildInternal(newChild: LogicalPlan): ShowPaimonViews = + copy(namespace = newChild) +} + +/** Copy from spark 3.4+ */ +case class ResolvedIdentifier(catalog: CatalogPlugin, identifier: Identifier) extends LeafNode { + + override def output: Seq[Attribute] = Nil + + override def stats: Statistics = Statistics.DUMMY +} diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/plans/logical/RenameTagCommand.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/plans/logical/RenameTagCommand.scala new file mode 100644 index 000000000000..df68c40382e7 --- /dev/null +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/plans/logical/RenameTagCommand.scala @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.catalyst.plans.logical + +import org.apache.paimon.spark.leafnode.PaimonLeafCommand + +import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference} + +case class RenameTagCommand(table: Seq[String], sourceTag: String, targetTag: String) + extends PaimonLeafCommand { + + override def output: Seq[Attribute] = Nil + + override def simpleString(maxFields: Int): String = { + s"Rename tag from $sourceTag to $targetTag for table: $table" + } +} diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/plans/logical/ShowTagsCommand.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/plans/logical/ShowTagsCommand.scala new file mode 100644 index 000000000000..f5b62d333861 --- /dev/null +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/plans/logical/ShowTagsCommand.scala @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.catalyst.plans.logical + +import org.apache.paimon.spark.leafnode.PaimonLeafCommand + +import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference} +import org.apache.spark.sql.types.StringType + +case class ShowTagsCommand(table: Seq[String]) extends PaimonLeafCommand { + + override def output: Seq[Attribute] = + Seq(AttributeReference("tag", StringType, nullable = false)()) + + override def simpleString(maxFields: Int): String = { + s"Show Tags for table: $table" + } +} diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/plans/logical/TagOptions.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/plans/logical/TagOptions.scala new file mode 100644 index 000000000000..242e9dac15a6 --- /dev/null +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/plans/logical/TagOptions.scala @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.catalyst.plans.logical + +import java.time.Duration + +case class TagOptions(snapshotId: Option[Long], timeRetained: Option[Duration]) diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/commands/BucketProcessor.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/commands/BucketProcessor.scala index f252b3bb130b..57a8a8e4abfd 100644 --- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/commands/BucketProcessor.scala +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/commands/BucketProcessor.scala @@ -22,7 +22,7 @@ import org.apache.paimon.crosspartition.{GlobalIndexAssigner, KeyPartOrRow} import org.apache.paimon.data.{BinaryRow, GenericRow, InternalRow => PaimonInternalRow, JoinedRow} import org.apache.paimon.disk.IOManager import org.apache.paimon.index.HashBucketAssigner -import org.apache.paimon.spark.{SparkInternalRow, SparkRow} +import org.apache.paimon.spark.{DataConverter, SparkRow} import org.apache.paimon.spark.SparkUtils.createIOManager import org.apache.paimon.spark.util.EncoderUtils import org.apache.paimon.table.FileStoreTable @@ -179,7 +179,7 @@ class GlobalIndexAssignerIterator( extraRow.setField(1, bucket) queue.enqueue( encoderGroup.internalToRow( - SparkInternalRow.fromPaimon(new JoinedRow(row, extraRow), rowType))) + DataConverter.fromPaimon(new JoinedRow(row, extraRow), rowType))) } ) rowIterator.foreach { diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/commands/DeleteFromPaimonTableCommand.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/commands/DeleteFromPaimonTableCommand.scala index 2b3888911226..097823d730ce 100644 --- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/commands/DeleteFromPaimonTableCommand.scala +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/commands/DeleteFromPaimonTableCommand.scala @@ -47,8 +47,7 @@ case class DeleteFromPaimonTableCommand( extends PaimonLeafRunnableCommand with PaimonCommand with ExpressionHelper - with SupportsSubquery - with SQLHelper { + with SupportsSubquery { private lazy val writer = PaimonSparkWriter(table) @@ -60,7 +59,7 @@ case class DeleteFromPaimonTableCommand( } else { val (partitionCondition, otherCondition) = splitPruePartitionAndOtherPredicates( condition, - table.partitionKeys().asScala, + table.partitionKeys().asScala.toSeq, sparkSession.sessionState.conf.resolver) val partitionPredicate = if (partitionCondition.isEmpty) { @@ -83,7 +82,8 @@ case class DeleteFromPaimonTableCommand( val rowDataPartitionComputer = new InternalRowPartitionComputer( table.coreOptions().partitionDefaultName(), table.schema().logicalPartitionType(), - table.partitionKeys.asScala.toArray + table.partitionKeys.asScala.toArray, + table.coreOptions().legacyPartitionName() ) val dropPartitions = matchedPartitions.map { partition => rowDataPartitionComputer.generatePartValues(partition).asScala.asJava diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/commands/MergeIntoPaimonTable.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/commands/MergeIntoPaimonTable.scala index 51ae6e086448..52e704172fc8 100644 --- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/commands/MergeIntoPaimonTable.scala +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/commands/MergeIntoPaimonTable.scala @@ -28,7 +28,7 @@ import org.apache.paimon.table.FileStoreTable import org.apache.paimon.table.sink.CommitMessage import org.apache.paimon.types.RowKind -import org.apache.spark.sql.{Column, Dataset, Row, SparkSession} +import org.apache.spark.sql.{Dataset, Row, SparkSession} import org.apache.spark.sql.PaimonUtils._ import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder @@ -152,12 +152,12 @@ case class MergeIntoPaimonTable( } if (hasUpdate(matchedActions)) { touchedFilePathsSet ++= findTouchedFiles( - targetDS.join(sourceDS, new Column(mergeCondition), "inner"), + targetDS.join(sourceDS, toColumn(mergeCondition), "inner"), sparkSession) } if (hasUpdate(notMatchedBySourceActions)) { touchedFilePathsSet ++= findTouchedFiles( - targetDS.join(sourceDS, new Column(mergeCondition), "left_anti"), + targetDS.join(sourceDS, toColumn(mergeCondition), "left_anti"), sparkSession) } @@ -199,7 +199,7 @@ case class MergeIntoPaimonTable( val sourceDS = createDataset(sparkSession, sourceTable) .withColumn(SOURCE_ROW_COL, lit(true)) - val joinedDS = sourceDS.join(targetDS, new Column(mergeCondition), "fullOuter") + val joinedDS = sourceDS.join(targetDS, toColumn(mergeCondition), "fullOuter") val joinedPlan = joinedDS.queryExecution.analyzed def resolveOnJoinedPlan(exprs: Seq[Expression]): Seq[Expression] = { @@ -207,8 +207,10 @@ case class MergeIntoPaimonTable( } val targetOutput = filteredTargetPlan.output - val targetRowNotMatched = resolveOnJoinedPlan(Seq(col(SOURCE_ROW_COL).isNull.expr)).head - val sourceRowNotMatched = resolveOnJoinedPlan(Seq(col(TARGET_ROW_COL).isNull.expr)).head + val targetRowNotMatched = resolveOnJoinedPlan( + Seq(toExpression(sparkSession, col(SOURCE_ROW_COL).isNull))).head + val sourceRowNotMatched = resolveOnJoinedPlan( + Seq(toExpression(sparkSession, col(TARGET_ROW_COL).isNull))).head val matchedExprs = matchedActions.map(_.condition.getOrElse(TrueLiteral)) val notMatchedExprs = notMatchedActions.map(_.condition.getOrElse(TrueLiteral)) val notMatchedBySourceExprs = notMatchedBySourceActions.map(_.condition.getOrElse(TrueLiteral)) @@ -243,7 +245,7 @@ case class MergeIntoPaimonTable( val outputFields = mutable.ArrayBuffer(tableSchema.fields: _*) outputFields += StructField(ROW_KIND_COL, ByteType) outputFields ++= metadataCols.map(_.toStructField) - val outputSchema = StructType(outputFields) + val outputSchema = StructType(outputFields.toSeq) val joinedRowEncoder = EncoderUtils.encode(joinedPlan.schema) val outputEncoder = EncoderUtils.encode(outputSchema).resolveAndBind() @@ -272,7 +274,7 @@ case class MergeIntoPaimonTable( .withColumn(ROW_ID_COL, monotonically_increasing_id()) val sourceDS = createDataset(sparkSession, sourceTable) val count = sourceDS - .join(targetDS, new Column(mergeCondition), "inner") + .join(targetDS, toColumn(mergeCondition), "inner") .select(col(ROW_ID_COL), lit(1).as("one")) .groupBy(ROW_ID_COL) .agg(sum("one").as("count")) diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/commands/PaimonAnalyzeTableColumnCommand.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/commands/PaimonAnalyzeTableColumnCommand.scala index b13e5add01d3..9a88ca2e4c3a 100644 --- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/commands/PaimonAnalyzeTableColumnCommand.scala +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/commands/PaimonAnalyzeTableColumnCommand.scala @@ -18,6 +18,7 @@ package org.apache.paimon.spark.commands +import org.apache.paimon.manifest.PartitionEntry import org.apache.paimon.schema.TableSchema import org.apache.paimon.spark.SparkTable import org.apache.paimon.spark.leafnode.PaimonLeafRunnableCommand @@ -25,8 +26,8 @@ import org.apache.paimon.stats.{ColStats, Statistics} import org.apache.paimon.table.FileStoreTable import org.apache.paimon.table.sink.BatchWriteBuilder import org.apache.paimon.table.source.DataSplit +import org.apache.paimon.utils.Preconditions.checkState -import org.apache.parquet.Preconditions import org.apache.spark.sql.{PaimonStatsUtils, Row, SparkSession} import org.apache.spark.sql.catalyst.expressions.Attribute import org.apache.spark.sql.catalyst.plans.logical.ColumnStat @@ -64,19 +65,15 @@ case class PaimonAnalyzeTableColumnCommand( // compute stats val totalSize = table .newScan() - .plan() - .splits() + .listPartitionEntries() .asScala - .flatMap { case split: DataSplit => split.dataFiles().asScala } - .map(_.fileSize()) + .map(_.fileSizeInBytes()) .sum val (mergedRecordCount, colStats) = PaimonStatsUtils.computeColumnStats(sparkSession, relation, attributes) val totalRecordCount = currentSnapshot.totalRecordCount() - Preconditions.checkState( - totalRecordCount >= mergedRecordCount, - s"totalRecordCount: $totalRecordCount should be greater or equal than mergedRecordCount: $mergedRecordCount.") + checkState(totalRecordCount >= mergedRecordCount) val mergedRecordSize = totalSize * (mergedRecordCount.toDouble / totalRecordCount).toLong // convert to paimon stats @@ -97,6 +94,7 @@ case class PaimonAnalyzeTableColumnCommand( // commit stats val commit = table.store.newCommit(UUID.randomUUID.toString) commit.commitStatistics(stats, BatchWriteBuilder.COMMIT_IDENTIFIER) + commit.close() Seq.empty[Row] } diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/commands/PaimonCommand.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/commands/PaimonCommand.scala index 62b8ee8498fa..04118a438307 100644 --- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/commands/PaimonCommand.scala +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/commands/PaimonCommand.scala @@ -95,13 +95,17 @@ trait PaimonCommand extends WithFileStoreTable with ExpressionHelper with SQLCon output: Seq[Attribute]): Seq[DataSplit] = { // low level snapshot reader, it can not be affected by 'scan.mode' val snapshotReader = table.newSnapshotReader() + // dropStats after filter push down + if (table.coreOptions().manifestDeleteFileDropStats()) { + snapshotReader.dropStats() + } if (condition != TrueLiteral) { val filter = convertConditionToPaimonPredicate(condition, output, rowType, ignoreFailure = true) filter.foreach(snapshotReader.withFilter) } - snapshotReader.read().splits().asScala.collect { case s: DataSplit => s } + snapshotReader.read().splits().asScala.collect { case s: DataSplit => s }.toSeq } protected def findTouchedFiles( @@ -232,7 +236,7 @@ trait PaimonCommand extends WithFileStoreTable with ExpressionHelper with SQLCon .as[(String, Long)] .groupByKey(_._1) .mapGroups { - case (filePath, iter) => + (filePath, iter) => val dv = new BitmapDeletionVector() while (iter.hasNext) { dv.delete(iter.next()._2) @@ -241,12 +245,12 @@ trait PaimonCommand extends WithFileStoreTable with ExpressionHelper with SQLCon val relativeFilePath = location.toUri.relativize(new URI(filePath)).toString val (partition, bucket) = dataFileToPartitionAndBucket.toMap.apply(relativeFilePath) val pathFactory = my_table.store().pathFactory() - val partitionAndBucket = pathFactory - .relativePartitionAndBucketPath(partition, bucket) + val relativeBucketPath = pathFactory + .relativeBucketPath(partition, bucket) .toString SparkDeletionVectors( - partitionAndBucket, + relativeBucketPath, SerializationUtils.serializeBinaryRow(partition), bucket, Seq((new Path(filePath).getName, dv.serializeToBytes())) diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/commands/PaimonSparkWriter.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/commands/PaimonSparkWriter.scala index 391ba2b87c93..7d56fe867a1b 100644 --- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/commands/PaimonSparkWriter.scala +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/commands/PaimonSparkWriter.scala @@ -250,7 +250,7 @@ case class PaimonSparkWriter(table: FileStoreTable) { val serializedCommits = deletionVectors .groupByKey(_.partitionAndBucket) .mapGroups { - case (_, iter: Iterator[SparkDeletionVectors]) => + (_, iter: Iterator[SparkDeletionVectors]) => val indexHandler = table.store().newIndexFileHandler() var dvIndexFileMaintainer: AppendDeletionFileMaintainer = null while (iter.hasNext) { @@ -397,7 +397,7 @@ case class PaimonSparkWriter(table: FileStoreTable) { } private def repartitionByPartitionsAndBucket(df: DataFrame): DataFrame = { - val partitionCols = tableSchema.partitionKeys().asScala.map(col) + val partitionCols = tableSchema.partitionKeys().asScala.map(col).toSeq df.repartition(partitionCols ++ Seq(col(BUCKET_COL)): _*) } diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/commands/SparkDataFileMeta.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/commands/SparkDataFileMeta.scala index b380d36c3f81..569a84a74cf5 100644 --- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/commands/SparkDataFileMeta.scala +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/commands/SparkDataFileMeta.scala @@ -35,7 +35,7 @@ case class SparkDataFileMeta( def relativePath(fileStorePathFactory: FileStorePathFactory): String = { fileStorePathFactory - .relativePartitionAndBucketPath(partition, bucket) + .relativeBucketPath(partition, bucket) .toUri .toString + "/" + dataFileMeta.fileName() } @@ -58,7 +58,7 @@ object SparkDataFileMeta { file, dvFactory.create(file.fileName())) } - } + }.toSeq def convertToDataSplits( sparkDataFiles: Array[SparkDataFileMeta], diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/commands/SparkDeletionVectors.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/commands/SparkDeletionVectors.scala index 9f687e6e3c92..1f908aeb908b 100644 --- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/commands/SparkDeletionVectors.scala +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/commands/SparkDeletionVectors.scala @@ -36,7 +36,7 @@ case class SparkDeletionVectors( ) { def relativePaths(fileStorePathFactory: FileStorePathFactory): Seq[String] = { val prefix = fileStorePathFactory - .relativePartitionAndBucketPath(SerializationUtils.deserializeBinaryRow(partition), bucket) + .relativeBucketPath(SerializationUtils.deserializeBinaryRow(partition), bucket) .toUri .toString + "/" dataFileAndDeletionVector.map(prefix + _._1) diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/commands/UpdatePaimonTableCommand.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/commands/UpdatePaimonTableCommand.scala index 7a1124125346..47e3f77d0e2c 100644 --- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/commands/UpdatePaimonTableCommand.scala +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/commands/UpdatePaimonTableCommand.scala @@ -26,13 +26,14 @@ import org.apache.paimon.table.sink.CommitMessage import org.apache.paimon.table.source.DataSplit import org.apache.paimon.types.RowKind -import org.apache.spark.sql.{Column, Row, SparkSession} +import org.apache.spark.sql.{Row, SparkSession} import org.apache.spark.sql.PaimonUtils.createDataset import org.apache.spark.sql.catalyst.expressions.{Alias, Expression, If} import org.apache.spark.sql.catalyst.expressions.Literal.TrueLiteral import org.apache.spark.sql.catalyst.plans.logical.{Assignment, Filter, Project, SupportsSubquery} import org.apache.spark.sql.execution.datasources.v2.DataSourceV2Relation import org.apache.spark.sql.functions.lit +import org.apache.spark.sql.paimon.shims.SparkShimLoader case class UpdatePaimonTableCommand( relation: DataSourceV2Relation, @@ -133,7 +134,7 @@ case class UpdatePaimonTableCommand( touchedDataSplits: Array[DataSplit]): Seq[CommitMessage] = { val updateColumns = updateExpressions.zip(relation.output).map { case (update, origin) => - new Column(update).as(origin.name, origin.metadata) + SparkShimLoader.getSparkShim.column(update).as(origin.name, origin.metadata) } val toUpdateScanRelation = createNewRelation(touchedDataSplits, relation) @@ -156,7 +157,7 @@ case class UpdatePaimonTableCommand( } else { If(condition, update, origin) } - new Column(updated).as(origin.name, origin.metadata) + SparkShimLoader.getSparkShim.column(updated).as(origin.name, origin.metadata) } val data = createDataset(sparkSession, toUpdateScanRelation).select(updateColumns: _*) diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/commands/WriteIntoPaimonTable.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/commands/WriteIntoPaimonTable.scala index fe740ea8ca11..27d9a0786a56 100644 --- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/commands/WriteIntoPaimonTable.scala +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/commands/WriteIntoPaimonTable.scala @@ -18,11 +18,15 @@ package org.apache.paimon.spark.commands +import org.apache.paimon.CoreOptions import org.apache.paimon.CoreOptions.DYNAMIC_PARTITION_OVERWRITE import org.apache.paimon.options.Options +import org.apache.paimon.partition.actions.PartitionMarkDoneAction import org.apache.paimon.spark._ import org.apache.paimon.spark.schema.SparkSystemColumns import org.apache.paimon.table.FileStoreTable +import org.apache.paimon.table.sink.CommitMessage +import org.apache.paimon.utils.{InternalRowPartitionComputer, PartitionPathUtils, TypeUtils} import org.apache.spark.internal.Logging import org.apache.spark.sql.{DataFrame, Row, SparkSession} @@ -63,9 +67,29 @@ case class WriteIntoPaimonTable( val commitMessages = writer.write(data) writer.commit(commitMessages) + markDoneIfNeeded(commitMessages) Seq.empty } + private def markDoneIfNeeded(commitMessages: Seq[CommitMessage]): Unit = { + val coreOptions = table.coreOptions() + if (coreOptions.toConfiguration.get(CoreOptions.PARTITION_MARK_DONE_WHEN_END_INPUT)) { + val actions = PartitionMarkDoneAction.createActions(table, table.coreOptions()) + val partitionComputer = new InternalRowPartitionComputer( + coreOptions.partitionDefaultName, + TypeUtils.project(table.rowType(), table.partitionKeys()), + table.partitionKeys().asScala.toArray, + coreOptions.legacyPartitionName() + ) + val partitions = commitMessages + .map(c => c.partition()) + .map(p => PartitionPathUtils.generatePartitionPath(partitionComputer.generatePartValues(p))) + for (partition <- partitions) { + actions.forEach(a => a.markDone(partition)) + } + } + } + private def parseSaveMode(): (Boolean, Map[String, String]) = { var dynamicPartitionOverwriteMode = false val overwritePartition = saveMode match { diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/data/SparkArrayData.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/data/SparkArrayData.scala new file mode 100644 index 000000000000..c6539a493cee --- /dev/null +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/data/SparkArrayData.scala @@ -0,0 +1,118 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.data + +import org.apache.paimon.data.InternalArray +import org.apache.paimon.spark.DataConverter +import org.apache.paimon.types.{ArrayType => PaimonArrayType, BigIntType, DataType => PaimonDataType, DataTypeChecks, RowType} +import org.apache.paimon.utils.InternalRowUtils + +import org.apache.spark.sql.catalyst.InternalRow +import org.apache.spark.sql.catalyst.expressions.SpecializedGettersReader +import org.apache.spark.sql.catalyst.util.{ArrayData, MapData} +import org.apache.spark.sql.paimon.shims.SparkShimLoader +import org.apache.spark.sql.types.{DataType, Decimal} +import org.apache.spark.unsafe.types.{CalendarInterval, UTF8String} + +abstract class SparkArrayData extends org.apache.spark.sql.catalyst.util.ArrayData { + + def replace(array: InternalArray): SparkArrayData +} + +abstract class AbstractSparkArrayData extends SparkArrayData { + + val elementType: PaimonDataType + + var paimonArray: InternalArray = _ + + override def replace(array: InternalArray): SparkArrayData = { + this.paimonArray = array + this + } + + override def numElements(): Int = paimonArray.size() + + override def copy(): ArrayData = { + SparkArrayData.create(elementType).replace(InternalRowUtils.copyArray(paimonArray, elementType)) + } + + override def array: Array[Any] = { + Array.range(0, numElements()).map { + i => + DataConverter + .fromPaimon(InternalRowUtils.get(paimonArray, i, elementType), elementType) + } + } + + override def setNullAt(i: Int): Unit = throw new UnsupportedOperationException() + + override def update(i: Int, value: Any): Unit = throw new UnsupportedOperationException() + + override def isNullAt(ordinal: Int): Boolean = paimonArray.isNullAt(ordinal) + + override def getBoolean(ordinal: Int): Boolean = paimonArray.getBoolean(ordinal) + + override def getByte(ordinal: Int): Byte = paimonArray.getByte(ordinal) + + override def getShort(ordinal: Int): Short = paimonArray.getShort(ordinal) + + override def getInt(ordinal: Int): Int = paimonArray.getInt(ordinal) + + override def getLong(ordinal: Int): Long = elementType match { + case _: BigIntType => paimonArray.getLong(ordinal) + case _ => + DataConverter.fromPaimon( + paimonArray.getTimestamp(ordinal, DataTypeChecks.getPrecision(elementType))) + } + + override def getFloat(ordinal: Int): Float = paimonArray.getFloat(ordinal) + + override def getDouble(ordinal: Int): Double = paimonArray.getDouble(ordinal) + + override def getDecimal(ordinal: Int, precision: Int, scale: Int): Decimal = + DataConverter.fromPaimon(paimonArray.getDecimal(ordinal, precision, scale)) + + override def getUTF8String(ordinal: Int): UTF8String = + DataConverter.fromPaimon(paimonArray.getString(ordinal)) + + override def getBinary(ordinal: Int): Array[Byte] = paimonArray.getBinary(ordinal) + + override def getInterval(ordinal: Int): CalendarInterval = + throw new UnsupportedOperationException() + + override def getStruct(ordinal: Int, numFields: Int): InternalRow = DataConverter + .fromPaimon(paimonArray.getRow(ordinal, numFields), elementType.asInstanceOf[RowType]) + + override def getArray(ordinal: Int): ArrayData = DataConverter.fromPaimon( + paimonArray.getArray(ordinal), + elementType.asInstanceOf[PaimonArrayType]) + + override def getMap(ordinal: Int): MapData = + DataConverter.fromPaimon(paimonArray.getMap(ordinal), elementType) + + override def get(ordinal: Int, dataType: DataType): AnyRef = + SpecializedGettersReader.read(this, ordinal, dataType, true, true) + +} + +object SparkArrayData { + def create(elementType: PaimonDataType): SparkArrayData = { + SparkShimLoader.getSparkShim.createSparkArrayData(elementType) + } +} diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/data/SparkInternalRow.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/data/SparkInternalRow.scala new file mode 100644 index 000000000000..f3e607e9d7d2 --- /dev/null +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/data/SparkInternalRow.scala @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.data + +import org.apache.paimon.types.RowType + +import org.apache.spark.sql.catalyst.InternalRow +import org.apache.spark.sql.paimon.shims.SparkShimLoader + +abstract class SparkInternalRow extends InternalRow { + def replace(row: org.apache.paimon.data.InternalRow): SparkInternalRow +} + +object SparkInternalRow { + + def create(rowType: RowType): SparkInternalRow = { + SparkShimLoader.getSparkShim.createSparkInternalRow(rowType) + } + +} diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/execution/CreateOrReplaceTagExec.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/execution/CreateOrReplaceTagExec.scala new file mode 100644 index 000000000000..0506ed42f1f4 --- /dev/null +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/execution/CreateOrReplaceTagExec.scala @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.execution + +import org.apache.paimon.spark.SparkTable +import org.apache.paimon.spark.catalyst.plans.logical.TagOptions +import org.apache.paimon.spark.leafnode.PaimonLeafV2CommandExec +import org.apache.paimon.table.FileStoreTable + +import org.apache.spark.sql.catalyst.InternalRow +import org.apache.spark.sql.catalyst.expressions.Attribute +import org.apache.spark.sql.connector.catalog.{Identifier, TableCatalog} + +case class CreateOrReplaceTagExec( + catalog: TableCatalog, + ident: Identifier, + tagName: String, + tagOptions: TagOptions, + create: Boolean, + replace: Boolean, + ifNotExists: Boolean) + extends PaimonLeafV2CommandExec { + + override protected def run(): Seq[InternalRow] = { + val table = catalog.loadTable(ident) + assert(table.isInstanceOf[SparkTable]) + + table.asInstanceOf[SparkTable].getTable match { + case paimonTable: FileStoreTable => + val tagIsExists = paimonTable.tagManager().tagExists(tagName) + val timeRetained = tagOptions.timeRetained.orNull + val snapshotId = tagOptions.snapshotId + + if (create && replace && !tagIsExists) { + if (snapshotId.isEmpty) { + paimonTable.createTag(tagName, timeRetained) + } else { + paimonTable.createTag(tagName, snapshotId.get, timeRetained) + } + } else if (replace) { + paimonTable.replaceTag(tagName, snapshotId.get, timeRetained) + } else { + if (tagIsExists && ifNotExists) { + return Nil + } + + if (snapshotId.isEmpty) { + paimonTable.createTag(tagName, timeRetained) + } else { + paimonTable.createTag(tagName, snapshotId.get, timeRetained) + } + } + case t => + throw new UnsupportedOperationException( + s"Can not create tag for non-paimon FileStoreTable: $t") + } + Nil + } + + override def output: Seq[Attribute] = Nil +} diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/execution/DeleteTagExec.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/execution/DeleteTagExec.scala new file mode 100644 index 000000000000..d27839bdc6b6 --- /dev/null +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/execution/DeleteTagExec.scala @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.execution + +import org.apache.paimon.spark.SparkTable +import org.apache.paimon.spark.leafnode.PaimonLeafV2CommandExec +import org.apache.paimon.table.FileStoreTable + +import org.apache.spark.sql.catalyst.InternalRow +import org.apache.spark.sql.catalyst.expressions.Attribute +import org.apache.spark.sql.connector.catalog.{Identifier, TableCatalog} + +case class DeleteTagExec( + catalog: TableCatalog, + ident: Identifier, + tagStr: String, + ifExists: Boolean) + extends PaimonLeafV2CommandExec { + + private val DELIMITER = "," + + override protected def run(): Seq[InternalRow] = { + val table = catalog.loadTable(ident) + assert(table.isInstanceOf[SparkTable]) + + table.asInstanceOf[SparkTable].getTable match { + case paimonTable: FileStoreTable => + val tagNames = tagStr.split(DELIMITER).map(_.trim) + for (tagName <- tagNames) { + val tagIsExists = paimonTable.tagManager().tagExists(tagName) + if (tagIsExists || !ifExists) { + paimonTable.deleteTag(tagName) + } + } + case t => + throw new UnsupportedOperationException( + s"Can not delete tag for non-paimon FileStoreTable: $t") + } + Nil + } + + override def output: Seq[Attribute] = Nil +} diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/execution/PaimonStrategy.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/execution/PaimonStrategy.scala index c6c6fc8759c0..fb7bc6b22cd3 100644 --- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/execution/PaimonStrategy.scala +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/execution/PaimonStrategy.scala @@ -18,16 +18,28 @@ package org.apache.paimon.spark.execution -import org.apache.paimon.spark.catalyst.plans.logical.PaimonCallCommand +import org.apache.paimon.spark.{SparkCatalog, SparkUtils} +import org.apache.paimon.spark.catalog.SupportView +import org.apache.paimon.spark.catalyst.analysis.ResolvedPaimonView +import org.apache.paimon.spark.catalyst.plans.logical.{CreateOrReplaceTagCommand, CreatePaimonView, DeleteTagCommand, DropPaimonView, PaimonCallCommand, RenameTagCommand, ResolvedIdentifier, ShowPaimonViews, ShowTagsCommand} import org.apache.spark.sql.{SparkSession, Strategy} import org.apache.spark.sql.catalyst.InternalRow +import org.apache.spark.sql.catalyst.analysis.ResolvedNamespace import org.apache.spark.sql.catalyst.expressions.{Expression, GenericInternalRow, PredicateHelper} -import org.apache.spark.sql.catalyst.plans.logical.{CreateTableAsSelect, LogicalPlan} +import org.apache.spark.sql.catalyst.plans.logical.{CreateTableAsSelect, DescribeRelation, LogicalPlan, ShowCreateTable} +import org.apache.spark.sql.connector.catalog.{Identifier, PaimonLookupCatalog, TableCatalog} import org.apache.spark.sql.execution.SparkPlan import org.apache.spark.sql.execution.shim.PaimonCreateTableAsSelectStrategy -case class PaimonStrategy(spark: SparkSession) extends Strategy with PredicateHelper { +import scala.collection.JavaConverters._ + +case class PaimonStrategy(spark: SparkSession) + extends Strategy + with PredicateHelper + with PaimonLookupCatalog { + + protected lazy val catalogManager = spark.sessionState.catalogManager override def apply(plan: LogicalPlan): Seq[SparkPlan] = plan match { @@ -37,6 +49,64 @@ case class PaimonStrategy(spark: SparkSession) extends Strategy with PredicateHe case c @ PaimonCallCommand(procedure, args) => val input = buildInternalRow(args) PaimonCallExec(c.output, procedure, input) :: Nil + + case t @ ShowTagsCommand(PaimonCatalogAndIdentifier(catalog, ident)) => + ShowTagsExec(catalog, ident, t.output) :: Nil + + case CreateOrReplaceTagCommand( + PaimonCatalogAndIdentifier(table, ident), + tagName, + tagOptions, + create, + replace, + ifNotExists) => + CreateOrReplaceTagExec(table, ident, tagName, tagOptions, create, replace, ifNotExists) :: Nil + + case DeleteTagCommand(PaimonCatalogAndIdentifier(catalog, ident), tagStr, ifExists) => + DeleteTagExec(catalog, ident, tagStr, ifExists) :: Nil + + case RenameTagCommand(PaimonCatalogAndIdentifier(catalog, ident), sourceTag, targetTag) => + RenameTagExec(catalog, ident, sourceTag, targetTag) :: Nil + + case CreatePaimonView( + ResolvedIdentifier(viewCatalog: SupportView, ident), + queryText, + query, + columnAliases, + columnComments, + queryColumnNames, + comment, + properties, + allowExisting, + replace) => + CreatePaimonViewExec( + viewCatalog, + ident, + queryText, + query.schema, + columnAliases, + columnComments, + queryColumnNames, + comment, + properties, + allowExisting, + replace) :: Nil + + case DropPaimonView(ResolvedIdentifier(viewCatalog: SupportView, ident), ifExists) => + DropPaimonViewExec(viewCatalog, ident, ifExists) :: Nil + + // A new member was added to ResolvedNamespace since spark4.0, + // unapply pattern matching is not used here to ensure compatibility across multiple spark versions. + case ShowPaimonViews(r: ResolvedNamespace, pattern, output) + if r.catalog.isInstanceOf[SupportView] => + ShowPaimonViewsExec(output, r.catalog.asInstanceOf[SupportView], r.namespace, pattern) :: Nil + + case ShowCreateTable(ResolvedPaimonView(viewCatalog, ident), _, output) => + ShowCreatePaimonViewExec(output, viewCatalog, ident) :: Nil + + case DescribeRelation(ResolvedPaimonView(viewCatalog, ident), _, isExtended, output) => + DescribePaimonViewExec(output, viewCatalog, ident, isExtended) :: Nil + case _ => Nil } @@ -48,4 +118,16 @@ case class PaimonStrategy(spark: SparkSession) extends Strategy with PredicateHe new GenericInternalRow(values) } + private object PaimonCatalogAndIdentifier { + def unapply(identifier: Seq[String]): Option[(TableCatalog, Identifier)] = { + val catalogAndIdentifier = + SparkUtils.catalogAndIdentifier(spark, identifier.asJava, catalogManager.currentCatalog) + catalogAndIdentifier.catalog match { + case paimonCatalog: SparkCatalog => + Some((paimonCatalog, catalogAndIdentifier.identifier())) + case _ => + None + } + } + } } diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/execution/PaimonViewExec.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/execution/PaimonViewExec.scala new file mode 100644 index 000000000000..2282f7c34411 --- /dev/null +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/execution/PaimonViewExec.scala @@ -0,0 +1,232 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.execution + +import org.apache.paimon.spark.catalog.SupportView +import org.apache.paimon.spark.leafnode.PaimonLeafV2CommandExec +import org.apache.paimon.view.View + +import org.apache.spark.sql.catalyst.InternalRow +import org.apache.spark.sql.catalyst.catalog.CatalogTable +import org.apache.spark.sql.catalyst.expressions.{Attribute, GenericInternalRow} +import org.apache.spark.sql.catalyst.util.{escapeSingleQuotedString, quoteIfNeeded, StringUtils} +import org.apache.spark.sql.connector.catalog.Identifier +import org.apache.spark.sql.types.StructType +import org.apache.spark.unsafe.types.UTF8String + +import scala.collection.JavaConverters._ +import scala.collection.mutable.ArrayBuffer + +case class CreatePaimonViewExec( + catalog: SupportView, + ident: Identifier, + queryText: String, + viewSchema: StructType, + columnAliases: Seq[String], + columnComments: Seq[Option[String]], + queryColumnNames: Seq[String], + comment: Option[String], + properties: Map[String, String], + allowExisting: Boolean, + replace: Boolean +) extends PaimonLeafV2CommandExec { + + override def output: Seq[Attribute] = Nil + + override protected def run(): Seq[InternalRow] = { + if (columnAliases.nonEmpty || columnComments.nonEmpty || queryColumnNames.nonEmpty) { + throw new UnsupportedOperationException( + "columnAliases, columnComments and queryColumnNames are not supported now") + } + + // Note: for replace just drop then create ,this operation is non-atomic. + if (replace) { + catalog.dropView(ident, true) + } + + catalog.createView( + ident, + viewSchema, + queryText, + comment.orNull, + properties.asJava, + allowExisting) + + Nil + } + + override def simpleString(maxFields: Int): String = { + s"CreatePaimonViewExec: $ident" + } +} + +case class DropPaimonViewExec(catalog: SupportView, ident: Identifier, ifExists: Boolean) + extends PaimonLeafV2CommandExec { + + override lazy val output: Seq[Attribute] = Nil + + override protected def run(): Seq[InternalRow] = { + catalog.dropView(ident, ifExists) + Nil + } + + override def simpleString(maxFields: Int): String = { + s"DropPaimonViewExec: $ident" + } +} + +case class ShowPaimonViewsExec( + output: Seq[Attribute], + catalog: SupportView, + namespace: Seq[String], + pattern: Option[String]) + extends PaimonLeafV2CommandExec { + + override protected def run(): Seq[InternalRow] = { + val rows = new ArrayBuffer[InternalRow]() + catalog.listViews(namespace.toArray).asScala.map { + viewName => + if (pattern.forall(StringUtils.filterPattern(Seq(viewName), _).nonEmpty)) { + rows += new GenericInternalRow( + Array( + UTF8String.fromString(namespace.mkString(".")), + UTF8String.fromString(viewName), + false)) + } + } + rows.toSeq + } + + override def simpleString(maxFields: Int): String = { + s"ShowPaimonViewsExec: $namespace" + } +} + +case class ShowCreatePaimonViewExec(output: Seq[Attribute], catalog: SupportView, ident: Identifier) + extends PaimonLeafV2CommandExec { + + override protected def run(): Seq[InternalRow] = { + val view = catalog.loadView(ident) + + val builder = new StringBuilder + builder ++= s"CREATE VIEW ${view.fullName()} " + showDataColumns(view, builder) + showComment(view, builder) + showProperties(view, builder) + builder ++= s"AS\n${view.query}\n" + + Seq(new GenericInternalRow(values = Array(UTF8String.fromString(builder.toString)))) + } + + private def showDataColumns(view: View, builder: StringBuilder): Unit = { + if (view.rowType().getFields.size() > 0) { + val viewColumns = view.rowType().getFields.asScala.map { + f => + val comment = if (f.description() != null) s" COMMENT '${f.description()}'" else "" + // view columns shouldn't have data type info + s"${quoteIfNeeded(f.name)}$comment" + } + builder ++= concatByMultiLines(viewColumns) + } + } + + private def showComment(view: View, builder: StringBuilder): Unit = { + if (view.comment().isPresent) { + builder ++= s"COMMENT '${view.comment().get()}'\n" + } + } + + private def showProperties(view: View, builder: StringBuilder): Unit = { + if (!view.options().isEmpty) { + val props = view.options().asScala.toSeq.sortBy(_._1).map { + case (key, value) => + s"'${escapeSingleQuotedString(key)}' = '${escapeSingleQuotedString(value)}'" + } + builder ++= s"TBLPROPERTIES ${concatByMultiLines(props)}" + } + } + + private def concatByMultiLines(iter: Iterable[String]): String = { + iter.mkString("(\n ", ",\n ", ")\n") + } + + override def simpleString(maxFields: Int): String = { + s"ShowCreatePaimonViewExec: $ident" + } +} + +case class DescribePaimonViewExec( + output: Seq[Attribute], + catalog: SupportView, + ident: Identifier, + isExtended: Boolean) + extends PaimonLeafV2CommandExec { + + override protected def run(): Seq[InternalRow] = { + val rows = new ArrayBuffer[InternalRow]() + val view = catalog.loadView(ident) + + describeColumns(view, rows) + if (isExtended) { + describeExtended(view, rows) + } + + rows.toSeq + } + + private def describeColumns(view: View, rows: ArrayBuffer[InternalRow]) = { + view + .rowType() + .getFields + .asScala + .map(f => rows += row(f.name(), f.`type`().toString, f.description())) + } + + private def describeExtended(view: View, rows: ArrayBuffer[InternalRow]) = { + rows += row("", "", "") + rows += row("# Detailed View Information", "", "") + rows += row("Name", view.fullName(), "") + rows += row("Comment", view.comment().orElse(""), "") + rows += row("View Text", view.query, "") + rows += row( + "View Query Output Columns", + view.rowType().getFieldNames.asScala.mkString("[", ", ", "]"), + "") + rows += row( + "View Properties", + view + .options() + .asScala + .toSeq + .sortBy(_._1) + .map { case (k, v) => s"$k=$v" } + .mkString("[", ", ", "]"), + "") + } + + private def row(s1: String, s2: String, s3: String): InternalRow = { + new GenericInternalRow( + values = + Array(UTF8String.fromString(s1), UTF8String.fromString(s2), UTF8String.fromString(s3))) + } + + override def simpleString(maxFields: Int): String = { + s"DescribePaimonViewExec: $ident" + } +} diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/execution/RenameTagExec.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/execution/RenameTagExec.scala new file mode 100644 index 000000000000..655cf70b9cc1 --- /dev/null +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/execution/RenameTagExec.scala @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.execution + +import org.apache.paimon.spark.SparkTable +import org.apache.paimon.spark.leafnode.PaimonLeafV2CommandExec +import org.apache.paimon.table.FileStoreTable + +import org.apache.spark.sql.catalyst.InternalRow +import org.apache.spark.sql.catalyst.expressions.Attribute +import org.apache.spark.sql.connector.catalog.{Identifier, TableCatalog} + +case class RenameTagExec( + catalog: TableCatalog, + ident: Identifier, + sourceTag: String, + targetTag: String) + extends PaimonLeafV2CommandExec { + + override protected def run(): Seq[InternalRow] = { + val table = catalog.loadTable(ident) + assert(table.isInstanceOf[SparkTable]) + + table.asInstanceOf[SparkTable].getTable match { + case paimonTable: FileStoreTable => + paimonTable.renameTag(sourceTag, targetTag) + case t => + throw new UnsupportedOperationException( + s"Can not rename tag for non-paimon FileStoreTable: $t") + } + Nil + } + + override def output: Seq[Attribute] = Nil +} diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/execution/ShowTagsExec.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/execution/ShowTagsExec.scala new file mode 100644 index 000000000000..8f8d2b4665a5 --- /dev/null +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/execution/ShowTagsExec.scala @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.execution + +import org.apache.paimon.spark.SparkTable +import org.apache.paimon.spark.leafnode.PaimonLeafV2CommandExec +import org.apache.paimon.table.FileStoreTable + +import org.apache.spark.sql.catalyst.InternalRow +import org.apache.spark.sql.catalyst.expressions.Attribute +import org.apache.spark.sql.connector.catalog.{Identifier, TableCatalog} +import org.apache.spark.unsafe.types.UTF8String + +import scala.collection.JavaConverters._ + +case class ShowTagsExec(catalog: TableCatalog, ident: Identifier, out: Seq[Attribute]) + extends PaimonLeafV2CommandExec { + + override protected def run(): Seq[InternalRow] = { + val table = catalog.loadTable(ident) + assert(table.isInstanceOf[SparkTable]) + + var tags: Seq[InternalRow] = Nil + table.asInstanceOf[SparkTable].getTable match { + case paimonTable: FileStoreTable => + val tagNames = paimonTable.tagManager().allTagNames() + tags = tagNames.asScala.toList.sorted.map(t => InternalRow(UTF8String.fromString(t))) + case t => + throw new UnsupportedOperationException( + s"Can not show tags for non-paimon FileStoreTable: $t") + } + tags + } + + override def output: Seq[Attribute] = out +} diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/extensions/PaimonSparkSessionExtensions.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/extensions/PaimonSparkSessionExtensions.scala index 4fe217ee09bd..f73df64fb8ab 100644 --- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/extensions/PaimonSparkSessionExtensions.scala +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/extensions/PaimonSparkSessionExtensions.scala @@ -18,25 +18,30 @@ package org.apache.paimon.spark.extensions -import org.apache.paimon.spark.catalyst.analysis.{PaimonAnalysis, PaimonDeleteTable, PaimonIncompatiblePHRRules, PaimonIncompatibleResolutionRules, PaimonMergeInto, PaimonPostHocResolutionRules, PaimonProcedureResolver, PaimonUpdateTable} +import org.apache.paimon.spark.catalyst.analysis.{PaimonAnalysis, PaimonDeleteTable, PaimonIncompatiblePHRRules, PaimonIncompatibleResolutionRules, PaimonMergeInto, PaimonPostHocResolutionRules, PaimonProcedureResolver, PaimonUpdateTable, PaimonViewResolver} import org.apache.paimon.spark.catalyst.optimizer.{EvalSubqueriesForDeleteTable, MergePaimonScalarSubqueries} import org.apache.paimon.spark.catalyst.plans.logical.PaimonTableValuedFunctions import org.apache.paimon.spark.execution.PaimonStrategy import org.apache.paimon.spark.execution.adaptive.DisableUnnecessaryPaimonBucketedScan import org.apache.spark.sql.SparkSessionExtensions -import org.apache.spark.sql.catalyst.parser.extensions.PaimonSparkSqlExtensionsParser +import org.apache.spark.sql.paimon.shims.SparkShimLoader /** Spark session extension to extends the syntax and adds the rules. */ class PaimonSparkSessionExtensions extends (SparkSessionExtensions => Unit) { override def apply(extensions: SparkSessionExtensions): Unit = { // parser extensions - extensions.injectParser { case (_, parser) => new PaimonSparkSqlExtensionsParser(parser) } + extensions.injectParser { + case (_, parser) => SparkShimLoader.getSparkShim.createSparkParser(parser) + } // analyzer extensions extensions.injectResolutionRule(spark => new PaimonAnalysis(spark)) extensions.injectResolutionRule(spark => PaimonProcedureResolver(spark)) + extensions.injectResolutionRule(spark => PaimonViewResolver(spark)) + extensions.injectResolutionRule( + spark => SparkShimLoader.getSparkShim.createCustomResolution(spark)) extensions.injectResolutionRule(spark => PaimonIncompatibleResolutionRules(spark)) extensions.injectPostHocResolutionRule(spark => PaimonPostHocResolutionRules(spark)) diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/leafnode/package.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/leafnode/package.scala index 5befb88dae43..6ebab038480a 100644 --- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/leafnode/package.scala +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/leafnode/package.scala @@ -18,7 +18,7 @@ package org.apache.paimon.spark -import org.apache.spark.sql.catalyst.plans.logical.{LeafCommand, LeafParsedStatement} +import org.apache.spark.sql.catalyst.plans.logical.{BinaryCommand, LeafCommand, LeafParsedStatement, UnaryCommand} import org.apache.spark.sql.execution.command.LeafRunnableCommand import org.apache.spark.sql.execution.datasources.v2.LeafV2CommandExec @@ -30,6 +30,9 @@ package object leafnode { trait PaimonLeafCommand extends LeafCommand - trait PaimonLeafV2CommandExec extends LeafV2CommandExec + trait PaimonUnaryCommand extends UnaryCommand + + trait PaimonBinaryCommand extends BinaryCommand + trait PaimonLeafV2CommandExec extends LeafV2CommandExec } diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/orphan/SparkOrphanFilesClean.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/orphan/SparkOrphanFilesClean.scala index d79105e24eec..fca0493ede28 100644 --- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/orphan/SparkOrphanFilesClean.scala +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/orphan/SparkOrphanFilesClean.scala @@ -22,15 +22,14 @@ import org.apache.paimon.{utils, Snapshot} import org.apache.paimon.catalog.{Catalog, Identifier} import org.apache.paimon.fs.Path import org.apache.paimon.manifest.{ManifestEntry, ManifestFile} -import org.apache.paimon.operation.OrphanFilesClean +import org.apache.paimon.operation.{CleanOrphanFilesResult, OrphanFilesClean} import org.apache.paimon.operation.OrphanFilesClean.retryReadingFiles import org.apache.paimon.table.FileStoreTable import org.apache.paimon.utils.SerializableConsumer import org.apache.spark.internal.Logging -import org.apache.spark.sql.{Dataset, SparkSession} +import org.apache.spark.sql.{functions, Dataset, SparkSession} import org.apache.spark.sql.catalyst.SQLConfHelper -import org.apache.spark.sql.functions.sum import java.util import java.util.Collections @@ -50,19 +49,23 @@ case class SparkOrphanFilesClean( with SQLConfHelper with Logging { - def doOrphanClean(): (Dataset[Long], Dataset[BranchAndManifestFile]) = { + def doOrphanClean(): (Dataset[(Long, Long)], Dataset[BranchAndManifestFile]) = { import spark.implicits._ val branches = validBranches() - val deletedInLocal = new AtomicLong(0) + val deletedFilesCountInLocal = new AtomicLong(0) + val deletedFilesLenInBytesInLocal = new AtomicLong(0) // snapshot and changelog files are the root of everything, so they are handled specially // here, and subsequently, we will not count their orphan files. - cleanSnapshotDir(branches, (_: Path) => deletedInLocal.incrementAndGet) + cleanSnapshotDir( + branches, + (_: Path) => deletedFilesCountInLocal.incrementAndGet, + size => deletedFilesLenInBytesInLocal.addAndGet(size)) val maxBranchParallelism = Math.min(branches.size(), parallelism) // find snapshots using branch and find manifests(manifest, index, statistics) using snapshot val usedManifestFiles = spark.sparkContext - .parallelize(branches.asScala, maxBranchParallelism) + .parallelize(branches.asScala.toSeq, maxBranchParallelism) .mapPartitions(_.flatMap { branch => safelyGetAllSnapshots(branch).asScala.map(snapshot => (branch, snapshot.toJson)) }) @@ -114,17 +117,17 @@ case class SparkOrphanFilesClean( .toDF("used_name") // find candidate files which can be removed - val fileDirs = listPaimonFileDirs.asScala.map(_.toUri.toString) + val fileDirs = listPaimonFileDirs.asScala.map(_.toUri.toString).toSeq val maxFileDirsParallelism = Math.min(fileDirs.size, parallelism) val candidates = spark.sparkContext .parallelize(fileDirs, maxFileDirsParallelism) .flatMap { dir => tryBestListingDirs(new Path(dir)).asScala.filter(oldEnough).map { - file => (file.getPath.getName, file.getPath.toUri.toString) + file => (file.getPath.getName, file.getPath.toUri.toString, file.getLen) } } - .toDF("name", "path") + .toDF("name", "path", "len") .repartition(parallelism) // use left anti to filter files which is not used @@ -132,21 +135,30 @@ case class SparkOrphanFilesClean( .join(usedFiles, $"name" === $"used_name", "left_anti") .mapPartitions { it => - var deleted = 0L + var deletedFilesCount = 0L + var deletedFilesLenInBytes = 0L + while (it.hasNext) { - val pathToClean = it.next().getString(1) - specifiedFileCleaner.accept(new Path(pathToClean)) + val fileInfo = it.next(); + val pathToClean = fileInfo.getString(1) + val deletedPath = new Path(pathToClean) + deletedFilesLenInBytes += fileInfo.getLong(2) + specifiedFileCleaner.accept(deletedPath) logInfo(s"Cleaned file: $pathToClean") - deleted += 1 + deletedFilesCount += 1 } - logInfo(s"Total cleaned files: $deleted"); - Iterator.single(deleted) + logInfo( + s"Total cleaned files: $deletedFilesCount, Total cleaned files len : $deletedFilesLenInBytes") + Iterator.single((deletedFilesCount, deletedFilesLenInBytes)) + } + val finalDeletedDataset = + if (deletedFilesCountInLocal.get() != 0 || deletedFilesLenInBytesInLocal.get() != 0) { + deleted.union( + spark.createDataset( + Seq((deletedFilesCountInLocal.get(), deletedFilesLenInBytesInLocal.get())))) + } else { + deleted } - val finalDeletedDataset = if (deletedInLocal.get() != 0) { - deleted.union(spark.createDataset(Seq(deletedInLocal.get()))) - } else { - deleted - } (finalDeletedDataset, usedManifestFiles) } @@ -169,7 +181,7 @@ object SparkOrphanFilesClean extends SQLConfHelper { tableName: String, olderThanMillis: Long, fileCleaner: SerializableConsumer[Path], - parallelismOpt: Integer): Long = { + parallelismOpt: Integer): CleanOrphanFilesResult = { val spark = SparkSession.active val parallelism = if (parallelismOpt == null) { Math.max(spark.sparkContext.defaultParallelism, conf.numShufflePartitions) @@ -192,7 +204,7 @@ object SparkOrphanFilesClean extends SQLConfHelper { table.asInstanceOf[FileStoreTable] } if (tables.isEmpty) { - return 0 + return new CleanOrphanFilesResult(0, 0) } val (deleted, waitToRelease) = tables.map { table => @@ -207,15 +219,15 @@ object SparkOrphanFilesClean extends SQLConfHelper { try { val result = deleted .reduce((l, r) => l.union(r)) - .toDF("deleted") - .agg(sum("deleted")) + .toDF("deletedFilesCount", "deletedFilesLenInBytes") + .agg(functions.sum("deletedFilesCount"), functions.sum("deletedFilesLenInBytes")) .head() - assert(result.schema.size == 1, result.schema) + assert(result.schema.size == 2, result.schema) if (result.isNullAt(0)) { // no files can be deleted - 0 + new CleanOrphanFilesResult(0, 0) } else { - result.getLong(0) + new CleanOrphanFilesResult(result.getLong(0), result.getLong(1)) } } finally { waitToRelease.foreach(_.unpersist()) diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/sources/StreamHelper.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/sources/StreamHelper.scala index 91df04e6dc47..7e61d71ac183 100644 --- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/sources/StreamHelper.scala +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/sources/StreamHelper.scala @@ -44,17 +44,21 @@ private[spark] trait StreamHelper { var lastTriggerMillis: Long - private lazy val streamScan: StreamDataTableScan = table.newStreamScan() + private lazy val streamScan: StreamDataTableScan = + table.newStreamScan().dropStats().asInstanceOf[StreamDataTableScan] private lazy val partitionSchema: StructType = SparkTypeUtils.fromPaimonRowType(TypeUtils.project(table.rowType(), table.partitionKeys())) - private lazy val partitionComputer: InternalRowPartitionComputer = + private lazy val partitionComputer: InternalRowPartitionComputer = { + val options = new CoreOptions(table.options) new InternalRowPartitionComputer( - new CoreOptions(table.options).partitionDefaultName, + options.partitionDefaultName, TypeUtils.project(table.rowType(), table.partitionKeys()), - table.partitionKeys().asScala.toArray + table.partitionKeys().asScala.toArray, + options.legacyPartitionName() ) + } // Used to get the initial offset. lazy val streamScanStartingContext: StartingContext = streamScan.startingContext() diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/statistics/StatisticsHelperBase.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/statistics/StatisticsHelperBase.scala index 32fa48210c7f..627c6a168819 100644 --- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/statistics/StatisticsHelperBase.scala +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/statistics/StatisticsHelperBase.scala @@ -77,11 +77,11 @@ trait StatisticsHelperBase extends SQLConfHelper { private def toV2Stats(v1Stats: logical.Statistics): Statistics = { new Statistics() { override def sizeInBytes(): OptionalLong = if (v1Stats.sizeInBytes != null) - OptionalLong.of(v1Stats.sizeInBytes.longValue()) + OptionalLong.of(v1Stats.sizeInBytes.longValue) else OptionalLong.empty() override def numRows(): OptionalLong = if (v1Stats.rowCount.isDefined) - OptionalLong.of(v1Stats.rowCount.get.longValue()) + OptionalLong.of(v1Stats.rowCount.get.longValue) else OptionalLong.empty() override def columnStats(): java.util.Map[NamedReference, ColumnStatistics] = { diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/util/OptionUtils.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/util/OptionUtils.scala index af7ff7204cda..b60dd1fb2173 100644 --- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/util/OptionUtils.scala +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/util/OptionUtils.scala @@ -18,33 +18,61 @@ package org.apache.paimon.spark.util +import org.apache.paimon.catalog.Identifier import org.apache.paimon.table.Table import org.apache.spark.sql.catalyst.SQLConfHelper -import java.util.{HashMap => JHashMap, Map => JMap} +import java.util.{Map => JMap} +import java.util.regex.Pattern import scala.collection.JavaConverters._ object OptionUtils extends SQLConfHelper { private val PAIMON_OPTION_PREFIX = "spark.paimon." + private val SPARK_CATALOG_PREFIX = "spark.sql.catalog." - def mergeSQLConf(extraOptions: JMap[String, String]): JMap[String, String] = { - val mergedOptions = new JHashMap[String, String]( - conf.getAllConfs - .filterKeys(_.startsWith(PAIMON_OPTION_PREFIX)) - .map { - case (key, value) => - key.stripPrefix(PAIMON_OPTION_PREFIX) -> value - } - .asJava) + def extractCatalogName(): Option[String] = { + val sparkCatalogTemplate = String.format("%s([^.]*)$", SPARK_CATALOG_PREFIX) + val sparkCatalogPattern = Pattern.compile(sparkCatalogTemplate) + conf.getAllConfs.filterKeys(_.startsWith(SPARK_CATALOG_PREFIX)).foreach { + case (key, _) => + val matcher = sparkCatalogPattern.matcher(key) + if (matcher.find()) + return Option(matcher.group(1)) + } + Option.empty + } + + def mergeSQLConfWithIdentifier( + extraOptions: JMap[String, String], + catalogName: String, + ident: Identifier): JMap[String, String] = { + val tableOptionsTemplate = String.format( + "(%s)(%s|\\*)\\.(%s|\\*)\\.(%s|\\*)\\.(.+)", + PAIMON_OPTION_PREFIX, + catalogName, + ident.getDatabaseName, + ident.getObjectName) + val tableOptionsPattern = Pattern.compile(tableOptionsTemplate) + val mergedOptions = org.apache.paimon.options.OptionsUtils + .convertToDynamicTableProperties( + conf.getAllConfs.asJava, + PAIMON_OPTION_PREFIX, + tableOptionsPattern, + 5) mergedOptions.putAll(extraOptions) mergedOptions } - def copyWithSQLConf[T <: Table](table: T, extraOptions: JMap[String, String]): T = { - val mergedOptions = mergeSQLConf(extraOptions) + def copyWithSQLConf[T <: Table]( + table: T, + catalogName: String, + ident: Identifier, + extraOptions: JMap[String, String]): T = { + val mergedOptions: JMap[String, String] = + mergeSQLConfWithIdentifier(extraOptions, catalogName, ident) if (mergedOptions.isEmpty) { table } else { diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/spark/sql/PaimonUtils.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/spark/sql/PaimonUtils.scala index 4492d856ad50..cc49e787dc81 100644 --- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/spark/sql/PaimonUtils.scala +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/spark/sql/PaimonUtils.scala @@ -20,11 +20,15 @@ package org.apache.spark.sql import org.apache.spark.executor.OutputMetrics import org.apache.spark.rdd.InputFileBlockHolder +import org.apache.spark.sql.catalyst.analysis.Resolver +import org.apache.spark.sql.catalyst.catalog.CatalogTypes.TablePartitionSpec import org.apache.spark.sql.catalyst.expressions.{Attribute, Expression} import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan import org.apache.spark.sql.connector.expressions.{FieldReference, NamedReference} import org.apache.spark.sql.execution.datasources.DataSourceStrategy import org.apache.spark.sql.sources.Filter +import org.apache.spark.sql.types.StructType +import org.apache.spark.sql.util.PartitioningUtils import org.apache.spark.util.{Utils => SparkUtils} /** @@ -87,4 +91,19 @@ object PaimonUtils { outputMetrics.setBytesWritten(bytesWritten) outputMetrics.setRecordsWritten(recordsWritten) } + + def normalizePartitionSpec[T]( + partitionSpec: Map[String, T], + partCols: StructType, + tblName: String, + resolver: Resolver): Map[String, T] = { + PartitioningUtils.normalizePartitionSpec(partitionSpec, partCols, tblName, resolver) + } + + def requireExactMatchedPartitionSpec( + tableName: String, + spec: TablePartitionSpec, + partitionColumnNames: Seq[String]): Unit = { + PartitioningUtils.requireExactMatchedPartitionSpec(tableName, spec, partitionColumnNames) + } } diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/PaimonSparkSqlExtensionsParser.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/AbstractPaimonSparkSqlExtensionsParser.scala similarity index 88% rename from paimon-spark/paimon-spark-common/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/PaimonSparkSqlExtensionsParser.scala rename to paimon-spark/paimon-spark-common/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/AbstractPaimonSparkSqlExtensionsParser.scala index 26a351bc673a..557b0735c74d 100644 --- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/PaimonSparkSqlExtensionsParser.scala +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/AbstractPaimonSparkSqlExtensionsParser.scala @@ -18,12 +18,14 @@ package org.apache.spark.sql.catalyst.parser.extensions +import org.apache.paimon.spark.SparkProcedures + import org.antlr.v4.runtime._ import org.antlr.v4.runtime.atn.PredictionMode import org.antlr.v4.runtime.misc.{Interval, ParseCancellationException} import org.antlr.v4.runtime.tree.TerminalNodeImpl import org.apache.spark.internal.Logging -import org.apache.spark.sql.AnalysisException +import org.apache.spark.sql.{AnalysisException, SparkSession} import org.apache.spark.sql.catalyst.{FunctionIdentifier, TableIdentifier} import org.apache.spark.sql.catalyst.expressions.Expression import org.apache.spark.sql.catalyst.parser.{ParseException, ParserInterface} @@ -34,6 +36,8 @@ import org.apache.spark.sql.types.{DataType, StructType} import java.util.Locale +import scala.collection.JavaConverters._ + /* This file is based on source code from the Iceberg Project (http://iceberg.apache.org/), licensed by the Apache * Software Foundation (ASF) under the Apache License, Version 2.0. See the NOTICE file distributed with this work for * additional information regarding copyright ownership. */ @@ -47,8 +51,8 @@ import java.util.Locale * @param delegate * The extension parser. */ -class PaimonSparkSqlExtensionsParser(delegate: ParserInterface) - extends ParserInterface +abstract class AbstractPaimonSparkSqlExtensionsParser(val delegate: ParserInterface) + extends org.apache.spark.sql.catalyst.parser.ParserInterface with Logging { private lazy val substitutor = new VariableSubstitution() @@ -57,11 +61,11 @@ class PaimonSparkSqlExtensionsParser(delegate: ParserInterface) /** Parses a string to a LogicalPlan. */ override def parsePlan(sqlText: String): LogicalPlan = { val sqlTextAfterSubstitution = substitutor.substitute(sqlText) - if (isCommand(sqlTextAfterSubstitution)) { + if (isPaimonCommand(sqlTextAfterSubstitution)) { parse(sqlTextAfterSubstitution)(parser => astBuilder.visit(parser.singleStatement())) .asInstanceOf[LogicalPlan] } else { - delegate.parsePlan(sqlText) + RewritePaimonViewCommands(SparkSession.active).apply(delegate.parsePlan(sqlText)) } } @@ -93,15 +97,31 @@ class PaimonSparkSqlExtensionsParser(delegate: ParserInterface) delegate.parseMultipartIdentifier(sqlText) /** Returns whether SQL text is command. */ - private def isCommand(sqlText: String): Boolean = { + private def isPaimonCommand(sqlText: String): Boolean = { val normalized = sqlText .toLowerCase(Locale.ROOT) .trim() .replaceAll("--.*?\\n", " ") .replaceAll("\\s+", " ") .replaceAll("/\\*.*?\\*/", " ") + .replaceAll("`", "") .trim() - normalized.startsWith("call") + isPaimonProcedure(normalized) || isTagRefDdl(normalized) + } + + // All builtin paimon procedures are under the 'sys' namespace + private def isPaimonProcedure(normalized: String): Boolean = { + normalized.startsWith("call") && + SparkProcedures.names().asScala.map("sys." + _).exists(normalized.contains) + } + + private def isTagRefDdl(normalized: String): Boolean = { + normalized.startsWith("show tags") || + (normalized.startsWith("alter table") && + (normalized.contains("create tag") || + normalized.contains("replace tag") || + normalized.contains("rename tag") || + normalized.contains("delete tag"))) } protected def parse[T](command: String)(toResult: PaimonSqlExtensionsParser => T): T = { diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/PaimonSqlExtensionsAstBuilder.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/PaimonSqlExtensionsAstBuilder.scala index 397424c636d1..a1289a5f0b50 100644 --- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/PaimonSqlExtensionsAstBuilder.scala +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/PaimonSqlExtensionsAstBuilder.scala @@ -19,7 +19,8 @@ package org.apache.spark.sql.catalyst.parser.extensions import org.apache.paimon.spark.catalyst.plans.logical -import org.apache.paimon.spark.catalyst.plans.logical.{PaimonCallArgument, PaimonCallStatement, PaimonNamedArgument, PaimonPositionalArgument} +import org.apache.paimon.spark.catalyst.plans.logical._ +import org.apache.paimon.utils.TimeUtils import org.antlr.v4.runtime._ import org.antlr.v4.runtime.misc.Interval @@ -57,8 +58,8 @@ class PaimonSqlExtensionsAstBuilder(delegate: ParserInterface) /** Creates a [[PaimonCallStatement]] for a stored procedure call. */ override def visitCall(ctx: CallContext): PaimonCallStatement = withOrigin(ctx) { - val name = toSeq(ctx.multipartIdentifier.parts).map(_.getText) - val args = toSeq(ctx.callArgument).map(typedVisit[PaimonCallArgument]) + val name = ctx.multipartIdentifier.parts.asScala.map(_.getText).toSeq + val args = ctx.callArgument.asScala.map(typedVisit[PaimonCallArgument]).toSeq logical.PaimonCallStatement(name, args) } @@ -89,9 +90,69 @@ class PaimonSqlExtensionsAstBuilder(delegate: ParserInterface) /** Returns a multi-part identifier as Seq[String]. */ override def visitMultipartIdentifier(ctx: MultipartIdentifierContext): Seq[String] = withOrigin(ctx) { - ctx.parts.asScala.map(_.getText) + ctx.parts.asScala.map(_.getText).toSeq } + /** Create a SHOW TAGS logical command. */ + override def visitShowTags(ctx: ShowTagsContext): ShowTagsCommand = withOrigin(ctx) { + ShowTagsCommand(typedVisit[Seq[String]](ctx.multipartIdentifier)) + } + + /** Create a CREATE OR REPLACE TAG logical command. */ + override def visitCreateOrReplaceTag(ctx: CreateOrReplaceTagContext): CreateOrReplaceTagCommand = + withOrigin(ctx) { + val createTagClause = ctx.createReplaceTagClause() + + val tagName = createTagClause.identifier().getText + val tagOptionsContext = Option(createTagClause.tagOptions()) + val snapshotId = + tagOptionsContext + .flatMap(tagOptions => Option(tagOptions.snapshotId())) + .map(_.getText.toLong) + val timeRetainCtx = tagOptionsContext.flatMap(tagOptions => Option(tagOptions.timeRetain())) + val timeRetained = if (timeRetainCtx.nonEmpty) { + val (number, timeUnit) = + timeRetainCtx + .map(retain => (retain.number().getText.toLong, retain.timeUnit().getText)) + .get + Option(TimeUtils.parseDuration(number, timeUnit)) + } else { + None + } + val tagOptions = TagOptions( + snapshotId, + timeRetained + ) + + val create = createTagClause.CREATE() != null + val replace = createTagClause.REPLACE() != null + val ifNotExists = createTagClause.EXISTS() != null + + CreateOrReplaceTagCommand( + typedVisit[Seq[String]](ctx.multipartIdentifier), + tagName, + tagOptions, + create, + replace, + ifNotExists) + } + + /** Create a DELETE TAG logical command. */ + override def visitDeleteTag(ctx: DeleteTagContext): DeleteTagCommand = withOrigin(ctx) { + DeleteTagCommand( + typedVisit[Seq[String]](ctx.multipartIdentifier), + ctx.identifier().getText, + ctx.EXISTS() != null) + } + + /** Create a RENAME TAG logical command. */ + override def visitRenameTag(ctx: RenameTagContext): RenameTagCommand = withOrigin(ctx) { + RenameTagCommand( + typedVisit[Seq[String]](ctx.multipartIdentifier), + ctx.identifier(0).getText, + ctx.identifier(1).getText) + } + private def toBuffer[T](list: java.util.List[T]) = list.asScala private def toSeq[T](list: java.util.List[T]) = toBuffer(list) @@ -151,5 +212,16 @@ object CurrentOrigin { def get: Origin = value.get() def set(o: Origin): Unit = value.set(o) def reset(): Unit = value.set(Origin()) + + def withOrigin[A](o: Origin)(f: => A): A = { + // remember the previous one so it can be reset to this + // way withOrigin can be recursive + val previous = get + set(o) + val ret = + try f + finally { set(previous) } + ret + } } /* Apache Spark copy end */ diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/RewritePaimonViewCommands.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/RewritePaimonViewCommands.scala new file mode 100644 index 000000000000..f69e5d92038e --- /dev/null +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/RewritePaimonViewCommands.scala @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.catalyst.parser.extensions + +import org.apache.paimon.spark.catalog.SupportView +import org.apache.paimon.spark.catalyst.plans.logical.{CreatePaimonView, DropPaimonView, ResolvedIdentifier, ShowPaimonViews} + +import org.apache.spark.sql.SparkSession +import org.apache.spark.sql.catalyst.analysis.{CTESubstitution, ResolvedNamespace, UnresolvedIdentifier} +import org.apache.spark.sql.catalyst.plans.logical._ +import org.apache.spark.sql.catalyst.rules.Rule +import org.apache.spark.sql.connector.catalog.{CatalogManager, LookupCatalog} + +case class RewritePaimonViewCommands(spark: SparkSession) + extends Rule[LogicalPlan] + with LookupCatalog { + + protected lazy val catalogManager: CatalogManager = spark.sessionState.catalogManager + + override def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperatorsUp { + + // A new member was added to CreatePaimonView since spark4.0, + // unapply pattern matching is not used here to ensure compatibility across multiple spark versions. + case c: CreateView => + ResolvedIdent + .unapply(c.child) + .map { + resolved => + CreatePaimonView( + child = resolved, + queryText = c.originalText.get, + query = CTESubstitution.apply(c.query), + columnAliases = c.userSpecifiedColumns.map(_._1), + columnComments = c.userSpecifiedColumns.map(_._2.orElse(None)), + comment = c.comment, + properties = c.properties, + allowExisting = c.allowExisting, + replace = c.replace + ) + } + .getOrElse(c) + + case DropView(ResolvedIdent(resolved), ifExists: Boolean) => + DropPaimonView(resolved, ifExists) + + case ShowViews(_, pattern, output) if catalogManager.currentCatalog.isInstanceOf[SupportView] => + ShowPaimonViews( + ResolvedNamespace(catalogManager.currentCatalog, catalogManager.currentNamespace), + pattern, + output) + } + + private object ResolvedIdent { + def unapply(unresolved: Any): Option[ResolvedIdentifier] = unresolved match { + case UnresolvedIdentifier(CatalogAndIdentifier(viewCatalog: SupportView, ident), _) => + Some(ResolvedIdentifier(viewCatalog, ident)) + case _ => + None + } + } +} diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/spark/sql/connector/catalog/PaimonCatalogImplicits.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/spark/sql/connector/catalog/PaimonCatalogImplicits.scala new file mode 100644 index 000000000000..f1f20fb6fb31 --- /dev/null +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/spark/sql/connector/catalog/PaimonCatalogImplicits.scala @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.connector.catalog + +object PaimonCatalogImplicits { + + import CatalogV2Implicits._ + + implicit class PaimonCatalogHelper(plugin: CatalogPlugin) extends CatalogHelper(plugin) + + implicit class PaimonNamespaceHelper(namespace: Array[String]) extends NamespaceHelper(namespace) + +// implicit class PaimonTableHelper(table: Table) extends TableHelper(table) +} diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/spark/sql/connector/catalog/PaimonCatalogUtils.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/spark/sql/connector/catalog/PaimonCatalogUtils.scala index 265c82866195..5db6894ba093 100644 --- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/spark/sql/connector/catalog/PaimonCatalogUtils.scala +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/spark/sql/connector/catalog/PaimonCatalogUtils.scala @@ -22,11 +22,9 @@ import org.apache.hadoop.conf.Configuration import org.apache.spark.SparkConf import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.catalog.ExternalCatalog +import org.apache.spark.sql.connector.catalog.CatalogV2Util import org.apache.spark.sql.internal.StaticSQLConf.CATALOG_IMPLEMENTATION -import org.apache.spark.util.Utils - -import scala.reflect.ClassTag -import scala.util.control.NonFatal +import org.apache.spark.sql.paimon.ReflectUtils object PaimonCatalogUtils { @@ -37,22 +35,12 @@ object PaimonCatalogUtils { } else { "org.apache.spark.sql.catalyst.catalog.InMemoryCatalog" } - reflect[ExternalCatalog, SparkConf, Configuration](externalCatalogClassName, conf, hadoopConf) + ReflectUtils.reflect[ExternalCatalog, SparkConf, Configuration]( + externalCatalogClassName, + conf, + hadoopConf) } - private def reflect[T, Arg1 <: AnyRef, Arg2 <: AnyRef]( - className: String, - ctorArg1: Arg1, - ctorArg2: Arg2)(implicit ctorArgTag1: ClassTag[Arg1], ctorArgTag2: ClassTag[Arg2]): T = { - try { - val clazz = Utils.classForName(className) - val ctor = clazz.getDeclaredConstructor(ctorArgTag1.runtimeClass, ctorArgTag2.runtimeClass) - val args = Array[AnyRef](ctorArg1, ctorArg2) - ctor.newInstance(args: _*).asInstanceOf[T] - } catch { - case NonFatal(e) => - throw new IllegalArgumentException(s"Error while instantiating '$className':", e) - } - } + val TABLE_RESERVED_PROPERTIES: Seq[String] = CatalogV2Util.TABLE_RESERVED_PROPERTIES } diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/spark/sql/paimon/ReflectUtils.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/spark/sql/paimon/ReflectUtils.scala new file mode 100644 index 000000000000..bedac542ab8b --- /dev/null +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/spark/sql/paimon/ReflectUtils.scala @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.paimon + +import org.apache.spark.util.Utils + +import scala.reflect.ClassTag +import scala.util.control.NonFatal + +object ReflectUtils { + + def reflect[T, Arg1 <: AnyRef, Arg2 <: AnyRef](className: String, ctorArg1: Arg1, ctorArg2: Arg2)( + implicit + ctorArgTag1: ClassTag[Arg1], + ctorArgTag2: ClassTag[Arg2]): T = { + try { + val clazz = Utils.classForName(className) + val ctor = clazz.getDeclaredConstructor(ctorArgTag1.runtimeClass, ctorArgTag2.runtimeClass) + val args = Array[AnyRef](ctorArg1, ctorArg2) + ctor.newInstance(args: _*).asInstanceOf[T] + } catch { + case NonFatal(e) => + throw new IllegalArgumentException(s"Error while instantiating '$className':", e) + } + } + +} diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/spark/sql/paimon/shims/SparkShim.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/spark/sql/paimon/shims/SparkShim.scala new file mode 100644 index 000000000000..334bd6e93180 --- /dev/null +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/spark/sql/paimon/shims/SparkShim.scala @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.paimon.shims + +import org.apache.paimon.spark.data.{SparkArrayData, SparkInternalRow} +import org.apache.paimon.types.{DataType, RowType} + +import org.apache.spark.sql.{Column, SparkSession} +import org.apache.spark.sql.catalyst.expressions.{Attribute, Expression} +import org.apache.spark.sql.catalyst.parser.ParserInterface +import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan +import org.apache.spark.sql.catalyst.rules.Rule +import org.apache.spark.sql.connector.catalog.{Identifier, Table, TableCatalog} +import org.apache.spark.sql.connector.expressions.Transform +import org.apache.spark.sql.types.StructType + +import java.util.{Map => JMap} + +/** + * A spark shim trait. It declare methods which have incompatible implementations between Spark 3 + * and Spark 4. The specific SparkShim implementation will be loaded through Service Provider + * Interface. + */ +trait SparkShim { + + def createSparkParser(delegate: ParserInterface): ParserInterface + + def createCustomResolution(spark: SparkSession): Rule[LogicalPlan] + + def createSparkInternalRow(rowType: RowType): SparkInternalRow + + def createSparkArrayData(elementType: DataType): SparkArrayData + + def supportsHashAggregate( + aggregateBufferAttributes: Seq[Attribute], + groupingExpression: Seq[Expression]): Boolean + + def createTable( + tableCatalog: TableCatalog, + ident: Identifier, + schema: StructType, + partitions: Array[Transform], + properties: JMap[String, String]): Table + + def column(expr: Expression): Column + + def convertToExpression(spark: SparkSession, column: Column): Expression + +} diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/spark/sql/paimon/shims/SparkShimLoader.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/spark/sql/paimon/shims/SparkShimLoader.scala new file mode 100644 index 000000000000..920896547a1e --- /dev/null +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/spark/sql/paimon/shims/SparkShimLoader.scala @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.paimon.shims + +import java.util.ServiceLoader + +import scala.collection.JavaConverters._ + +/** Load a [[SparkShim]]'s implementation. */ +object SparkShimLoader { + + private lazy val sparkShim: SparkShim = loadSparkShim() + + def getSparkShim: SparkShim = { + sparkShim + } + + private def loadSparkShim(): SparkShim = { + val shims = ServiceLoader.load(classOf[SparkShim]).asScala + if (shims.isEmpty) { + throw new IllegalStateException("No available spark shim here.") + } else if (shims.size > 1) { + throw new IllegalStateException("Found more than one spark shim here.") + } + shims.head + } +} diff --git a/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/DDLWithHiveCatalogTestBase.scala b/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/DDLWithHiveCatalogTestBase.scala deleted file mode 100644 index a371f3e31be0..000000000000 --- a/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/DDLWithHiveCatalogTestBase.scala +++ /dev/null @@ -1,184 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.paimon.spark.sql - -import org.apache.paimon.spark.PaimonHiveTestBase -import org.apache.paimon.table.FileStoreTable - -import org.apache.spark.sql.{Row, SparkSession} -import org.junit.jupiter.api.Assertions - -abstract class DDLWithHiveCatalogTestBase extends PaimonHiveTestBase { - - test("Paimon DDL with hive catalog: create database with location and comment") { - Seq(sparkCatalogName, paimonHiveCatalogName).foreach { - catalogName => - spark.sql(s"USE $catalogName") - withTempDir { - dBLocation => - withDatabase("paimon_db") { - val comment = "this is a test comment" - spark.sql( - s"CREATE DATABASE paimon_db LOCATION '${dBLocation.getCanonicalPath}' COMMENT '$comment'") - Assertions.assertEquals(getDatabaseLocation("paimon_db"), dBLocation.getCanonicalPath) - Assertions.assertEquals(getDatabaseComment("paimon_db"), comment) - - withTable("paimon_db.paimon_tbl") { - spark.sql(s""" - |CREATE TABLE paimon_db.paimon_tbl (id STRING, name STRING, pt STRING) - |USING PAIMON - |TBLPROPERTIES ('primary-key' = 'id') - |""".stripMargin) - Assertions.assertEquals( - getTableLocation("paimon_db.paimon_tbl"), - s"${dBLocation.getCanonicalPath}/paimon_tbl") - - val fileStoreTable = getPaimonScan("SELECT * FROM paimon_db.paimon_tbl").table - .asInstanceOf[FileStoreTable] - Assertions.assertEquals("paimon_tbl", fileStoreTable.name()) - Assertions.assertEquals("paimon_db.paimon_tbl", fileStoreTable.fullName()) - } - } - } - } - } - - test("Paimon DDL with hive catalog: create database with props") { - Seq(sparkCatalogName, paimonHiveCatalogName).foreach { - catalogName => - spark.sql(s"USE $catalogName") - withDatabase("paimon_db") { - spark.sql(s"CREATE DATABASE paimon_db WITH DBPROPERTIES ('k1' = 'v1', 'k2' = 'v2')") - val props = getDatabaseProps("paimon_db") - Assertions.assertEquals(props("k1"), "v1") - Assertions.assertEquals(props("k2"), "v2") - } - } - } - - test("Paimon DDL with hive catalog: set default database") { - var reusedSpark = spark - - Seq("paimon", sparkCatalogName, paimonHiveCatalogName).foreach { - catalogName => - { - val dbName = s"${catalogName}_default_db" - val tblName = s"${dbName}_tbl" - - reusedSpark.sql(s"use $catalogName") - reusedSpark.sql(s"create database $dbName") - reusedSpark.sql(s"use $dbName") - reusedSpark.sql(s"create table $tblName (id int, name string, dt string) using paimon") - reusedSpark.stop() - - reusedSpark = SparkSession - .builder() - .master("local[2]") - .config(sparkConf) - .config("spark.sql.defaultCatalog", catalogName) - .config(s"spark.sql.catalog.$catalogName.defaultDatabase", dbName) - .getOrCreate() - - if (catalogName.equals(sparkCatalogName) && !gteqSpark3_4) { - checkAnswer(reusedSpark.sql("show tables").select("tableName"), Nil) - reusedSpark.sql(s"use $dbName") - } - checkAnswer(reusedSpark.sql("show tables").select("tableName"), Row(tblName) :: Nil) - - reusedSpark.sql(s"drop table $tblName") - } - } - - // Since we created a new sparkContext, we need to stop it and reset the default sparkContext - reusedSpark.stop() - reset() - } - - test("Paimon DDL with hive catalog: drop database cascade which contains paimon table") { - // Spark supports DROP DATABASE CASCADE since 3.3 - if (gteqSpark3_3) { - Seq(sparkCatalogName, paimonHiveCatalogName).foreach { - catalogName => - spark.sql(s"USE $catalogName") - spark.sql(s"CREATE DATABASE paimon_db") - spark.sql(s"USE paimon_db") - spark.sql(s"CREATE TABLE paimon_tbl (id int, name string, dt string) using paimon") - // Currently, only spark_catalog supports create other table or view - if (catalogName.equals(sparkCatalogName)) { - spark.sql(s"CREATE TABLE parquet_tbl (id int, name string, dt string) using parquet") - spark.sql(s"CREATE VIEW parquet_tbl_view AS SELECT * FROM parquet_tbl") - spark.sql(s"CREATE VIEW paimon_tbl_view AS SELECT * FROM paimon_tbl") - } - spark.sql(s"USE default") - spark.sql(s"DROP DATABASE paimon_db CASCADE") - } - } - } - - def getDatabaseLocation(dbName: String): String = { - spark - .sql(s"DESC DATABASE $dbName") - .filter("info_name == 'Location'") - .head() - .getAs[String]("info_value") - .split(":")(1) - } - - def getDatabaseComment(dbName: String): String = { - spark - .sql(s"DESC DATABASE $dbName") - .filter("info_name == 'Comment'") - .head() - .getAs[String]("info_value") - } - - def getDatabaseProps(dbName: String): Map[String, String] = { - val dbPropsStr = spark - .sql(s"DESC DATABASE EXTENDED $dbName") - .filter("info_name == 'Properties'") - .head() - .getAs[String]("info_value") - val pattern = "\\(([^,]+),([^)]+)\\)".r - pattern - .findAllIn(dbPropsStr.drop(1).dropRight(1)) - .matchData - .map { - m => - val key = m.group(1).trim - val value = m.group(2).trim - (key, value) - } - .toMap - } - - def getTableLocation(tblName: String): String = { - val tablePropsStr = spark - .sql(s"DESC TABLE EXTENDED $tblName") - .filter("col_name == 'Table Properties'") - .head() - .getAs[String]("data_type") - val tableProps = tablePropsStr - .substring(1, tablePropsStr.length - 1) - .split(",") - .map(_.split("=")) - .map { case Array(key, value) => (key, value) } - .toMap - tableProps("path").split(":")(1) - } -} diff --git a/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/PaimonOptionTest.scala b/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/PaimonOptionTest.scala deleted file mode 100644 index d35ac1d709c3..000000000000 --- a/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/PaimonOptionTest.scala +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.paimon.spark.sql - -import org.apache.paimon.spark.PaimonSparkTestBase -import org.apache.paimon.table.FileStoreTableFactory - -import org.apache.spark.sql.Row -import org.junit.jupiter.api.Assertions - -class PaimonOptionTest extends PaimonSparkTestBase { - - import testImplicits._ - - test("Paimon Option: create table with sql conf") { - withSQLConf("spark.paimon.scan.snapshot-id" -> "2") { - sql("CREATE TABLE T (id INT)") - val table = loadTable("T") - // check options in schema file directly - val fileStoreTable = FileStoreTableFactory.create(table.fileIO(), table.location()) - Assertions.assertNull(fileStoreTable.options().get("scan.snapshot-id")) - } - } - - test("Paimon Option: create table by dataframe with sql conf") { - withSQLConf("spark.paimon.scan.snapshot-id" -> "2") { - Seq((1L, "x1"), (2L, "x2")) - .toDF("a", "b") - .write - .format("paimon") - .mode("append") - .saveAsTable("T") - val table = loadTable("T") - // check options in schema file directly - val fileStoreTable = FileStoreTableFactory.create(table.fileIO(), table.location()) - Assertions.assertNull(fileStoreTable.options().get("scan.snapshot-id")) - } - } - - test("Paimon Option: query table with sql conf") { - sql("CREATE TABLE T (id INT)") - sql("INSERT INTO T VALUES 1") - sql("INSERT INTO T VALUES 2") - checkAnswer(sql("SELECT * FROM T ORDER BY id"), Row(1) :: Row(2) :: Nil) - val table = loadTable("T") - - // query with mutable option - withSQLConf("spark.paimon.scan.snapshot-id" -> "1") { - checkAnswer(sql("SELECT * FROM T ORDER BY id"), Row(1)) - checkAnswer(spark.read.format("paimon").load(table.location().toString), Row(1)) - } - - // query with immutable option - withSQLConf("spark.paimon.bucket" -> "1") { - assertThrows[UnsupportedOperationException] { - sql("SELECT * FROM T ORDER BY id") - } - assertThrows[UnsupportedOperationException] { - spark.read.format("paimon").load(table.location().toString) - } - } - } -} diff --git a/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/TableValuedFunctionsTest.scala b/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/TableValuedFunctionsTest.scala deleted file mode 100644 index 2a689b631acd..000000000000 --- a/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/TableValuedFunctionsTest.scala +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.paimon.spark.sql - -import org.apache.paimon.spark.PaimonSparkTestBase - -import org.apache.spark.sql.{DataFrame, Row} - -class TableValuedFunctionsTest extends PaimonSparkTestBase { - - withPk.foreach { - hasPk => - bucketModes.foreach { - bucket => - test(s"incremental query: hasPk: $hasPk, bucket: $bucket") { - val prop = if (hasPk) { - s"'primary-key'='a,b', 'bucket' = '$bucket' " - } else if (bucket != -1) { - s"'bucket-key'='b', 'bucket' = '$bucket' " - } else { - "'write-only'='true'" - } - - spark.sql(s""" - |CREATE TABLE T (a INT, b INT, c STRING) - |USING paimon - |TBLPROPERTIES ($prop) - |PARTITIONED BY (a) - |""".stripMargin) - - spark.sql("INSERT INTO T values (1, 1, '1'), (2, 2, '2')") - spark.sql("INSERT INTO T VALUES (1, 3, '3'), (2, 4, '4')") - spark.sql("INSERT INTO T VALUES (1, 5, '5'), (1, 7, '7')") - - checkAnswer( - incrementalDF("T", 0, 1).orderBy("a", "b"), - Row(1, 1, "1") :: Row(2, 2, "2") :: Nil) - checkAnswer( - spark.sql("SELECT * FROM paimon_incremental_query('T', '0', '1') ORDER BY a, b"), - Row(1, 1, "1") :: Row(2, 2, "2") :: Nil) - - checkAnswer( - incrementalDF("T", 1, 2).orderBy("a", "b"), - Row(1, 3, "3") :: Row(2, 4, "4") :: Nil) - checkAnswer( - spark.sql("SELECT * FROM paimon_incremental_query('T', '1', '2') ORDER BY a, b"), - Row(1, 3, "3") :: Row(2, 4, "4") :: Nil) - - checkAnswer( - incrementalDF("T", 2, 3).orderBy("a", "b"), - Row(1, 5, "5") :: Row(1, 7, "7") :: Nil) - checkAnswer( - spark.sql("SELECT * FROM paimon_incremental_query('T', '2', '3') ORDER BY a, b"), - Row(1, 5, "5") :: Row(1, 7, "7") :: Nil) - - checkAnswer( - incrementalDF("T", 1, 3).orderBy("a", "b"), - Row(1, 3, "3") :: Row(1, 5, "5") :: Row(1, 7, "7") :: Row(2, 4, "4") :: Nil - ) - checkAnswer( - spark.sql("SELECT * FROM paimon_incremental_query('T', '1', '3') ORDER BY a, b"), - Row(1, 3, "3") :: Row(1, 5, "5") :: Row(1, 7, "7") :: Row(2, 4, "4") :: Nil) - } - } - } - - private def incrementalDF(tableIdent: String, start: Int, end: Int): DataFrame = { - spark.read - .format("paimon") - .option("incremental-between", s"$start,$end") - .table(tableIdent) - } -} diff --git a/paimon-spark/paimon-spark-ut/pom.xml b/paimon-spark/paimon-spark-ut/pom.xml new file mode 100644 index 000000000000..0a1840596487 --- /dev/null +++ b/paimon-spark/paimon-spark-ut/pom.xml @@ -0,0 +1,180 @@ + + + + 4.0.0 + + + org.apache.paimon + paimon-spark + 1.0-SNAPSHOT + + + paimon-spark-ut + Paimon : Spark : UT + + + ${paimon-spark-common.spark.version} + ${paimon.shade.jackson.version} + + + + + org.apache.paimon + ${paimon-sparkx-common} + ${project.version} + test + + + + org.apache.paimon + paimon-spark-common_${scala.binary.version} + ${project.version} + test + + + + org.apache.spark + spark-core_${scala.binary.version} + ${spark.version} + test + + + com.fasterxml.jackson.core + * + + + com.fasterxml.jackson.module + * + + + + + + org.apache.spark + spark-sql_${scala.binary.version} + ${spark.version} + test + + + com.fasterxml.jackson.core + * + + + + + + org.apache.spark + spark-sql_${scala.binary.version} + ${spark.version} + tests + test + + + com.fasterxml.jackson.core + * + + + + + + org.apache.spark + spark-catalyst_${scala.binary.version} + ${spark.version} + tests + test + + + + org.apache.spark + spark-core_${scala.binary.version} + ${spark.version} + tests + test + + + com.fasterxml.jackson.core + * + + + com.fasterxml.jackson.module + * + + + + + + org.apache.spark + spark-hive_${scala.binary.version} + ${spark.version} + test + + + com.fasterxml.jackson.core + * + + + com.google.protobuf + protobuf-java + + + + + + org.apache.spark + spark-avro_${scala.binary.version} + ${spark.version} + test + + + + com.fasterxml.jackson.module + jackson-module-scala_${scala.binary.version} + ${jackson.version} + test + + + + com.google.protobuf + protobuf-java + ${protobuf-java.version} + + + + + + + org.apache.maven.plugins + maven-jar-plugin + + + prepare-test-jar + test-compile + + test-jar + + + + + + + + diff --git a/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/SparkCatalogWithHiveTest.java b/paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/SparkCatalogWithHiveTest.java similarity index 58% rename from paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/SparkCatalogWithHiveTest.java rename to paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/SparkCatalogWithHiveTest.java index 22811af39d0c..45ccd06479f2 100644 --- a/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/SparkCatalogWithHiveTest.java +++ b/paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/SparkCatalogWithHiveTest.java @@ -19,18 +19,27 @@ package org.apache.paimon.spark; import org.apache.paimon.fs.Path; +import org.apache.paimon.fs.local.LocalFileIO; import org.apache.paimon.hive.TestHiveMetastore; +import org.apache.paimon.table.FileStoreTableFactory; +import org.apache.spark.sql.Row; import org.apache.spark.sql.SparkSession; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; +import java.io.FileNotFoundException; +import java.util.stream.Collectors; + import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatCode; +import static org.assertj.core.api.Assertions.assertThatThrownBy; /** Base tests for spark read. */ public class SparkCatalogWithHiveTest { + private static TestHiveMetastore testHiveMetastore; private static final int PORT = 9087; @@ -48,7 +57,6 @@ public static void closeMetastore() throws Exception { @Test public void testCreateFormatTable(@TempDir java.nio.file.Path tempDir) { - // firstly, we use hive metastore to creata table, and check the result. Path warehousePath = new Path("file:" + tempDir.toString()); SparkSession spark = SparkSession.builder() @@ -70,6 +78,9 @@ public void testCreateFormatTable(@TempDir java.nio.file.Path tempDir) { spark.sql("CREATE DATABASE IF NOT EXISTS my_db1"); spark.sql("USE spark_catalog.my_db1"); + + // test orc table + spark.sql("CREATE TABLE IF NOT EXISTS table_orc (a INT, bb INT, c STRING) USING orc"); assertThat( @@ -77,9 +88,62 @@ public void testCreateFormatTable(@TempDir java.nio.file.Path tempDir) { .map(s -> s.get(1)) .map(Object::toString)) .containsExactlyInAnyOrder("table_orc"); + + assertThat( + spark.sql("EXPLAIN EXTENDED SELECT * from table_orc").collectAsList() + .stream() + .map(s -> s.get(0)) + .map(Object::toString) + .filter(s -> s.contains("OrcScan")) + .count()) + .isGreaterThan(0); + + // todo: There are some bugs with Spark CSV table's options. In Spark 3.x, both reading and + // writing using the default delimiter value ',' even if we specific it. In Spark 4.x, + // reading is correct, but writing is still incorrect, just skip setting it for now. + // test csv table + + spark.sql( + "CREATE TABLE IF NOT EXISTS table_csv (a INT, bb INT, c STRING) USING csv OPTIONS ('field-delimiter' ',')"); + spark.sql("INSERT INTO table_csv VALUES (1, 1, '1'), (2, 2, '2')").collect(); + assertThat(spark.sql("DESCRIBE FORMATTED table_csv").collectAsList().toString()) + .contains("sep=,"); + assertThat( + spark.sql("SELECT * FROM table_csv").collectAsList().stream() + .map(Row::toString) + .collect(Collectors.toList())) + .containsExactlyInAnyOrder("[1,1,1]", "[2,2,2]"); + spark.close(); + } - SparkSession spark1 = + @Test + public void testSpecifyHiveConfDir(@TempDir java.nio.file.Path tempDir) { + Path warehousePath = new Path("file:" + tempDir.toString()); + SparkSession spark = + SparkSession.builder() + .config("spark.sql.catalog.spark_catalog.hive-conf-dir", "nonExistentPath") + .config("spark.sql.warehouse.dir", warehousePath.toString()) + // with hive metastore + .config("spark.sql.catalogImplementation", "hive") + .config( + "spark.sql.catalog.spark_catalog", + SparkGenericCatalog.class.getName()) + .master("local[2]") + .getOrCreate(); + + assertThatThrownBy(() -> spark.sql("CREATE DATABASE my_db")) + .rootCause() + .isInstanceOf(FileNotFoundException.class) + .hasMessageContaining("nonExistentPath"); + + spark.close(); + } + + @Test + public void testCreateExternalTable(@TempDir java.nio.file.Path tempDir) { + Path warehousePath = new Path("file:" + tempDir.toString()); + SparkSession spark = SparkSession.builder() .config("spark.sql.warehouse.dir", warehousePath.toString()) // with hive metastore @@ -90,21 +154,31 @@ public void testCreateFormatTable(@TempDir java.nio.file.Path tempDir) { .config( "spark.sql.catalog.spark_catalog.hive.metastore.uris", "thrift://localhost:" + PORT) - .config("spark.sql.catalog.spark_catalog.format-table.enabled", "true") .config( "spark.sql.catalog.spark_catalog.warehouse", warehousePath.toString()) .master("local[2]") .getOrCreate(); - spark1.sql("USE spark_catalog.my_db1"); - assertThat( - spark1.sql("EXPLAIN EXTENDED SELECT * from table_orc").collectAsList() - .stream() - .map(s -> s.get(0)) - .map(Object::toString) - .filter(s -> s.contains("OrcScan")) - .count()) - .isGreaterThan(0); - spark1.close(); + + spark.sql("CREATE DATABASE IF NOT EXISTS test_db"); + spark.sql("USE spark_catalog.test_db"); + + // create hive external table + spark.sql("CREATE EXTERNAL TABLE t1 (a INT, bb INT, c STRING)"); + + // drop hive external table + spark.sql("DROP TABLE t1"); + + // file system table exists + assertThatCode( + () -> + FileStoreTableFactory.create( + LocalFileIO.create(), + new Path( + warehousePath, + String.format("%s.db/%s", "test_db", "t1")))) + .doesNotThrowAnyException(); + + spark.close(); } } diff --git a/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/SparkFileIndexITCase.java b/paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/SparkFileIndexITCase.java similarity index 75% rename from paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/SparkFileIndexITCase.java rename to paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/SparkFileIndexITCase.java index 7af83186ef6e..0360def685b6 100644 --- a/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/SparkFileIndexITCase.java +++ b/paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/SparkFileIndexITCase.java @@ -27,7 +27,7 @@ import org.apache.paimon.fileindex.FileIndexFormat; import org.apache.paimon.fileindex.FileIndexReader; import org.apache.paimon.fileindex.FileIndexResult; -import org.apache.paimon.fileindex.bitmap.BitmapIndexResultLazy; +import org.apache.paimon.fileindex.bitmap.BitmapIndexResult; import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.Path; import org.apache.paimon.fs.local.LocalFileIO; @@ -103,22 +103,49 @@ public void testReadWriteTableWithBitmapIndex() throws Catalog.TableNotExistExce + "'file-index.in-manifest-threshold'='1B');"); spark.sql("INSERT INTO T VALUES (0),(1),(2),(3),(4),(5);"); + List rows1 = spark.sql("SELECT a FROM T where a>3;").collectAsList(); + assertThat(rows1.toString()).isEqualTo("[[4], [5]]"); + // check query result - List rows = spark.sql("SELECT a FROM T where a='3';").collectAsList(); - assertThat(rows.toString()).isEqualTo("[[3]]"); + List rows2 = spark.sql("SELECT a FROM T where a=3;").collectAsList(); + assertThat(rows2.toString()).isEqualTo("[[3]]"); // check index reader foreachIndexReader( fileIndexReader -> { FileIndexResult fileIndexResult = fileIndexReader.visitEqual(new FieldRef(0, "", new IntType()), 3); - assert fileIndexResult instanceof BitmapIndexResultLazy; - RoaringBitmap32 roaringBitmap32 = - ((BitmapIndexResultLazy) fileIndexResult).get(); + assert fileIndexResult instanceof BitmapIndexResult; + RoaringBitmap32 roaringBitmap32 = ((BitmapIndexResult) fileIndexResult).get(); assert roaringBitmap32.equals(RoaringBitmap32.bitmapOf(3)); }); } + @Test + public void testReadWriteTableWithBitSliceIndex() throws Catalog.TableNotExistException { + + spark.sql( + "CREATE TABLE T(a int) TBLPROPERTIES (" + + "'file-index.bsi.columns'='a'," + + "'file-index.in-manifest-threshold'='1B');"); + spark.sql("INSERT INTO T VALUES (0),(1),(2),(3),(4),(5);"); + + // check query result + List rows = spark.sql("SELECT a FROM T where a>=3;").collectAsList(); + assertThat(rows.toString()).isEqualTo("[[3], [4], [5]]"); + + // check index reader + foreachIndexReader( + fileIndexReader -> { + FileIndexResult fileIndexResult = + fileIndexReader.visitGreaterOrEqual( + new FieldRef(0, "", new IntType()), 3); + assertThat(fileIndexResult).isInstanceOf(BitmapIndexResult.class); + RoaringBitmap32 roaringBitmap32 = ((BitmapIndexResult) fileIndexResult).get(); + assertThat(roaringBitmap32).isEqualTo(RoaringBitmap32.bitmapOf(3, 4, 5)); + }); + } + protected void foreachIndexReader(Consumer consumer) throws Catalog.TableNotExistException { Path tableRoot = fileSystemCatalog.getTableLocation(Identifier.create("db", "T")); @@ -128,9 +155,13 @@ protected void foreachIndexReader(Consumer consumer) tableRoot, RowType.of(), new CoreOptions(new Options()).partitionDefaultName(), - CoreOptions.FILE_FORMAT.defaultValue().toString(), + CoreOptions.FILE_FORMAT.defaultValue(), CoreOptions.DATA_FILE_PREFIX.defaultValue(), - CoreOptions.CHANGELOG_FILE_PREFIX.defaultValue()); + CoreOptions.CHANGELOG_FILE_PREFIX.defaultValue(), + CoreOptions.PARTITION_GENERATE_LEGCY_NAME.defaultValue(), + CoreOptions.FILE_SUFFIX_INCLUDE_COMPRESSION.defaultValue(), + CoreOptions.FILE_COMPRESSION.defaultValue(), + null); Table table = fileSystemCatalog.getTable(Identifier.create("db", "T")); ReadBuilder readBuilder = table.newReadBuilder(); @@ -151,12 +182,11 @@ protected void foreachIndexReader(Consumer consumer) .collect(Collectors.toList()); // assert index file exist and only one index file assert indexFiles.size() == 1; - try { - FileIndexFormat.Reader reader = - FileIndexFormat.createReader( - fileIO.newInputStream( - dataFilePathFactory.toPath(indexFiles.get(0))), - tableSchema.logicalRowType()); + try (FileIndexFormat.Reader reader = + FileIndexFormat.createReader( + fileIO.newInputStream( + dataFilePathFactory.toPath(indexFiles.get(0))), + tableSchema.logicalRowType())) { Optional fileIndexReader = reader.readColumnIndex("a").stream().findFirst(); // assert index reader exist diff --git a/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/SparkFilterConverterTest.java b/paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/SparkFilterConverterTest.java similarity index 100% rename from paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/SparkFilterConverterTest.java rename to paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/SparkFilterConverterTest.java diff --git a/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/SparkGenericCatalogTest.java b/paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/SparkGenericCatalogTest.java similarity index 100% rename from paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/SparkGenericCatalogTest.java rename to paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/SparkGenericCatalogTest.java diff --git a/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/SparkGenericCatalogWithHiveTest.java b/paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/SparkGenericCatalogWithHiveTest.java similarity index 84% rename from paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/SparkGenericCatalogWithHiveTest.java rename to paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/SparkGenericCatalogWithHiveTest.java index b0f1749dfeb3..84ea1ab5cba2 100644 --- a/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/SparkGenericCatalogWithHiveTest.java +++ b/paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/SparkGenericCatalogWithHiveTest.java @@ -27,7 +27,10 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; +import java.io.FileNotFoundException; + import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.junit.jupiter.api.Assertions.assertThrows; /** Base tests for spark read. */ @@ -100,7 +103,7 @@ public void testCreateTableCaseSensitive(@TempDir java.nio.file.Path tempDir) { @Test public void testBuildWithHive(@TempDir java.nio.file.Path tempDir) { - // firstly, we use hive metastore to creata table, and check the result. + // firstly, we use hive metastore to create table, and check the result. Path warehousePath = new Path("file:" + tempDir.toString()); SparkSession spark = SparkSession.builder() @@ -148,4 +151,27 @@ public void testBuildWithHive(@TempDir java.nio.file.Path tempDir) { .map(Object::toString)) .containsExactlyInAnyOrder("t1"); } + + @Test + public void testHiveCatalogOptions(@TempDir java.nio.file.Path tempDir) { + Path warehousePath = new Path("file:" + tempDir.toString()); + SparkSession spark = + SparkSession.builder() + .config("spark.sql.catalog.spark_catalog.hive-conf-dir", "nonExistentPath") + .config("spark.sql.warehouse.dir", warehousePath.toString()) + // with hive metastore + .config("spark.sql.catalogImplementation", "hive") + .config( + "spark.sql.catalog.spark_catalog", + SparkGenericCatalog.class.getName()) + .master("local[2]") + .getOrCreate(); + + assertThatThrownBy(() -> spark.sql("CREATE DATABASE my_db")) + .rootCause() + .isInstanceOf(FileNotFoundException.class) + .hasMessageContaining("nonExistentPath"); + + spark.close(); + } } diff --git a/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/SparkInternalRowTest.java b/paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/SparkInternalRowTest.java similarity index 89% rename from paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/SparkInternalRowTest.java rename to paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/SparkInternalRowTest.java index 9af886d8369f..1117ad58c737 100644 --- a/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/SparkInternalRowTest.java +++ b/paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/SparkInternalRowTest.java @@ -25,6 +25,7 @@ import org.apache.paimon.data.GenericRow; import org.apache.paimon.data.InternalRow; import org.apache.paimon.data.Timestamp; +import org.apache.paimon.spark.data.SparkInternalRow; import org.apache.paimon.utils.DateTimeUtils; import org.apache.spark.sql.catalyst.CatalystTypeConverters; @@ -95,7 +96,7 @@ public void test() { SparkTypeUtils.fromPaimonType(ALL_TYPES))); org.apache.spark.sql.Row sparkRow = (org.apache.spark.sql.Row) - sparkConverter.apply(new SparkInternalRow(ALL_TYPES).replace(rowData)); + sparkConverter.apply(SparkInternalRow.create(ALL_TYPES).replace(rowData)); String expected = "1," @@ -104,8 +105,8 @@ public void test() { + "paimon," + "22.2," + "Map(key2 -> [2.4,3.5], key1 -> [1.2,2.3])," - + "WrappedArray(v1, v5)," - + "WrappedArray(10, 30)," + + "[v1, v5]," + + "[10, 30]," + "true," + "22," + "356," @@ -122,13 +123,20 @@ public void test() { SparkRow sparkRowData = new SparkRow(ALL_TYPES, sparkRow); sparkRow = (org.apache.spark.sql.Row) - sparkConverter.apply(new SparkInternalRow(ALL_TYPES).replace(sparkRowData)); + sparkConverter.apply( + SparkInternalRow.create(ALL_TYPES).replace(sparkRowData)); assertThat(sparkRowToString(sparkRow)).isEqualTo(expected); TimeZone.setDefault(tz); } private String sparkRowToString(org.apache.spark.sql.Row row) { return JavaConverters.seqAsJavaList(row.toSeq()).stream() + .map( + x -> + (x instanceof scala.collection.Seq) + ? JavaConverters.seqAsJavaList( + (scala.collection.Seq) x) + : x) .map(Object::toString) // Since the toString result of Spark's binary col is unstable, replace it .map(x -> x.startsWith("[B@") ? "[B@" : x) diff --git a/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/SparkReadITCase.java b/paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/SparkReadITCase.java similarity index 91% rename from paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/SparkReadITCase.java rename to paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/SparkReadITCase.java index be6264f7b2d0..b00267410a7f 100644 --- a/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/SparkReadITCase.java +++ b/paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/SparkReadITCase.java @@ -190,17 +190,20 @@ public void testCreateTableAs() { spark.sql("INSERT INTO partitionedTable VALUES(1,'aaa','bbb')"); spark.sql( "CREATE TABLE partitionedTableAs PARTITIONED BY (a) AS SELECT * FROM partitionedTable"); + Path tablePath = new Path(warehousePath, "default.db/partitionedTableAs"); assertThat(spark.sql("SHOW CREATE TABLE partitionedTableAs").collectAsList().toString()) .isEqualTo( String.format( "[[%s" + "PARTITIONED BY (a)\n" + + "LOCATION '%s'\n" + "TBLPROPERTIES (\n" + " 'path' = '%s')\n" + "]]", showCreateString( "partitionedTableAs", "a BIGINT", "b STRING", "c STRING"), - new Path(warehousePath, "default.db/partitionedTableAs"))); + tablePath, + tablePath)); List resultPartition = spark.sql("SELECT * FROM partitionedTableAs").collectAsList(); assertThat(resultPartition.stream().map(Row::toString)) .containsExactlyInAnyOrder("[1,aaa,bbb]"); @@ -217,17 +220,20 @@ public void testCreateTableAs() { spark.sql("INSERT INTO testTable VALUES(1,'a','b')"); spark.sql( "CREATE TABLE testTableAs TBLPROPERTIES ('file.format' = 'parquet') AS SELECT * FROM testTable"); + tablePath = new Path(warehousePath, "default.db/testTableAs"); assertThat(spark.sql("SHOW CREATE TABLE testTableAs").collectAsList().toString()) .isEqualTo( String.format( "[[%s" + + "LOCATION '%s'\n" + "TBLPROPERTIES (\n" + " 'file.format' = 'parquet',\n" + " 'path' = '%s')\n" + "]]", showCreateString( "testTableAs", "a BIGINT", "b VARCHAR(10)", "c CHAR(10)"), - new Path(warehousePath, "default.db/testTableAs"))); + tablePath, + tablePath)); List resultProp = spark.sql("SELECT * FROM testTableAs").collectAsList(); assertThat(resultProp.stream().map(Row::toString)) @@ -245,13 +251,17 @@ public void testCreateTableAs() { + "COMMENT 'table comment'"); spark.sql("INSERT INTO t_pk VALUES(1,'aaa','bbb')"); spark.sql("CREATE TABLE t_pk_as TBLPROPERTIES ('primary-key' = 'a') AS SELECT * FROM t_pk"); + tablePath = new Path(warehousePath, "default.db/t_pk_as"); assertThat(spark.sql("SHOW CREATE TABLE t_pk_as").collectAsList().toString()) .isEqualTo( String.format( - "[[%sTBLPROPERTIES (\n 'path' = '%s',\n 'primary-key' = 'a')\n]]", + "[[%s" + + "LOCATION '%s'\n" + + "TBLPROPERTIES (\n 'path' = '%s',\n 'primary-key' = 'a')\n]]", showCreateString( "t_pk_as", "a BIGINT NOT NULL", "b STRING", "c STRING"), - new Path(warehousePath, "default.db/t_pk_as"))); + tablePath, + tablePath)); List resultPk = spark.sql("SELECT * FROM t_pk_as").collectAsList(); assertThat(resultPk.stream().map(Row::toString)).containsExactlyInAnyOrder("[1,aaa,bbb]"); @@ -270,11 +280,13 @@ public void testCreateTableAs() { spark.sql("INSERT INTO t_all VALUES(1,2,'bbb','2020-01-01','12')"); spark.sql( "CREATE TABLE t_all_as PARTITIONED BY (dt) TBLPROPERTIES ('primary-key' = 'dt,hh') AS SELECT * FROM t_all"); + tablePath = new Path(warehousePath, "default.db/t_all_as"); assertThat(spark.sql("SHOW CREATE TABLE t_all_as").collectAsList().toString()) .isEqualTo( String.format( "[[%s" + "PARTITIONED BY (dt)\n" + + "LOCATION '%s'\n" + "TBLPROPERTIES (\n" + " 'path' = '%s',\n" + " 'primary-key' = 'dt,hh')\n" @@ -286,7 +298,8 @@ public void testCreateTableAs() { "behavior STRING", "dt STRING NOT NULL", "hh STRING NOT NULL"), - new Path(warehousePath, "default.db/t_all_as"))); + tablePath, + tablePath)); List resultAll = spark.sql("SELECT * FROM t_all_as").collectAsList(); assertThat(resultAll.stream().map(Row::toString)) .containsExactlyInAnyOrder("[1,2,bbb,2020-01-01,12]"); @@ -363,12 +376,14 @@ public void testShowCreateTable() { + " 'k1' = 'v1'\n" + ")"); + Path tablePath = new Path(warehousePath, "default.db/tbl"); assertThat(spark.sql("SHOW CREATE TABLE tbl").collectAsList().toString()) .isEqualTo( String.format( "[[%s" + "PARTITIONED BY (b)\n" + "COMMENT 'tbl comment'\n" + + "LOCATION '%s'\n" + "TBLPROPERTIES (\n" + " 'k1' = 'v1',\n" + " 'path' = '%s',\n" @@ -377,7 +392,8 @@ public void testShowCreateTable() { "tbl", "a INT NOT NULL COMMENT 'a comment'", "b STRING NOT NULL"), - new Path(warehousePath, "default.db/tbl"))); + tablePath, + tablePath)); } @Test @@ -440,6 +456,38 @@ public void testCreateAndDropTable() { innerTest("MyTable6", false, true); } + @Test + public void testReadNestedColumnTable() { + String tableName = "testAddNestedColumnTable"; + spark.sql( + "CREATE TABLE paimon.default." + + tableName + + " (k INT NOT NULL, v STRUCT>) " + + "TBLPROPERTIES ('bucket' = '1', 'primary-key' = 'k', 'file.format' = 'parquet')"); + spark.sql( + "INSERT INTO paimon.default." + + tableName + + " VALUES (1, STRUCT(10, STRUCT('apple', 100)))"); + spark.sql( + "INSERT INTO paimon.default." + + tableName + + " VALUES (2, STRUCT(20, STRUCT('banana', 200)))"); + assertThat( + spark.sql("SELECT v.f2.f1, k FROM paimon.default." + tableName) + .collectAsList().stream() + .map(Row::toString)) + .containsExactlyInAnyOrder("[apple,1]", "[banana,2]"); + spark.sql( + "INSERT INTO paimon.default." + + tableName + + " VALUES (1, STRUCT(30, STRUCT('cat', 100)))"); + assertThat( + spark.sql("SELECT v.f2.f1, k FROM paimon.default." + tableName) + .collectAsList().stream() + .map(Row::toString)) + .containsExactlyInAnyOrder("[cat,1]", "[banana,2]"); + } + private void innerTest(String tableName, boolean hasPk, boolean partitioned) { String ddlTemplate = "CREATE TABLE default.%s (\n" diff --git a/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/SparkReadTestBase.java b/paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/SparkReadTestBase.java similarity index 100% rename from paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/SparkReadTestBase.java rename to paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/SparkReadTestBase.java diff --git a/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/SparkS3ITCase.java b/paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/SparkS3ITCase.java similarity index 100% rename from paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/SparkS3ITCase.java rename to paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/SparkS3ITCase.java diff --git a/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/SparkSchemaEvolutionITCase.java b/paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/SparkSchemaEvolutionITCase.java similarity index 58% rename from paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/SparkSchemaEvolutionITCase.java rename to paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/SparkSchemaEvolutionITCase.java index 7d94e7d5df73..fb4dab38ed94 100644 --- a/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/SparkSchemaEvolutionITCase.java +++ b/paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/SparkSchemaEvolutionITCase.java @@ -23,6 +23,8 @@ import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Row; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; import java.util.HashMap; import java.util.List; @@ -203,7 +205,7 @@ public void testRenamePartitionKey() { .satisfies( anyCauseMatches( UnsupportedOperationException.class, - "Cannot drop/rename partition key[a]")); + "Cannot rename partition column: [a]")); } @Test @@ -254,7 +256,7 @@ public void testDropPartitionKey() { .satisfies( anyCauseMatches( UnsupportedOperationException.class, - "Cannot drop/rename partition key[a]")); + "Cannot drop partition key or primary key: [a]")); } @Test @@ -276,7 +278,72 @@ public void testDropPrimaryKey() { .satisfies( anyCauseMatches( UnsupportedOperationException.class, - "Cannot drop/rename primary key[b]")); + "Cannot drop partition key or primary key: [b]")); + } + + @Test + public void testRenamePrimaryKey() { + spark.sql( + "CREATE TABLE test_rename_primary_key_table (\n" + + "a BIGINT NOT NULL,\n" + + "b STRING)\n" + + "TBLPROPERTIES ('primary-key' = 'a')"); + + spark.sql("INSERT INTO test_rename_primary_key_table VALUES(1, 'aaa'), (2, 'bbb')"); + + spark.sql("ALTER TABLE test_rename_primary_key_table RENAME COLUMN a to a_"); + + List result = + spark.sql("SHOW CREATE TABLE test_rename_primary_key_table").collectAsList(); + assertThat(result.toString()) + .contains( + showCreateString( + "test_rename_primary_key_table", "a_ BIGINT NOT NULL", "b STRING")) + .contains("'primary-key' = 'a_'"); + + List actual = + spark.sql("SELECT * FROM test_rename_primary_key_table").collectAsList().stream() + .map(Row::toString) + .collect(Collectors.toList()); + + assertThat(actual).containsExactlyInAnyOrder("[1,aaa]", "[2,bbb]"); + + spark.sql("INSERT INTO test_rename_primary_key_table VALUES(1, 'AAA'), (2, 'BBB')"); + + actual = + spark.sql("SELECT * FROM test_rename_primary_key_table").collectAsList().stream() + .map(Row::toString) + .collect(Collectors.toList()); + assertThat(actual).containsExactlyInAnyOrder("[1,AAA]", "[2,BBB]"); + } + + @Test + public void testRenameBucketKey() { + spark.sql( + "CREATE TABLE test_rename_bucket_key_table (\n" + + "a BIGINT NOT NULL,\n" + + "b STRING)\n" + + "TBLPROPERTIES ('bucket-key' = 'a,b', 'bucket'='16')"); + + spark.sql("INSERT INTO test_rename_bucket_key_table VALUES(1, 'aaa'), (2, 'bbb')"); + + spark.sql("ALTER TABLE test_rename_bucket_key_table RENAME COLUMN b to b_"); + + List result = + spark.sql("SHOW CREATE TABLE test_rename_bucket_key_table").collectAsList(); + assertThat(result.toString()) + .contains( + showCreateString( + "test_rename_bucket_key_table", "a BIGINT NOT NULL", "b_ STRING")) + .contains("'bucket-key' = 'a,b_'"); + + List actual = + spark.sql("SELECT * FROM test_rename_bucket_key_table where b_ = 'bbb'") + .collectAsList().stream() + .map(Row::toString) + .collect(Collectors.toList()); + + assertThat(actual).containsExactlyInAnyOrder("[2,bbb]"); } @Test @@ -640,4 +707,371 @@ private List getFieldStatsList(List fieldStatsRows) { ",")) .collect(Collectors.toList()); } + + @ParameterizedTest() + @ValueSource(strings = {"orc", "avro", "parquet"}) + public void testAddAndDropNestedColumn(String formatType) { + String tableName = "testAddNestedColumnTable"; + spark.sql( + "CREATE TABLE paimon.default." + + tableName + + " (k INT NOT NULL, v STRUCT>) " + + "TBLPROPERTIES ('bucket' = '1', 'primary-key' = 'k', 'file.format' = '" + + formatType + + "')"); + spark.sql( + "INSERT INTO paimon.default." + + tableName + + " VALUES (1, STRUCT(10, STRUCT('apple', 100))), (2, STRUCT(20, STRUCT('banana', 200)))"); + assertThat( + spark.sql("SELECT * FROM paimon.default." + tableName).collectAsList() + .stream() + .map(Row::toString)) + .containsExactlyInAnyOrder("[1,[10,[apple,100]]]", "[2,[20,[banana,200]]]"); + assertThat( + spark.sql("SELECT v.f2.f1, k FROM paimon.default." + tableName) + .collectAsList().stream() + .map(Row::toString)) + .containsExactlyInAnyOrder("[apple,1]", "[banana,2]"); + + spark.sql("ALTER TABLE paimon.default." + tableName + " ADD COLUMN v.f3 STRING"); + spark.sql("ALTER TABLE paimon.default." + tableName + " ADD COLUMN v.f2.f3 BIGINT"); + spark.sql( + "INSERT INTO paimon.default." + + tableName + + " VALUES (1, STRUCT(11, STRUCT('APPLE', 101, 1001), 'one')), (3, STRUCT(31, STRUCT('CHERRY', 301, 3001), 'three'))"); + assertThat( + spark.sql("SELECT * FROM paimon.default." + tableName).collectAsList() + .stream() + .map(Row::toString)) + .containsExactlyInAnyOrder( + "[1,[11,[APPLE,101,1001],one]]", + "[2,[20,[banana,200,null],null]]", + "[3,[31,[CHERRY,301,3001],three]]"); + assertThat( + spark.sql("SELECT v.f2.f2, v.f3, k FROM paimon.default." + tableName) + .collectAsList().stream() + .map(Row::toString)) + .containsExactlyInAnyOrder("[101,one,1]", "[200,null,2]", "[301,three,3]"); + + spark.sql("ALTER TABLE paimon.default." + tableName + " DROP COLUMN v.f2.f1"); + spark.sql( + "INSERT INTO paimon.default." + + tableName + + " VALUES (1, STRUCT(12, STRUCT(102, 1002), 'one')), (4, STRUCT(42, STRUCT(402, 4002), 'four'))"); + assertThat( + spark.sql("SELECT * FROM paimon.default." + tableName).collectAsList() + .stream() + .map(Row::toString)) + .containsExactlyInAnyOrder( + "[1,[12,[102,1002],one]]", + "[2,[20,[200,null],null]]", + "[3,[31,[301,3001],three]]", + "[4,[42,[402,4002],four]]"); + + spark.sql( + "ALTER TABLE paimon.default." + + tableName + + " ADD COLUMN v.f2.f1 DECIMAL(5, 2) AFTER f2"); + spark.sql( + "INSERT INTO paimon.default." + + tableName + + " VALUES (1, STRUCT(13, STRUCT(103, 100.03, 1003), 'one')), (5, STRUCT(53, STRUCT(503, 500.03, 5003), 'five'))"); + assertThat( + spark.sql("SELECT * FROM paimon.default." + tableName).collectAsList() + .stream() + .map(Row::toString)) + .containsExactlyInAnyOrder( + "[1,[13,[103,100.03,1003],one]]", + "[2,[20,[200,null,null],null]]", + "[3,[31,[301,null,3001],three]]", + "[4,[42,[402,null,4002],four]]", + "[5,[53,[503,500.03,5003],five]]"); + } + + @ParameterizedTest() + @ValueSource(strings = {"orc", "avro", "parquet"}) + public void testAddAndDropNestedColumnInArray(String formatType) { + String tableName = "testAddNestedColumnTable"; + spark.sql( + "CREATE TABLE paimon.default." + + tableName + + " (k INT NOT NULL, v ARRAY>) " + + "TBLPROPERTIES ('bucket' = '1', 'primary-key' = 'k', 'file.format' = '" + + formatType + + "')"); + spark.sql( + "INSERT INTO paimon.default." + + tableName + + " VALUES (1, ARRAY(STRUCT('apple', 100), STRUCT('banana', 101))), " + + "(2, ARRAY(STRUCT('cat', 200), STRUCT('dog', 201)))"); + assertThat( + spark.sql("SELECT * FROM paimon.default." + tableName).collectAsList() + .stream() + .map(Row::toString)) + .containsExactlyInAnyOrder( + "[1,WrappedArray([apple,100], [banana,101])]", + "[2,WrappedArray([cat,200], [dog,201])]"); + + spark.sql( + "ALTER TABLE paimon.default." + + tableName + + " ADD COLUMN v.element.f3 STRING AFTER f2"); + spark.sql("ALTER TABLE paimon.default." + tableName + " DROP COLUMN v.element.f1"); + spark.sql( + "INSERT INTO paimon.default." + + tableName + + " VALUES (1, ARRAY(STRUCT(110, 'APPLE'), STRUCT(111, 'BANANA'))), " + + "(3, ARRAY(STRUCT(310, 'FLOWER')))"); + assertThat( + spark.sql("SELECT * FROM paimon.default." + tableName).collectAsList() + .stream() + .map(Row::toString)) + .containsExactlyInAnyOrder( + "[1,WrappedArray([110,APPLE], [111,BANANA])]", + "[2,WrappedArray([200,null], [201,null])]", + "[3,WrappedArray([310,FLOWER])]"); + } + + @ParameterizedTest() + @ValueSource(strings = {"orc", "avro", "parquet"}) + public void testAddAndDropNestedColumnInMap(String formatType) { + String tableName = "testAddNestedColumnTable"; + spark.sql( + "CREATE TABLE paimon.default." + + tableName + + " (k INT NOT NULL, v MAP>) " + + "TBLPROPERTIES ('bucket' = '1', 'primary-key' = 'k', 'file.format' = '" + + formatType + + "')"); + spark.sql( + "INSERT INTO paimon.default." + + tableName + + " VALUES (1, MAP(10, STRUCT('apple', 100), 20, STRUCT('banana', 101))), " + + "(2, MAP(10, STRUCT('cat', 200), 20, STRUCT('dog', 201)))"); + assertThat( + spark.sql("SELECT k, v[10].f1, v[10].f2 FROM paimon.default." + tableName) + .collectAsList().stream() + .map(Row::toString)) + .containsExactlyInAnyOrder("[1,apple,100]", "[2,cat,200]"); + + spark.sql( + "ALTER TABLE paimon.default." + + tableName + + " ADD COLUMN v.value.f3 STRING AFTER f2"); + spark.sql("ALTER TABLE paimon.default." + tableName + " DROP COLUMN v.value.f1"); + spark.sql( + "INSERT INTO paimon.default." + + tableName + + " VALUES (1, MAP(10, STRUCT(110, 'APPLE'), 20, STRUCT(111, 'BANANA'))), " + + "(3, MAP(10, STRUCT(310, 'FLOWER')))"); + assertThat( + spark.sql("SELECT k, v[10].f2, v[10].f3 FROM paimon.default." + tableName) + .collectAsList().stream() + .map(Row::toString)) + .containsExactlyInAnyOrder("[1,110,APPLE]", "[2,200,null]", "[3,310,FLOWER]"); + } + + @ParameterizedTest() + @ValueSource(strings = {"orc", "avro", "parquet"}) + public void testRenameNestedColumn(String formatType) { + String tableName = "testRenameNestedColumnTable"; + spark.sql( + "CREATE TABLE paimon.default." + + tableName + + " (k INT NOT NULL, v STRUCT>) " + + "TBLPROPERTIES ('file.format' = '" + + formatType + + "')"); + spark.sql( + "INSERT INTO paimon.default." + + tableName + + " VALUES (1, STRUCT(10, STRUCT('apple', 100))), (2, STRUCT(20, STRUCT('banana', 200)))"); + assertThat( + spark.sql("SELECT v.f2.f1, k FROM paimon.default." + tableName) + .collectAsList().stream() + .map(Row::toString)) + .containsExactlyInAnyOrder("[apple,1]", "[banana,2]"); + + spark.sql("ALTER TABLE paimon.default." + tableName + " RENAME COLUMN v.f2.f1 to f100"); + assertThat( + spark.sql("SELECT v.f2.f100, k FROM paimon.default." + tableName) + .collectAsList().stream() + .map(Row::toString)) + .containsExactlyInAnyOrder("[apple,1]", "[banana,2]"); + } + + @ParameterizedTest() + @ValueSource(strings = {"orc", "avro", "parquet"}) + public void testRenameNestedColumnInArray(String formatType) { + String tableName = "testRenameNestedColumnTable"; + spark.sql( + "CREATE TABLE paimon.default." + + tableName + + " (k INT NOT NULL, v ARRAY>) " + + "TBLPROPERTIES ('file.format' = '" + + formatType + + "')"); + spark.sql( + "INSERT INTO paimon.default." + + tableName + + " VALUES (1, ARRAY(STRUCT('apple', 100), STRUCT('banana', 101))), " + + "(2, ARRAY(STRUCT('cat', 200), STRUCT('dog', 201)))"); + assertThat( + spark.sql("SELECT v[0].f1, k FROM paimon.default." + tableName) + .collectAsList().stream() + .map(Row::toString)) + .containsExactlyInAnyOrder("[apple,1]", "[cat,2]"); + + spark.sql( + "ALTER TABLE paimon.default." + tableName + " RENAME COLUMN v.element.f1 to f100"); + assertThat( + spark.sql("SELECT v[0].f100, k FROM paimon.default." + tableName) + .collectAsList().stream() + .map(Row::toString)) + .containsExactlyInAnyOrder("[apple,1]", "[cat,2]"); + } + + @ParameterizedTest() + @ValueSource(strings = {"orc", "avro", "parquet"}) + public void testRenameNestedColumnInMap(String formatType) { + String tableName = "testRenameNestedColumnTable"; + spark.sql( + "CREATE TABLE paimon.default." + + tableName + + " (k INT NOT NULL, v MAP>) " + + "TBLPROPERTIES ('file.format' = '" + + formatType + + "')"); + spark.sql( + "INSERT INTO paimon.default." + + tableName + + " VALUES (1, MAP(10, STRUCT('apple', 100), 20, STRUCT('banana', 101))), " + + "(2, MAP(10, STRUCT('cat', 200), 20, STRUCT('dog', 201)))"); + assertThat( + spark.sql("SELECT v[10].f1, k FROM paimon.default." + tableName) + .collectAsList().stream() + .map(Row::toString)) + .containsExactlyInAnyOrder("[apple,1]", "[cat,2]"); + + spark.sql("ALTER TABLE paimon.default." + tableName + " RENAME COLUMN v.value.f1 to f100"); + assertThat( + spark.sql("SELECT v[10].f100, k FROM paimon.default." + tableName) + .collectAsList().stream() + .map(Row::toString)) + .containsExactlyInAnyOrder("[apple,1]", "[cat,2]"); + } + + @ParameterizedTest() + @ValueSource(strings = {"orc", "avro", "parquet"}) + public void testUpdateNestedColumnType(String formatType) { + String tableName = "testRenameNestedColumnTable"; + spark.sql( + "CREATE TABLE paimon.default." + + tableName + + " (k INT NOT NULL, v STRUCT>) " + + "TBLPROPERTIES ('bucket' = '1', 'primary-key' = 'k', 'file.format' = '" + + formatType + + "')"); + spark.sql( + "INSERT INTO paimon.default." + + tableName + + " VALUES (1, STRUCT(10, STRUCT('apple', 100))), (2, STRUCT(20, STRUCT('banana', 200)))"); + assertThat( + spark.sql("SELECT v.f2.f2, k FROM paimon.default." + tableName) + .collectAsList().stream() + .map(Row::toString)) + .containsExactlyInAnyOrder("[100,1]", "[200,2]"); + + spark.sql("ALTER TABLE paimon.default." + tableName + " CHANGE COLUMN v.f2.f2 f2 BIGINT"); + spark.sql( + "INSERT INTO paimon.default." + + tableName + + " VALUES (1, STRUCT(11, STRUCT('APPLE', 101))), (3, STRUCT(31, STRUCT('CHERRY', 3000000000000)))"); + assertThat( + spark.sql("SELECT v.f2.f2, k FROM paimon.default." + tableName) + .collectAsList().stream() + .map(Row::toString)) + .containsExactlyInAnyOrder("[101,1]", "[200,2]", "[3000000000000,3]"); + } + + @ParameterizedTest() + @ValueSource(strings = {"orc", "avro", "parquet"}) + public void testUpdateNestedColumnTypeInArray(String formatType) { + String tableName = "testRenameNestedColumnTable"; + spark.sql( + "CREATE TABLE paimon.default." + + tableName + + " (k INT NOT NULL, v ARRAY>) " + + "TBLPROPERTIES ('bucket' = '1', 'primary-key' = 'k', 'file.format' = '" + + formatType + + "')"); + spark.sql( + "INSERT INTO paimon.default." + + tableName + + " VALUES (1, ARRAY(STRUCT('apple', 100), STRUCT('banana', 101))), " + + "(2, ARRAY(STRUCT('cat', 200), STRUCT('dog', 201)))"); + assertThat( + spark.sql("SELECT * FROM paimon.default." + tableName).collectAsList() + .stream() + .map(Row::toString)) + .containsExactlyInAnyOrder( + "[1,WrappedArray([apple,100], [banana,101])]", + "[2,WrappedArray([cat,200], [dog,201])]"); + + spark.sql( + "ALTER TABLE paimon.default." + + tableName + + " CHANGE COLUMN v.element.f2 f2 BIGINT"); + spark.sql( + "INSERT INTO paimon.default." + + tableName + + " VALUES (1, ARRAY(STRUCT('APPLE', 1000000000000), STRUCT('BANANA', 111))), " + + "(3, ARRAY(STRUCT('FLOWER', 3000000000000)))"); + assertThat( + spark.sql("SELECT * FROM paimon.default." + tableName).collectAsList() + .stream() + .map(Row::toString)) + .containsExactlyInAnyOrder( + "[1,WrappedArray([APPLE,1000000000000], [BANANA,111])]", + "[2,WrappedArray([cat,200], [dog,201])]", + "[3,WrappedArray([FLOWER,3000000000000])]"); + } + + @ParameterizedTest() + @ValueSource(strings = {"orc", "avro", "parquet"}) + public void testUpdateNestedColumnTypeInMap(String formatType) { + String tableName = "testRenameNestedColumnTable"; + spark.sql( + "CREATE TABLE paimon.default." + + tableName + + " (k INT NOT NULL, v MAP>) " + + "TBLPROPERTIES ('bucket' = '1', 'primary-key' = 'k', 'file.format' = '" + + formatType + + "')"); + spark.sql( + "INSERT INTO paimon.default." + + tableName + + " VALUES (1, MAP(10, STRUCT('apple', 100), 20, STRUCT('banana', 101))), " + + "(2, MAP(10, STRUCT('cat', 200), 20, STRUCT('dog', 201)))"); + assertThat( + spark.sql("SELECT k, v[10].f1, v[10].f2 FROM paimon.default." + tableName) + .collectAsList().stream() + .map(Row::toString)) + .containsExactlyInAnyOrder("[1,apple,100]", "[2,cat,200]"); + + spark.sql( + "ALTER TABLE paimon.default." + tableName + " CHANGE COLUMN v.value.f2 f2 BIGINT"); + spark.sql( + "INSERT INTO paimon.default." + + tableName + + " VALUES (1, MAP(10, STRUCT('APPLE', 1000000000000), 20, STRUCT('BANANA', 111))), " + + "(3, MAP(10, STRUCT('FLOWER', 3000000000000)))"); + assertThat( + spark.sql("SELECT k, v[10].f1, v[10].f2 FROM paimon.default." + tableName) + .collectAsList().stream() + .map(Row::toString)) + .containsExactlyInAnyOrder( + "[1,APPLE,1000000000000]", "[2,cat,200]", "[3,FLOWER,3000000000000]"); + } } diff --git a/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/SparkTimeTravelITCase.java b/paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/SparkTimeTravelITCase.java similarity index 100% rename from paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/SparkTimeTravelITCase.java rename to paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/SparkTimeTravelITCase.java diff --git a/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/SparkTimeTravelWithDataFrameITCase.java b/paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/SparkTimeTravelWithDataFrameITCase.java similarity index 100% rename from paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/SparkTimeTravelWithDataFrameITCase.java rename to paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/SparkTimeTravelWithDataFrameITCase.java diff --git a/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/SparkTypeTest.java b/paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/SparkTypeTest.java similarity index 100% rename from paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/SparkTypeTest.java rename to paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/SparkTypeTest.java diff --git a/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/SparkWriteITCase.java b/paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/SparkWriteITCase.java similarity index 74% rename from paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/SparkWriteITCase.java rename to paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/SparkWriteITCase.java index 0cc17639fd80..b0d5b380c1f2 100644 --- a/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/SparkWriteITCase.java +++ b/paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/SparkWriteITCase.java @@ -36,6 +36,7 @@ import org.junit.jupiter.api.TestInstance; import org.junit.jupiter.api.io.TempDir; +import java.io.IOException; import java.util.Arrays; import java.util.Comparator; import java.util.List; @@ -295,6 +296,9 @@ public void testDataFilePrefixForPKTable() { for (String fileName : fileNames) { Assertions.assertTrue(fileName.startsWith("test-")); } + + // reset config, it will affect other tests + spark.conf().unset("spark.paimon.data-file.prefix"); } @Test @@ -316,6 +320,114 @@ public void testChangelogFilePrefixForPkTable() throws Exception { spark.sql("INSERT INTO T VALUES (2, 2, 'bb')"); FileStatus[] files2 = fileIO.listStatus(new Path(tabLocation, "bucket-0")); Assertions.assertEquals(1, dataFileCount(files2, "test-changelog-")); + + // reset config, it will affect other tests + spark.conf().unset("spark.paimon.changelog-file.prefix"); + } + + @Test + public void testMarkDone() throws IOException { + spark.sql( + "CREATE TABLE T (a INT, b INT, c STRING) PARTITIONED BY (c) TBLPROPERTIES (" + + "'partition.end-input-to-done' = 'true', 'partition.mark-done-action' = 'success-file')"); + spark.sql("INSERT INTO T VALUES (1, 1, 'aa')"); + + FileStoreTable table = getTable("T"); + FileIO fileIO = table.fileIO(); + Path tabLocation = table.location(); + + Assertions.assertTrue(fileIO.exists(new Path(tabLocation, "c=aa/_SUCCESS"))); + } + + @Test + public void testDataFileSuffixName() { + spark.sql( + "CREATE TABLE T (a INT, b INT, c STRING)" + + " TBLPROPERTIES (" + + "'bucket' = '1', " + + "'primary-key'='a', " + + "'write-only' = 'true', " + + "'file.format' = 'parquet', " + + "'file.compression' = 'zstd')"); + + spark.sql("INSERT INTO T VALUES (1, 1, 'aa')"); + spark.sql("INSERT INTO T VALUES (2, 2, 'bb')"); + + // enable file suffix + spark.conf().set("spark.paimon.file.suffix.include.compression", true); + spark.sql("INSERT INTO T VALUES (3, 3, 'cc')"); + spark.sql("INSERT INTO T VALUES (4, 4, 'dd')"); + + List data2 = spark.sql("SELECT * FROM T order by a").collectAsList(); + assertThat(data2.toString()).isEqualTo("[[1,1,aa], [2,2,bb], [3,3,cc], [4,4,dd]]"); + + // check files suffix name + List files = + spark.sql("select file_path from `T$files`").collectAsList().stream() + .map(x -> x.getString(0)) + .collect(Collectors.toList()); + Assertions.assertEquals(4, files.size()); + + String defaultExtension = "." + "parquet"; + String newExtension = "." + "zstd" + "." + "parquet"; + // two data files end with ".parquet", two data file end with ".zstd.parquet" + Assertions.assertEquals( + 2, + files.stream() + .filter( + name -> + name.endsWith(defaultExtension) + && !name.endsWith(newExtension)) + .count()); + Assertions.assertEquals( + 2, files.stream().filter(name -> name.endsWith(newExtension)).count()); + + // reset config + spark.conf().unset("spark.paimon.file.suffix.include.compression"); + } + + @Test + public void testChangelogFileSuffixName() throws Exception { + spark.sql( + "CREATE TABLE T (a INT, b INT, c STRING) " + + "TBLPROPERTIES (" + + "'primary-key'='a', " + + "'bucket' = '1', " + + "'changelog-producer' = 'lookup', " + + "'file.format' = 'parquet', " + + "'file.compression' = 'zstd')"); + + FileStoreTable table = getTable("T"); + Path tabLocation = table.location(); + FileIO fileIO = table.fileIO(); + + spark.sql("INSERT INTO T VALUES (1, 1, 'aa')"); + + spark.conf().set("spark.paimon.file.suffix.include.compression", true); + spark.sql("INSERT INTO T VALUES (2, 2, 'bb')"); + + // collect changelog files + List files = + Arrays.stream(fileIO.listStatus(new Path(tabLocation, "bucket-0"))) + .map(name -> name.getPath().getName()) + .filter(name -> name.startsWith("changelog-")) + .collect(Collectors.toList()); + String defaultExtension = "." + "parquet"; + String newExtension = "." + "zstd" + "." + "parquet"; + // one changelog file end with ".parquet", one changelog file end with ".zstd.parquet" + Assertions.assertEquals( + 1, + files.stream() + .filter( + name -> + name.endsWith(defaultExtension) + && !name.endsWith(newExtension)) + .count()); + Assertions.assertEquals( + 1, files.stream().filter(name -> name.endsWith(newExtension)).count()); + + // reset config + spark.conf().unset("spark.paimon.file.suffix.include.compression"); } protected static FileStoreTable getTable(String tableName) { diff --git a/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/SparkWriteWithKyroITCase.java b/paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/SparkWriteWithKyroITCase.java similarity index 100% rename from paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/SparkWriteWithKyroITCase.java rename to paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/SparkWriteWithKyroITCase.java diff --git a/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/extensions/CallStatementParserTest.java b/paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/extensions/CallStatementParserTest.java similarity index 80% rename from paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/extensions/CallStatementParserTest.java rename to paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/extensions/CallStatementParserTest.java index 61e06016cbd3..e4e571e96bc9 100644 --- a/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/extensions/CallStatementParserTest.java +++ b/paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/extensions/CallStatementParserTest.java @@ -79,14 +79,37 @@ public void stopSparkSession() { } } + @Test + public void testDelegateUnsupportedProcedure() { + assertThatThrownBy(() -> parser.parsePlan("CALL cat.d.t()")) + .isInstanceOf(ParseException.class) + .satisfies( + exception -> { + ParseException parseException = (ParseException) exception; + assertThat(parseException.getErrorClass()) + .isEqualTo("PARSE_SYNTAX_ERROR"); + assertThat(parseException.getMessageParameters().get("error")) + .isEqualTo("'CALL'"); + }); + } + + @Test + public void testCallWithBackticks() throws ParseException { + PaimonCallStatement call = + (PaimonCallStatement) parser.parsePlan("CALL cat.`sys`.`rollback`()"); + assertThat(JavaConverters.seqAsJavaList(call.name())) + .isEqualTo(Arrays.asList("cat", "sys", "rollback")); + assertThat(call.args().size()).isEqualTo(0); + } + @Test public void testCallWithNamedArguments() throws ParseException { PaimonCallStatement callStatement = (PaimonCallStatement) parser.parsePlan( - "CALL catalog.system.named_args_func(arg1 => 1, arg2 => 'test', arg3 => true)"); + "CALL catalog.sys.rollback(arg1 => 1, arg2 => 'test', arg3 => true)"); assertThat(JavaConverters.seqAsJavaList(callStatement.name())) - .isEqualTo(Arrays.asList("catalog", "system", "named_args_func")); + .isEqualTo(Arrays.asList("catalog", "sys", "rollback")); assertThat(callStatement.args().size()).isEqualTo(3); assertArgument(callStatement, 0, "arg1", 1, DataTypes.IntegerType); assertArgument(callStatement, 1, "arg2", "test", DataTypes.StringType); @@ -98,11 +121,11 @@ public void testCallWithPositionalArguments() throws ParseException { PaimonCallStatement callStatement = (PaimonCallStatement) parser.parsePlan( - "CALL catalog.system.positional_args_func(1, '${spark.sql.extensions}', 2L, true, 3.0D, 4" + "CALL catalog.sys.rollback(1, '${spark.sql.extensions}', 2L, true, 3.0D, 4" + ".0e1,500e-1BD, " + "TIMESTAMP '2017-02-03T10:37:30.00Z')"); assertThat(JavaConverters.seqAsJavaList(callStatement.name())) - .isEqualTo(Arrays.asList("catalog", "system", "positional_args_func")); + .isEqualTo(Arrays.asList("catalog", "sys", "rollback")); assertThat(callStatement.args().size()).isEqualTo(8); assertArgument(callStatement, 0, 1, DataTypes.IntegerType); assertArgument( @@ -127,9 +150,9 @@ public void testCallWithPositionalArguments() throws ParseException { public void testCallWithMixedArguments() throws ParseException { PaimonCallStatement callStatement = (PaimonCallStatement) - parser.parsePlan("CALL catalog.system.mixed_function(arg1 => 1, 'test')"); + parser.parsePlan("CALL catalog.sys.rollback(arg1 => 1, 'test')"); assertThat(JavaConverters.seqAsJavaList(callStatement.name())) - .isEqualTo(Arrays.asList("catalog", "system", "mixed_function")); + .isEqualTo(Arrays.asList("catalog", "sys", "rollback")); assertThat(callStatement.args().size()).isEqualTo(2); assertArgument(callStatement, 0, "arg1", 1, DataTypes.IntegerType); assertArgument(callStatement, 1, "test", DataTypes.StringType); @@ -137,9 +160,9 @@ public void testCallWithMixedArguments() throws ParseException { @Test public void testCallWithParseException() { - assertThatThrownBy(() -> parser.parsePlan("CALL catalog.system func abc")) + assertThatThrownBy(() -> parser.parsePlan("CALL catalog.sys.rollback abc")) .isInstanceOf(PaimonParseException.class) - .hasMessageContaining("missing '(' at 'func'"); + .hasMessageContaining("missing '(' at 'abc'"); } private void assertArgument( diff --git a/paimon-spark/paimon-spark-common/src/test/resources/META-INF/services/org.junit.jupiter.api.extension.Extension b/paimon-spark/paimon-spark-ut/src/test/resources/META-INF/services/org.junit.jupiter.api.extension.Extension similarity index 100% rename from paimon-spark/paimon-spark-common/src/test/resources/META-INF/services/org.junit.jupiter.api.extension.Extension rename to paimon-spark/paimon-spark-ut/src/test/resources/META-INF/services/org.junit.jupiter.api.extension.Extension diff --git a/paimon-spark/paimon-spark-common/src/test/resources/hive-site.xml b/paimon-spark/paimon-spark-ut/src/test/resources/hive-site.xml similarity index 86% rename from paimon-spark/paimon-spark-common/src/test/resources/hive-site.xml rename to paimon-spark/paimon-spark-ut/src/test/resources/hive-site.xml index 4972efc5900e..c4a016d51d04 100644 --- a/paimon-spark/paimon-spark-common/src/test/resources/hive-site.xml +++ b/paimon-spark/paimon-spark-ut/src/test/resources/hive-site.xml @@ -42,6 +42,12 @@ true + + + datanucleus.connectionPoolingType + DBCP + + hive.metastore.uris thrift://localhost:9083 diff --git a/paimon-spark/paimon-spark-ut/src/test/resources/log4j2-test.properties b/paimon-spark/paimon-spark-ut/src/test/resources/log4j2-test.properties new file mode 100644 index 000000000000..6f324f5863ac --- /dev/null +++ b/paimon-spark/paimon-spark-ut/src/test/resources/log4j2-test.properties @@ -0,0 +1,38 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +# Set root logger level to OFF to not flood build logs +# set manually to INFO for debugging purposes +rootLogger.level = OFF +rootLogger.appenderRef.test.ref = TestLogger + +appender.testlogger.name = TestLogger +appender.testlogger.type = CONSOLE +appender.testlogger.target = SYSTEM_ERR +appender.testlogger.layout.type = PatternLayout +appender.testlogger.layout.pattern = %-4r [%tid %t] %-5p %c %x - %m%n + +logger.kafka.name = kafka +logger.kafka.level = OFF +logger.kafka2.name = state.change +logger.kafka2.level = OFF + +logger.zookeeper.name = org.apache.zookeeper +logger.zookeeper.level = OFF +logger.I0Itec.name = org.I0Itec +logger.I0Itec.level = OFF diff --git a/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/PaimonCDCSourceTest.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/PaimonCDCSourceTest.scala similarity index 100% rename from paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/PaimonCDCSourceTest.scala rename to paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/PaimonCDCSourceTest.scala diff --git a/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/PaimonCommitTest.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/PaimonCommitTest.scala similarity index 100% rename from paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/PaimonCommitTest.scala rename to paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/PaimonCommitTest.scala diff --git a/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/PaimonHiveTestBase.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/PaimonHiveTestBase.scala similarity index 97% rename from paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/PaimonHiveTestBase.scala rename to paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/PaimonHiveTestBase.scala index ccd705e26967..6d2ffea04df5 100644 --- a/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/PaimonHiveTestBase.scala +++ b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/PaimonHiveTestBase.scala @@ -18,12 +18,11 @@ package org.apache.paimon.spark -import org.apache.paimon.Snapshot import org.apache.paimon.hive.TestHiveMetastore import org.apache.hadoop.conf.Configuration import org.apache.spark.SparkConf -import org.apache.spark.paimon.Utils +import org.apache.spark.sql.paimon.Utils import java.io.File diff --git a/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/PaimonSinkTest.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/PaimonSinkTest.scala similarity index 98% rename from paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/PaimonSinkTest.scala rename to paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/PaimonSinkTest.scala index 63203122ac40..61bf5524942d 100644 --- a/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/PaimonSinkTest.scala +++ b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/PaimonSinkTest.scala @@ -18,6 +18,7 @@ package org.apache.paimon.spark +import org.apache.spark.SparkConf import org.apache.spark.sql.{Dataset, Row} import org.apache.spark.sql.execution.streaming.MemoryStream import org.apache.spark.sql.functions.{col, mean, window} @@ -27,6 +28,10 @@ import java.sql.Date class PaimonSinkTest extends PaimonSparkTestBase with StreamTest { + override protected def sparkConf: SparkConf = { + super.sparkConf.set("spark.sql.catalog.paimon.cache-enabled", "false") + } + import testImplicits._ test("Paimon Sink: forEachBatch") { diff --git a/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/PaimonSourceTest.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/PaimonSourceTest.scala similarity index 100% rename from paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/PaimonSourceTest.scala rename to paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/PaimonSourceTest.scala diff --git a/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/PaimonSparkTestBase.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/PaimonSparkTestBase.scala similarity index 72% rename from paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/PaimonSparkTestBase.scala rename to paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/PaimonSparkTestBase.scala index 3deb91cbcba7..867b3e5e3337 100644 --- a/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/PaimonSparkTestBase.scala +++ b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/PaimonSparkTestBase.scala @@ -18,25 +18,24 @@ package org.apache.paimon.spark -import org.apache.paimon.catalog.{Catalog, CatalogContext, CatalogFactory, Identifier} -import org.apache.paimon.options.{CatalogOptions, Options} -import org.apache.paimon.spark.catalog.Catalogs +import org.apache.paimon.catalog.{Catalog, Identifier} +import org.apache.paimon.spark.catalog.WithPaimonCatalog import org.apache.paimon.spark.extensions.PaimonSparkSessionExtensions import org.apache.paimon.spark.sql.{SparkVersionSupport, WithTableOptions} import org.apache.paimon.table.FileStoreTable import org.apache.spark.SparkConf -import org.apache.spark.paimon.Utils import org.apache.spark.sql.QueryTest import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan import org.apache.spark.sql.connector.catalog.{Identifier => SparkIdentifier} import org.apache.spark.sql.execution.datasources.v2.{DataSourceV2Relation, DataSourceV2ScanRelation} +import org.apache.spark.sql.internal.SQLConf +import org.apache.spark.sql.paimon.Utils import org.apache.spark.sql.test.SharedSparkSession import org.scalactic.source.Position import org.scalatest.Tag import java.io.File -import java.util.{HashMap => JHashMap} import java.util.TimeZone import scala.util.Random @@ -49,7 +48,9 @@ class PaimonSparkTestBase protected lazy val tempDBDir: File = Utils.createTempDir - protected lazy val catalog: Catalog = initCatalog() + protected def paimonCatalog: Catalog = { + spark.sessionState.catalogManager.currentCatalog.asInstanceOf[WithPaimonCatalog].paimonCatalog() + } protected val dbName0: String = "test" @@ -65,7 +66,6 @@ class PaimonSparkTestBase super.sparkConf .set("spark.sql.catalog.paimon", classOf[SparkCatalog].getName) .set("spark.sql.catalog.paimon.warehouse", tempDBDir.getCanonicalPath) - .set("spark.sql.catalog.paimon.cache-enabled", "false") .set("spark.sql.extensions", classOf[PaimonSparkSessionExtensions].getName) .set("spark.serializer", serializer) } @@ -105,7 +105,7 @@ class PaimonSparkTestBase } protected def withTimeZone(timeZone: String)(f: => Unit): Unit = { - withSQLConf("spark.sql.session.timeZone" -> timeZone) { + withSparkSQLConf("spark.sql.session.timeZone" -> timeZone) { val originTimeZone = TimeZone.getDefault try { TimeZone.setDefault(TimeZone.getTimeZone(timeZone)) @@ -116,24 +116,52 @@ class PaimonSparkTestBase } } + // Since SPARK-46227 has changed the definition of withSQLConf that resulted in + // incompatibility between the Spark3.x and Spark4.x, So Paimon declare a separate method + // to provide the same function. + protected def withSparkSQLConf(pairs: (String, String)*)(f: => Unit): Unit = { + withSparkSQLConf0(pairs: _*)(f) + } + + private def withSparkSQLConf0(pairs: (String, String)*)(f: => Unit): Unit = { + val conf = SQLConf.get + val (keys, values) = pairs.unzip + val currentValues = keys.map { + key => + if (conf.contains(key)) { + Some(conf.getConfString(key)) + } else { + None + } + } + (keys, values).zipped.foreach { + (k, v) => + if (SQLConf.isStaticConfigKey(k)) { + throw new RuntimeException(s"Cannot modify the value of a static config: $k") + } + conf.setConfString(k, v) + } + try f + finally { + keys.zip(currentValues).foreach { + case (key, Some(value)) => conf.setConfString(key, value) + case (key, None) => conf.unsetConf(key) + } + } + } + override def test(testName: String, testTags: Tag*)(testFun: => Any)(implicit pos: Position): Unit = { println(testName) super.test(testName, testTags: _*)(testFun)(pos) } - private def initCatalog(): Catalog = { - val currentCatalog = spark.sessionState.catalogManager.currentCatalog.name() - val options = - new JHashMap[String, String](Catalogs.catalogOptions(currentCatalog, spark.sessionState.conf)) - options.put(CatalogOptions.CACHE_ENABLED.key(), "false") - val catalogContext = - CatalogContext.create(Options.fromMap(options), spark.sessionState.newHadoopConf()) - CatalogFactory.createCatalog(catalogContext) + def loadTable(tableName: String): FileStoreTable = { + loadTable(dbName0, tableName) } - def loadTable(tableName: String): FileStoreTable = { - catalog.getTable(Identifier.create(dbName0, tableName)).asInstanceOf[FileStoreTable] + def loadTable(dbName: String, tableName: String): FileStoreTable = { + paimonCatalog.getTable(Identifier.create(dbName, tableName)).asInstanceOf[FileStoreTable] } protected def createRelationV2(tableName: String): LogicalPlan = { diff --git a/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/PaimonTableTest.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/PaimonTableTest.scala similarity index 100% rename from paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/PaimonTableTest.scala rename to paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/PaimonTableTest.scala diff --git a/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/ScanHelperTest.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/ScanHelperTest.scala similarity index 82% rename from paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/ScanHelperTest.scala rename to paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/ScanHelperTest.scala index 63711393039b..a3223446f644 100644 --- a/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/ScanHelperTest.scala +++ b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/ScanHelperTest.scala @@ -34,7 +34,7 @@ import scala.collection.mutable class ScanHelperTest extends PaimonSparkTestBase { test("Paimon: reshuffle splits") { - withSQLConf(("spark.sql.leafNodeDefaultParallelism", "20")) { + withSparkSQLConf(("spark.sql.leafNodeDefaultParallelism", "20")) { val splitNum = 5 val fileNum = 100 @@ -42,7 +42,18 @@ class ScanHelperTest extends PaimonSparkTestBase { 0.until(fileNum).foreach { i => val path = s"f$i.parquet" - files += DataFileMeta.forAppend(path, 750000, 30000, null, 0, 29999, 1, FileSource.APPEND) + files += DataFileMeta.forAppend( + path, + 750000, + 30000, + null, + 0, + 29999, + 1, + new java.util.ArrayList[String](), + null, + FileSource.APPEND, + null) } val dataSplits = mutable.ArrayBuffer.empty[Split] @@ -67,7 +78,18 @@ class ScanHelperTest extends PaimonSparkTestBase { test("Paimon: reshuffle one split") { val files = List( - DataFileMeta.forAppend("f1.parquet", 750000, 30000, null, 0, 29999, 1, FileSource.APPEND) + DataFileMeta.forAppend( + "f1.parquet", + 750000, + 30000, + null, + 0, + 29999, + 1, + new java.util.ArrayList[String](), + null, + FileSource.APPEND, + null) ).asJava val dataSplits: Array[Split] = Array( diff --git a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/procedure/AlterBranchProcedureTest.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/procedure/AlterBranchProcedureTest.scala new file mode 100644 index 000000000000..316c36c40c56 --- /dev/null +++ b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/procedure/AlterBranchProcedureTest.scala @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.procedure + +import org.apache.paimon.spark.PaimonSparkTestBase + +import org.apache.spark.sql.{Dataset, Row} +import org.apache.spark.sql.execution.streaming.MemoryStream +import org.apache.spark.sql.streaming.StreamTest + +class AlterBranchProcedureTest extends PaimonSparkTestBase with StreamTest { + + import testImplicits._ + test("Paimon Procedure: alter schema structure and test $branch syntax.") { + withTempDir { + checkpointDir => + // define a change-log table and test `forEachBatch` api + spark.sql(s""" + |CREATE TABLE T (a INT, b STRING) + |TBLPROPERTIES ('primary-key'='a', 'bucket'='3') + |""".stripMargin) + val location = loadTable("T").location().toString + + val inputData = MemoryStream[(Int, String)] + val stream = inputData + .toDS() + .toDF("a", "b") + .writeStream + .option("checkpointLocation", checkpointDir.getCanonicalPath) + .foreachBatch { + (batch: Dataset[Row], _: Long) => + batch.write.format("paimon").mode("append").save(location) + } + .start() + + val query = () => spark.sql("SELECT * FROM T ORDER BY a") + try { + // snapshot-1 + inputData.addData((1, "a")) + stream.processAllAvailable() + checkAnswer(query(), Row(1, "a") :: Nil) + + // snapshot-2 + inputData.addData((2, "b")) + stream.processAllAvailable() + checkAnswer(query(), Row(1, "a") :: Row(2, "b") :: Nil) + + // snapshot-3 + inputData.addData((2, "b2")) + stream.processAllAvailable() + checkAnswer(query(), Row(1, "a") :: Row(2, "b2") :: Nil) + + val table = loadTable("T") + val branchManager = table.branchManager() + + // create branch with tag + checkAnswer( + spark.sql("CALL paimon.sys.create_tag(table => 'test.T', tag => 's_2', snapshot => 2)"), + Row(true) :: Nil) + checkAnswer( + spark.sql( + "CALL paimon.sys.create_branch(table => 'test.T', branch => 'snapshot_branch', tag => 's_2')"), + Row(true) :: Nil) + assert(branchManager.branchExists("snapshot_branch")) + + spark.sql("INSERT INTO T VALUES (1, 'APPLE'), (2,'DOG'), (2, 'horse')") + spark.sql("ALTER TABLE `T$branch_snapshot_branch` ADD COLUMNS(c INT)") + spark.sql( + "INSERT INTO `T$branch_snapshot_branch` VALUES " + "(1,'cherry', 100), (2,'bird', 200), (3, 'wolf', 400)") + + checkAnswer( + spark.sql("SELECT * FROM T ORDER BY a, b"), + Row(1, "APPLE") :: Row(2, "horse") :: Nil) + checkAnswer( + spark.sql("SELECT * FROM `T$branch_snapshot_branch` ORDER BY a, b,c"), + Row(1, "cherry", 100) :: Row(2, "bird", 200) :: Row(3, "wolf", 400) :: Nil) + assert(branchManager.branchExists("snapshot_branch")) + } + } + } +} diff --git a/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/procedure/BranchProcedureTest.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/procedure/BranchProcedureTest.scala similarity index 100% rename from paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/procedure/BranchProcedureTest.scala rename to paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/procedure/BranchProcedureTest.scala diff --git a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/procedure/CompactManifestProcedureTest.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/procedure/CompactManifestProcedureTest.scala new file mode 100644 index 000000000000..c1c90251338f --- /dev/null +++ b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/procedure/CompactManifestProcedureTest.scala @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.procedure + +import org.apache.paimon.spark.PaimonSparkTestBase + +import org.apache.spark.sql.streaming.StreamTest +import org.assertj.core.api.Assertions + +/** Test compact manifest procedure. See [[CompactManifestProcedure]]. */ +class CompactManifestProcedureTest extends PaimonSparkTestBase with StreamTest { + + test("Paimon Procedure: compact manifest") { + spark.sql( + s""" + |CREATE TABLE T (id INT, value STRING, dt STRING, hh INT) + |TBLPROPERTIES ('bucket'='-1', 'write-only'='true', 'compaction.min.file-num'='2', 'compaction.max.file-num'='2') + |PARTITIONED BY (dt, hh) + |""".stripMargin) + + spark.sql(s"INSERT INTO T VALUES (5, '5', '2024-01-02', 0), (6, '6', '2024-01-02', 1)") + spark.sql(s"INSERT OVERWRITE T VALUES (5, '5', '2024-01-02', 0), (6, '6', '2024-01-02', 1)") + spark.sql(s"INSERT OVERWRITE T VALUES (5, '5', '2024-01-02', 0), (6, '6', '2024-01-02', 1)") + spark.sql(s"INSERT OVERWRITE T VALUES (5, '5', '2024-01-02', 0), (6, '6', '2024-01-02', 1)") + + Thread.sleep(10000); + + var rows = spark.sql("SELECT sum(num_deleted_files) FROM `T$manifests`").collectAsList() + Assertions.assertThat(rows.get(0).getLong(0)).isEqualTo(6L) + spark.sql("CALL sys.compact_manifest(table => 'T')") + rows = spark.sql("SELECT sum(num_deleted_files) FROM `T$manifests`").collectAsList() + Assertions.assertThat(rows.get(0).getLong(0)).isEqualTo(0L) + } +} diff --git a/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/procedure/CompactProcedureTestBase.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/procedure/CompactProcedureTestBase.scala similarity index 93% rename from paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/procedure/CompactProcedureTestBase.scala rename to paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/procedure/CompactProcedureTestBase.scala index 130860c8351e..31f78f61c20d 100644 --- a/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/procedure/CompactProcedureTestBase.scala +++ b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/procedure/CompactProcedureTestBase.scala @@ -39,6 +39,56 @@ abstract class CompactProcedureTestBase extends PaimonSparkTestBase with StreamT import testImplicits._ + // ----------------------- Minor Compact ----------------------- + + test("Paimon Procedure: compact aware bucket pk table with minor compact strategy") { + withTable("T") { + spark.sql(s""" + |CREATE TABLE T (id INT, value STRING, pt STRING) + |TBLPROPERTIES ('primary-key'='id, pt', 'bucket'='1', 'write-only'='true') + |PARTITIONED BY (pt) + |""".stripMargin) + + val table = loadTable("T") + + spark.sql(s"INSERT INTO T VALUES (1, 'a', 'p1'), (2, 'b', 'p2')") + spark.sql(s"INSERT INTO T VALUES (3, 'c', 'p1'), (4, 'd', 'p2')") + + Assertions.assertThat(lastSnapshotCommand(table).equals(CommitKind.APPEND)).isTrue + Assertions.assertThat(lastSnapshotId(table)).isEqualTo(2) + + spark.sql( + "CALL sys.compact(table => 'T', compact_strategy => 'minor'," + + "options => 'num-sorted-run.compaction-trigger=3')") + + // Due to the limitation of parameter 'num-sorted-run.compaction-trigger' = 3, so compact is not + // performed. + Assertions.assertThat(lastSnapshotCommand(table).equals(CommitKind.APPEND)).isTrue + Assertions.assertThat(lastSnapshotId(table)).isEqualTo(2) + + // Make par-p1 has 3 datafile and par-p2 has 2 datafile, so par-p2 will not be picked out to + // compact. + spark.sql(s"INSERT INTO T VALUES (1, 'a', 'p1')") + + spark.sql( + "CALL sys.compact(table => 'T', compact_strategy => 'minor'," + + "options => 'num-sorted-run.compaction-trigger=3')") + + Assertions.assertThat(lastSnapshotId(table)).isEqualTo(4) + Assertions.assertThat(lastSnapshotCommand(table).equals(CommitKind.COMPACT)).isTrue + + val splits = table.newSnapshotReader.read.dataSplits + splits.forEach( + split => { + Assertions + .assertThat(split.dataFiles.size) + .isEqualTo(if (split.partition().getString(0).toString == "p2") 2 else 1) + }) + } + } + + // ----------------------- Sort Compact ----------------------- + test("Paimon Procedure: sort compact") { failAfter(streamingTimeout) { withTempDir { diff --git a/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/procedure/CreateAndDeleteTagProcedureTest.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/procedure/CreateAndDeleteTagProcedureTest.scala similarity index 83% rename from paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/procedure/CreateAndDeleteTagProcedureTest.scala rename to paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/procedure/CreateAndDeleteTagProcedureTest.scala index 3621d44b8395..4a4c7ae215df 100644 --- a/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/procedure/CreateAndDeleteTagProcedureTest.scala +++ b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/procedure/CreateAndDeleteTagProcedureTest.scala @@ -120,7 +120,7 @@ class CreateAndDeleteTagProcedureTest extends PaimonSparkTestBase with StreamTes spark.sql("SELECT tag_name FROM paimon.test.`T$tags`"), Row("test_tag_2") :: Row("test_tag_3") :: Nil) - // delete test_tag_1 and test_tag_2 + // delete test_tag_2 and test_tag_3 checkAnswer( spark.sql( "CALL paimon.sys.delete_tag(table => 'test.T', tag => 'test_tag_2,test_tag_3')"), @@ -172,19 +172,15 @@ class CreateAndDeleteTagProcedureTest extends PaimonSparkTestBase with StreamTes "table => 'test.T', tag => 'test_tag', snapshot => 1)"), Row(true) :: Nil) checkAnswer( - spark.sql( - "SELECT count(time_retained) FROM paimon.test.`T$tags` where tag_name = 'test_tag'"), - Row(0) :: Nil) + spark.sql("SELECT count(*) FROM paimon.test.`T$tags` where tag_name = 'test_tag'"), + Row(1) :: Nil) - checkAnswer( + // throw exception "Tag test_tag already exists" + assertThrows[IllegalArgumentException] { spark.sql( "CALL paimon.sys.create_tag(" + - "table => 'test.T', tag => 'test_tag', time_retained => '5 d', snapshot => 1)"), - Row(true) :: Nil) - checkAnswer( - spark.sql( - "SELECT count(time_retained) FROM paimon.test.`T$tags` where tag_name = 'test_tag'"), - Row(1) :: Nil) + "table => 'test.T', tag => 'test_tag', time_retained => '5 d', snapshot => 1)") + } } finally { stream.stop() } @@ -199,4 +195,30 @@ class CreateAndDeleteTagProcedureTest extends PaimonSparkTestBase with StreamTes spark.sql("CALL paimon.sys.delete_tag(table => 'test.T', tag => 'test_tag')"), Row(true) :: Nil) } + + test("Paimon Procedure: delete multiple tags") { + spark.sql("CREATE TABLE T (id INT, name STRING) USING PAIMON") + spark.sql("insert into T values (1, 'a')") + + // create four tags + spark.sql("CALL paimon.sys.create_tag(table => 'test.T', tag => 'tag-1')") + spark.sql("CALL paimon.sys.create_tag(table => 'test.T', tag => 'tag-2')") + spark.sql("CALL paimon.sys.create_tag(table => 'test.T', tag => 'tag-3')") + spark.sql("CALL paimon.sys.create_tag(table => 'test.T', tag => 'tag-4')") + checkAnswer(spark.sql("SELECT count(*) FROM paimon.test.`T$tags`"), Row(4) :: Nil) + + // multiple tags with no space + checkAnswer( + spark.sql("CALL paimon.sys.delete_tag(table => 'test.T', tag => 'tag-1,tag-2')"), + Row(true) :: Nil) + checkAnswer( + spark.sql("SELECT tag_name FROM paimon.test.`T$tags`"), + Row("tag-3") :: Row("tag-4") :: Nil) + + // multiple tags with space + checkAnswer( + spark.sql("CALL paimon.sys.delete_tag(table => 'test.T', tag => 'tag-3, tag-4')"), + Row(true) :: Nil) + checkAnswer(spark.sql("SELECT tag_name FROM paimon.test.`T$tags`"), Nil) + } } diff --git a/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/procedure/CreateTagFromTimestampProcedureTest.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/procedure/CreateTagFromTimestampProcedureTest.scala similarity index 100% rename from paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/procedure/CreateTagFromTimestampProcedureTest.scala rename to paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/procedure/CreateTagFromTimestampProcedureTest.scala diff --git a/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/procedure/ExpirePartitionsProcedureTest.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/procedure/ExpirePartitionsProcedureTest.scala similarity index 90% rename from paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/procedure/ExpirePartitionsProcedureTest.scala rename to paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/procedure/ExpirePartitionsProcedureTest.scala index 4561e532f538..9f0d23dc9379 100644 --- a/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/procedure/ExpirePartitionsProcedureTest.scala +++ b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/procedure/ExpirePartitionsProcedureTest.scala @@ -551,4 +551,69 @@ class ExpirePartitionsProcedureTest extends PaimonSparkTestBase with StreamTest } } } + + test("Paimon Procedure: expire partitions with default num") { + failAfter(streamingTimeout) { + withTempDir { + checkpointDir => + spark.sql( + s""" + |CREATE TABLE T (k STRING, pt STRING) + |TBLPROPERTIES ('primary-key'='k,pt', 'bucket'='1', 'partition.expiration-max-num'='2') + |PARTITIONED BY (pt) + |""".stripMargin) + val location = loadTable("T").location().toString + + val inputData = MemoryStream[(String, String)] + val stream = inputData + .toDS() + .toDF("k", "pt") + .writeStream + .option("checkpointLocation", checkpointDir.getCanonicalPath) + .foreachBatch { + (batch: Dataset[Row], _: Long) => + batch.write.format("paimon").mode("append").save(location) + } + .start() + + val query = () => spark.sql("SELECT * FROM T") + + try { + // snapshot-1 + inputData.addData(("a", "2024-06-01")) + stream.processAllAvailable() + + // snapshot-2 + inputData.addData(("b", "2024-06-02")) + stream.processAllAvailable() + + // snapshot-3 + inputData.addData(("c", "2024-06-03")) + stream.processAllAvailable() + + // This partition never expires. + inputData.addData(("Never-expire", "9999-09-09")) + stream.processAllAvailable() + + checkAnswer( + query(), + Row("a", "2024-06-01") :: Row("b", "2024-06-02") :: Row("c", "2024-06-03") :: Row( + "Never-expire", + "9999-09-09") :: Nil) + // call expire_partitions. + checkAnswer( + spark.sql( + "CALL paimon.sys.expire_partitions(table => 'test.T', expiration_time => '1 d'" + + ", timestamp_formatter => 'yyyy-MM-dd')"), + Row("pt=2024-06-01") :: Row("pt=2024-06-02") :: Nil + ) + + checkAnswer(query(), Row("c", "2024-06-03") :: Row("Never-expire", "9999-09-09") :: Nil) + + } finally { + stream.stop() + } + } + } + } } diff --git a/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/procedure/ExpireSnapshotsProcedureTest.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/procedure/ExpireSnapshotsProcedureTest.scala similarity index 100% rename from paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/procedure/ExpireSnapshotsProcedureTest.scala rename to paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/procedure/ExpireSnapshotsProcedureTest.scala diff --git a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/procedure/ExpireTagsProcedureTest.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/procedure/ExpireTagsProcedureTest.scala new file mode 100644 index 000000000000..65c0f2b9a203 --- /dev/null +++ b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/procedure/ExpireTagsProcedureTest.scala @@ -0,0 +1,132 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.procedure; + +import org.apache.paimon.data.Timestamp +import org.apache.paimon.spark.PaimonSparkTestBase +import org.apache.paimon.utils.SnapshotManager + +import org.apache.spark.sql.Row +import org.assertj.core.api.Assertions.assertThat + +class ExpireTagsProcedureTest extends PaimonSparkTestBase { + + test("Paimon procedure: expire tags that reached its timeRetained") { + spark.sql(s""" + |CREATE TABLE T (id STRING, name STRING) + |USING PAIMON + |""".stripMargin) + + val table = loadTable("T") + val snapshotManager = table.snapshotManager() + + // generate 5 snapshots + for (i <- 1 to 5) { + spark.sql(s"INSERT INTO T VALUES($i, '$i')") + } + checkSnapshots(snapshotManager, 1, 5) + + spark.sql("CALL paimon.sys.create_tag(table => 'test.T', tag => 'tag-1', snapshot => 1)") + spark.sql( + "CALL paimon.sys.create_tag(table => 'test.T', tag => 'tag-2', snapshot => 2, time_retained => '1h')") + + // no tags expired + checkAnswer( + spark.sql("CALL paimon.sys.expire_tags(table => 'test.T')"), + Row("No expired tags.") :: Nil) + + spark.sql( + "CALL paimon.sys.create_tag(table => 'test.T', tag => 'tag-3', snapshot => 3, time_retained => '1s')") + spark.sql( + "CALL paimon.sys.create_tag(table => 'test.T', tag => 'tag-4', snapshot => 4, time_retained => '1s')") + checkAnswer(spark.sql("select count(tag_name) from `T$tags`"), Row(4) :: Nil) + + Thread.sleep(2000) + // tag-3,tag-4 expired + checkAnswer( + spark.sql("CALL paimon.sys.expire_tags(table => 'test.T')"), + Row("tag-3") :: Row("tag-4") :: Nil) + + checkAnswer(spark.sql("select tag_name from `T$tags`"), Row("tag-1") :: Row("tag-2") :: Nil) + } + + test("Paimon procedure: expire tags that createTime less than specified older_than") { + spark.sql(s""" + |CREATE TABLE T (id STRING, name STRING) + |USING PAIMON + |""".stripMargin) + + val table = loadTable("T") + val snapshotManager = table.snapshotManager() + + // generate 5 snapshots + for (i <- 1 to 5) { + spark.sql(s"INSERT INTO T VALUES($i, '$i')") + } + checkSnapshots(snapshotManager, 1, 5) + + spark.sql("CALL paimon.sys.create_tag(table => 'test.T', tag => 'tag-1', snapshot => 1)") + spark.sql( + "CALL paimon.sys.create_tag(table => 'test.T', tag => 'tag-2', snapshot => 2, time_retained => '1d')") + spark.sql( + "CALL paimon.sys.create_tag(table => 'test.T', tag => 'tag-3', snapshot => 3, time_retained => '1d')") + spark.sql( + "CALL paimon.sys.create_tag(table => 'test.T', tag => 'tag-4', snapshot => 4, time_retained => '1d')") + checkAnswer(spark.sql("select count(tag_name) from `T$tags`"), Row(4) :: Nil) + + // no tags expired + checkAnswer( + spark.sql("CALL paimon.sys.expire_tags(table => 'test.T')"), + Row("No expired tags.") :: Nil) + + // tag-2 as the base older_than time. + // tag-1 expired by its file creation time. + val olderThanTime1 = table.tagManager().tag("tag-2").getTagCreateTime + val timestamp1 = + new java.sql.Timestamp(Timestamp.fromLocalDateTime(olderThanTime1).getMillisecond) + checkAnswer( + spark.sql( + s"CALL paimon.sys.expire_tags(table => 'test.T', older_than => '${timestamp1.toString}')"), + Row("tag-1") :: Nil + ) + + spark.sql( + "CALL paimon.sys.create_tag(table => 'test.T', tag => 'tag-5', snapshot => 5, time_retained => '1s')") + Thread.sleep(1000) + + // tag-4 as the base older_than time. + // tag-2,tag-3,tag-5 expired, tag-5 reached its tagTimeRetained. + val olderThanTime2 = table.tagManager().tag("tag-4").getTagCreateTime + val timestamp2 = + new java.sql.Timestamp(Timestamp.fromLocalDateTime(olderThanTime2).getMillisecond) + checkAnswer( + spark.sql( + s"CALL paimon.sys.expire_tags(table => 'test.T', older_than => '${timestamp2.toString}')"), + Row("tag-2") :: Row("tag-3") :: Row("tag-5") :: Nil + ) + + checkAnswer(spark.sql("select tag_name from `T$tags`"), Row("tag-4") :: Nil) + } + + private def checkSnapshots(sm: SnapshotManager, earliest: Int, latest: Int): Unit = { + assertThat(sm.snapshotCount).isEqualTo(latest - earliest + 1) + assertThat(sm.earliestSnapshotId).isEqualTo(earliest) + assertThat(sm.latestSnapshotId).isEqualTo(latest) + } +} diff --git a/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/procedure/FastForwardProcedureTest.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/procedure/FastForwardProcedureTest.scala similarity index 100% rename from paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/procedure/FastForwardProcedureTest.scala rename to paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/procedure/FastForwardProcedureTest.scala diff --git a/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/procedure/MarkPartitionDoneProcedureTest.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/procedure/MarkPartitionDoneProcedureTest.scala similarity index 100% rename from paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/procedure/MarkPartitionDoneProcedureTest.scala rename to paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/procedure/MarkPartitionDoneProcedureTest.scala diff --git a/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/procedure/MigrateDatabaseProcedureTest.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/procedure/MigrateDatabaseProcedureTest.scala similarity index 100% rename from paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/procedure/MigrateDatabaseProcedureTest.scala rename to paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/procedure/MigrateDatabaseProcedureTest.scala diff --git a/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/procedure/MigrateFileProcedureTest.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/procedure/MigrateFileProcedureTest.scala similarity index 100% rename from paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/procedure/MigrateFileProcedureTest.scala rename to paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/procedure/MigrateFileProcedureTest.scala diff --git a/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/procedure/MigrateTableProcedureTest.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/procedure/MigrateTableProcedureTest.scala similarity index 100% rename from paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/procedure/MigrateTableProcedureTest.scala rename to paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/procedure/MigrateTableProcedureTest.scala diff --git a/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/procedure/ProcedureTestBase.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/procedure/ProcedureTestBase.scala similarity index 93% rename from paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/procedure/ProcedureTestBase.scala rename to paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/procedure/ProcedureTestBase.scala index f3cb7fa26665..a5f9f3ffa01b 100644 --- a/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/procedure/ProcedureTestBase.scala +++ b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/procedure/ProcedureTestBase.scala @@ -19,8 +19,8 @@ package org.apache.paimon.spark.procedure import org.apache.paimon.spark.PaimonSparkTestBase -import org.apache.paimon.spark.analysis.NoSuchProcedureException +import org.apache.spark.sql.catalyst.parser.ParseException import org.apache.spark.sql.catalyst.parser.extensions.PaimonParseException import org.assertj.core.api.Assertions.assertThatThrownBy @@ -32,7 +32,7 @@ abstract class ProcedureTestBase extends PaimonSparkTestBase { |""".stripMargin) assertThatThrownBy(() => spark.sql("CALL sys.unknown_procedure(table => 'test.T')")) - .isInstanceOf(classOf[NoSuchProcedureException]) + .isInstanceOf(classOf[ParseException]) } test(s"test parse exception") { diff --git a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/procedure/PurgeFilesProcedureTest.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/procedure/PurgeFilesProcedureTest.scala new file mode 100644 index 000000000000..27eafe1c3d03 --- /dev/null +++ b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/procedure/PurgeFilesProcedureTest.scala @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.procedure + +import org.apache.paimon.spark.PaimonSparkTestBase + +import org.apache.spark.sql.Row + +class PurgeFilesProcedureTest extends PaimonSparkTestBase { + + test("Paimon procedure: purge files test") { + spark.sql(s""" + |CREATE TABLE T (id STRING, name STRING) + |USING PAIMON + |""".stripMargin) + + spark.sql("insert into T select '1', 'aa'"); + checkAnswer(spark.sql("select * from test.T"), Row("1", "aa") :: Nil) + + spark.sql("CALL paimon.sys.purge_files(table => 'test.T')") + checkAnswer(spark.sql("select * from test.T"), Nil) + + spark.sql("refresh table test.T"); + spark.sql("insert into T select '2', 'aa'"); + checkAnswer(spark.sql("select * from test.T"), Row("2", "aa") :: Nil) + } + +} diff --git a/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/procedure/RemoveOrphanFilesProcedureTest.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/procedure/RemoveOrphanFilesProcedureTest.scala similarity index 94% rename from paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/procedure/RemoveOrphanFilesProcedureTest.scala rename to paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/procedure/RemoveOrphanFilesProcedureTest.scala index d9d73811266d..3ffe7fba264f 100644 --- a/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/procedure/RemoveOrphanFilesProcedureTest.scala +++ b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/procedure/RemoveOrphanFilesProcedureTest.scala @@ -52,7 +52,7 @@ class RemoveOrphanFilesProcedureTest extends PaimonSparkTestBase { fileIO.tryToWriteAtomic(orphanFile2, "b") // by default, no file deleted - checkAnswer(spark.sql(s"CALL sys.remove_orphan_files(table => 'T')"), Row(0) :: Nil) + checkAnswer(spark.sql(s"CALL sys.remove_orphan_files(table => 'T')"), Row(0, 0) :: Nil) val orphanFile2ModTime = fileIO.getFileStatus(orphanFile2).getModificationTime val older_than1 = DateTimeUtils.formatLocalDateTime( @@ -63,7 +63,7 @@ class RemoveOrphanFilesProcedureTest extends PaimonSparkTestBase { checkAnswer( spark.sql(s"CALL sys.remove_orphan_files(table => 'T', older_than => '$older_than1')"), - Row(1) :: Nil) + Row(1, 1) :: Nil) val older_than2 = DateTimeUtils.formatLocalDateTime( DateTimeUtils.toLocalDateTime(System.currentTimeMillis()), @@ -71,9 +71,9 @@ class RemoveOrphanFilesProcedureTest extends PaimonSparkTestBase { checkAnswer( spark.sql(s"CALL sys.remove_orphan_files(table => 'T', older_than => '$older_than2')"), - Row(1) :: Nil) + Row(1, 1) :: Nil) - checkAnswer(spark.sql(s"CALL sys.remove_orphan_files(table => 'T')"), Row(0) :: Nil) + checkAnswer(spark.sql(s"CALL sys.remove_orphan_files(table => 'T')"), Row(0, 0) :: Nil) } test("Paimon procedure: dry run remove orphan files") { @@ -97,7 +97,7 @@ class RemoveOrphanFilesProcedureTest extends PaimonSparkTestBase { fileIO.writeFile(orphanFile2, "b", true) // by default, no file deleted - checkAnswer(spark.sql(s"CALL sys.remove_orphan_files(table => 'T')"), Row(0) :: Nil) + checkAnswer(spark.sql(s"CALL sys.remove_orphan_files(table => 'T')"), Row(0, 0) :: Nil) val older_than = DateTimeUtils.formatLocalDateTime( DateTimeUtils.toLocalDateTime(System.currentTimeMillis()), @@ -106,10 +106,10 @@ class RemoveOrphanFilesProcedureTest extends PaimonSparkTestBase { checkAnswer( spark.sql( s"CALL sys.remove_orphan_files(table => 'T', older_than => '$older_than', dry_run => true)"), - Row(2) :: Nil + Row(2, 2) :: Nil ) - checkAnswer(spark.sql(s"CALL sys.remove_orphan_files(table => 'T')"), Row(0) :: Nil) + checkAnswer(spark.sql(s"CALL sys.remove_orphan_files(table => 'T')"), Row(0, 0) :: Nil) } test("Paimon procedure: remove database orphan files") { @@ -146,7 +146,7 @@ class RemoveOrphanFilesProcedureTest extends PaimonSparkTestBase { fileIO2.tryToWriteAtomic(orphanFile22, "b") // by default, no file deleted - checkAnswer(spark.sql(s"CALL sys.remove_orphan_files(table => 'test.*')"), Row(0) :: Nil) + checkAnswer(spark.sql(s"CALL sys.remove_orphan_files(table => 'test.*')"), Row(0, 0) :: Nil) val orphanFile12ModTime = fileIO1.getFileStatus(orphanFile12).getModificationTime val older_than1 = DateTimeUtils.formatLocalDateTime( @@ -157,7 +157,7 @@ class RemoveOrphanFilesProcedureTest extends PaimonSparkTestBase { checkAnswer( spark.sql(s"CALL sys.remove_orphan_files(table => 'test.*', older_than => '$older_than1')"), - Row(2) :: Nil + Row(2, 2) :: Nil ) val older_than2 = DateTimeUtils.formatLocalDateTime( @@ -166,10 +166,10 @@ class RemoveOrphanFilesProcedureTest extends PaimonSparkTestBase { checkAnswer( spark.sql(s"CALL sys.remove_orphan_files(table => 'test.*', older_than => '$older_than2')"), - Row(2) :: Nil + Row(2, 2) :: Nil ) - checkAnswer(spark.sql(s"CALL sys.remove_orphan_files(table => 'test.*')"), Row(0) :: Nil) + checkAnswer(spark.sql(s"CALL sys.remove_orphan_files(table => 'test.*')"), Row(0, 0) :: Nil) } test("Paimon procedure: remove orphan files with mode") { @@ -193,7 +193,7 @@ class RemoveOrphanFilesProcedureTest extends PaimonSparkTestBase { fileIO.tryToWriteAtomic(orphanFile2, "b") // by default, no file deleted - checkAnswer(spark.sql(s"CALL sys.remove_orphan_files(table => 'T')"), Row(0) :: Nil) + checkAnswer(spark.sql(s"CALL sys.remove_orphan_files(table => 'T')"), Row(0, 0) :: Nil) val orphanFile2ModTime = fileIO.getFileStatus(orphanFile2).getModificationTime val older_than1 = DateTimeUtils.formatLocalDateTime( @@ -205,7 +205,7 @@ class RemoveOrphanFilesProcedureTest extends PaimonSparkTestBase { checkAnswer( spark.sql( s"CALL sys.remove_orphan_files(table => 'T', older_than => '$older_than1', mode => 'diSTributed')"), - Row(1) :: Nil) + Row(1, 1) :: Nil) val older_than2 = DateTimeUtils.formatLocalDateTime( DateTimeUtils.toLocalDateTime(System.currentTimeMillis()), @@ -214,9 +214,9 @@ class RemoveOrphanFilesProcedureTest extends PaimonSparkTestBase { checkAnswer( spark.sql( s"CALL sys.remove_orphan_files(table => 'T', older_than => '$older_than2', mode => 'local')"), - Row(1) :: Nil) + Row(1, 1) :: Nil) - checkAnswer(spark.sql(s"CALL sys.remove_orphan_files(table => 'T')"), Row(0) :: Nil) + checkAnswer(spark.sql(s"CALL sys.remove_orphan_files(table => 'T')"), Row(0, 0) :: Nil) } } diff --git a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/procedure/ReplaceTagProcedureTest.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/procedure/ReplaceTagProcedureTest.scala new file mode 100644 index 000000000000..5a9280887031 --- /dev/null +++ b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/procedure/ReplaceTagProcedureTest.scala @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.procedure + +import org.apache.paimon.spark.PaimonSparkTestBase + +import org.apache.spark.sql.Row + +class ReplaceTagProcedureTest extends PaimonSparkTestBase { + test("Paimon Procedure: replace tag to update tag meta") { + spark.sql(s""" + |CREATE TABLE T (a INT, b STRING) + |TBLPROPERTIES ('primary-key'='a', 'bucket'='3') + |""".stripMargin) + spark.sql("insert into T values(1, 'a')") + spark.sql("insert into T values(2, 'b')") + assertResult(2)(loadTable("T").snapshotManager().snapshotCount()) + + // throw exception "Tag test_tag does not exist" + assertThrows[IllegalArgumentException] { + spark.sql("CALL paimon.sys.replace_tag(table => 'test.T', tag => 'test_tag')") + } + + spark.sql("CALL paimon.sys.create_tag(table => 'test.T', tag => 'test_tag')") + checkAnswer( + spark.sql("select tag_name,snapshot_id,time_retained from `T$tags`"), + Row("test_tag", 2, null) :: Nil) + + // replace tag with new time_retained + spark.sql( + "CALL paimon.sys.replace_tag(table => 'test.T', tag => 'test_tag', time_retained => '1 d')") + checkAnswer( + spark.sql("select tag_name,snapshot_id,time_retained from `T$tags`"), + Row("test_tag", 2, "PT24H") :: Nil) + + // replace tag with new snapshot and time_retained + spark.sql( + "CALL paimon.sys.replace_tag(table => 'test.T', tag => 'test_tag', snapshot => 1, time_retained => '2 d')") + checkAnswer( + spark.sql("select tag_name,snapshot_id,time_retained from `T$tags`"), + Row("test_tag", 1, "PT48H") :: Nil) + } +} diff --git a/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/procedure/RollbackProcedureTest.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/procedure/RollbackProcedureTest.scala similarity index 64% rename from paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/procedure/RollbackProcedureTest.scala rename to paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/procedure/RollbackProcedureTest.scala index 945f70ce0e63..457c5ba513ec 100644 --- a/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/procedure/RollbackProcedureTest.scala +++ b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/procedure/RollbackProcedureTest.scala @@ -93,4 +93,62 @@ class RollbackProcedureTest extends PaimonSparkTestBase with StreamTest { } } } + + test("Paimon Procedure: rollback to timestamp") { + failAfter(streamingTimeout) { + withTempDir { + checkpointDir => + // define a change-log table and test `forEachBatch` api + spark.sql(s""" + |CREATE TABLE T (a INT, b STRING) + |TBLPROPERTIES ('primary-key'='a', 'bucket'='3') + |""".stripMargin) + val location = loadTable("T").location().toString + + val inputData = MemoryStream[(Int, String)] + val stream = inputData + .toDS() + .toDF("a", "b") + .writeStream + .option("checkpointLocation", checkpointDir.getCanonicalPath) + .foreachBatch { + (batch: Dataset[Row], _: Long) => + batch.write.format("paimon").mode("append").save(location) + } + .start() + + val query = () => spark.sql("SELECT * FROM T ORDER BY a") + + try { + // snapshot-1 + inputData.addData((1, "a")) + stream.processAllAvailable() + checkAnswer(query(), Row(1, "a") :: Nil) + + // snapshot-2 + inputData.addData((2, "b")) + stream.processAllAvailable() + checkAnswer(query(), Row(1, "a") :: Row(2, "b") :: Nil) + + val timestamp = System.currentTimeMillis() + + // snapshot-3 + inputData.addData((2, "b2")) + stream.processAllAvailable() + checkAnswer(query(), Row(1, "a") :: Row(2, "b2") :: Nil) + + // rollback to timestamp + checkAnswer( + spark.sql( + s"CALL paimon.sys.rollback_to_timestamp(table => 'test.T', timestamp => $timestamp)"), + Row("Success roll back to snapshot: 2 .") :: Nil) + checkAnswer(query(), Row(1, "a") :: Row(2, "b") :: Nil) + + } finally { + stream.stop() + } + } + } + } + } diff --git a/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/AnalyzeTableTestBase.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/AnalyzeTableTestBase.scala similarity index 95% rename from paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/AnalyzeTableTestBase.scala rename to paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/AnalyzeTableTestBase.scala index 1ccf4e38ec4f..4f8ccae22dd5 100644 --- a/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/AnalyzeTableTestBase.scala +++ b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/AnalyzeTableTestBase.scala @@ -86,11 +86,22 @@ abstract class AnalyzeTableTestBase extends PaimonSparkTestBase { spark.sql(s"ANALYZE TABLE T COMPUTE STATISTICS") + withSparkSQLConf("spark.paimon.scan.timestamp-millis" -> System.currentTimeMillis.toString) { + checkAnswer( + sql("SELECT snapshot_id, schema_id, mergedRecordCount, colstat FROM `T$statistics`"), + Row(2, 0, 2, "{ }")) + } + spark.sql(s"INSERT INTO T VALUES ('3', 'b', 2, 1)") spark.sql(s"INSERT INTO T VALUES ('4', 'bbb', 3, 2)") spark.sql(s"ANALYZE TABLE T COMPUTE STATISTICS") + withSparkSQLConf("spark.paimon.scan.timestamp-millis" -> System.currentTimeMillis.toString) { + checkAnswer( + sql("SELECT snapshot_id, schema_id, mergedRecordCount, colstat FROM `T$statistics`"), + Row(5, 0, 4, "{ }")) + } // create tag checkAnswer( spark.sql("CALL paimon.sys.create_tag(table => 'test.T', tag => 'test_tag5', snapshot => 5)"), @@ -100,42 +111,35 @@ abstract class AnalyzeTableTestBase extends PaimonSparkTestBase { spark.sql("CALL paimon.sys.create_tag(table => 'test.T', tag => 'test_tag6', snapshot => 6)"), Row(true) :: Nil) - withSQLConf("spark.paimon.scan.tag-name" -> "test_tag5") { + withSparkSQLConf("spark.paimon.scan.tag-name" -> "test_tag5") { checkAnswer( sql("SELECT snapshot_id, schema_id, mergedRecordCount, colstat FROM `T$statistics`"), Row(2, 0, 2, "{ }")) } - withSQLConf("spark.paimon.scan.tag-name" -> "test_tag6") { + withSparkSQLConf("spark.paimon.scan.tag-name" -> "test_tag6") { checkAnswer( sql("SELECT snapshot_id, schema_id, mergedRecordCount, colstat FROM `T$statistics`"), Row(5, 0, 4, "{ }")) } - withSQLConf("spark.paimon.scan.snapshot-id" -> "3") { + withSparkSQLConf("spark.paimon.scan.snapshot-id" -> "3") { checkAnswer( sql("SELECT snapshot_id, schema_id, mergedRecordCount, colstat FROM `T$statistics`"), Row(2, 0, 2, "{ }")) } - withSQLConf("spark.paimon.scan.snapshot-id" -> "4") { + withSparkSQLConf("spark.paimon.scan.snapshot-id" -> "4") { checkAnswer( sql("SELECT snapshot_id, schema_id, mergedRecordCount, colstat FROM `T$statistics`"), Row(2, 0, 2, "{ }")) } - withSQLConf("spark.paimon.scan.snapshot-id" -> "6") { + withSparkSQLConf("spark.paimon.scan.snapshot-id" -> "6") { checkAnswer( sql("SELECT snapshot_id, schema_id, mergedRecordCount, colstat FROM `T$statistics`"), Row(5, 0, 4, "{ }")) } - - withSQLConf("spark.paimon.scan.snapshot-id" -> "100") { - checkAnswer( - sql("SELECT snapshot_id, schema_id, mergedRecordCount, colstat FROM `T$statistics`"), - Row(5, 0, 4, "{ }")) - } - } test("Paimon analyze: analyze table without snapshot") { @@ -403,7 +407,7 @@ abstract class AnalyzeTableTestBase extends PaimonSparkTestBase { spark.sql(s"ANALYZE TABLE T COMPUTE STATISTICS") val stats = getScanStatistic("SELECT * FROM T") - Assertions.assertEquals(2L, stats.rowCount.get.longValue()) + Assertions.assertEquals(2L, stats.rowCount.get.longValue) } test("Paimon analyze: spark use col stats") { @@ -418,7 +422,7 @@ abstract class AnalyzeTableTestBase extends PaimonSparkTestBase { spark.sql(s"ANALYZE TABLE T COMPUTE STATISTICS FOR ALL COLUMNS") val stats = getScanStatistic("SELECT * FROM T") - Assertions.assertEquals(2L, stats.rowCount.get.longValue()) + Assertions.assertEquals(2L, stats.rowCount.get.longValue) Assertions.assertEquals(if (gteqSpark3_4) 4 else 0, stats.attributeStats.size) } @@ -437,19 +441,19 @@ abstract class AnalyzeTableTestBase extends PaimonSparkTestBase { var sql = "SELECT * FROM T WHERE pt < 1" Assertions.assertEquals( if (gteqSpark3_4) 0L else 4L, - getScanStatistic(sql).rowCount.get.longValue()) + getScanStatistic(sql).rowCount.get.longValue) checkAnswer(spark.sql(sql), Nil) // partition push down hit and select without it sql = "SELECT id FROM T WHERE pt < 1" Assertions.assertEquals( if (gteqSpark3_4) 0L else 4L, - getScanStatistic(sql).rowCount.get.longValue()) + getScanStatistic(sql).rowCount.get.longValue) checkAnswer(spark.sql(sql), Nil) // partition push down not hit sql = "SELECT * FROM T WHERE id < 1" - Assertions.assertEquals(4L, getScanStatistic(sql).rowCount.get.longValue()) + Assertions.assertEquals(4L, getScanStatistic(sql).rowCount.get.longValue) checkAnswer(spark.sql(sql), Nil) } @@ -468,10 +472,10 @@ abstract class AnalyzeTableTestBase extends PaimonSparkTestBase { // For col type such as char, varchar that don't have min and max, filter estimation on stats has no effect. var sqlText = "SELECT * FROM T WHERE pt < '1'" - Assertions.assertEquals(4L, getScanStatistic(sqlText).rowCount.get.longValue()) + Assertions.assertEquals(4L, getScanStatistic(sqlText).rowCount.get.longValue) sqlText = "SELECT id FROM T WHERE pt < '1'" - Assertions.assertEquals(4L, getScanStatistic(sqlText).rowCount.get.longValue()) + Assertions.assertEquals(4L, getScanStatistic(sqlText).rowCount.get.longValue) } }) } diff --git a/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/BucketedTableQueryTest.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/BucketedTableQueryTest.scala similarity index 93% rename from paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/BucketedTableQueryTest.scala rename to paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/BucketedTableQueryTest.scala index b8009ea8136a..35931924c487 100644 --- a/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/BucketedTableQueryTest.scala +++ b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/BucketedTableQueryTest.scala @@ -29,12 +29,12 @@ class BucketedTableQueryTest extends PaimonSparkTestBase with AdaptiveSparkPlanH private def checkAnswerAndShuffleSorts(query: String, numShuffles: Int, numSorts: Int): Unit = { var expectedResult: Array[Row] = null // avoid config default value change in future, so specify it manually - withSQLConf( + withSparkSQLConf( "spark.sql.sources.v2.bucketing.enabled" -> "false", "spark.sql.autoBroadcastJoinThreshold" -> "-1") { expectedResult = spark.sql(query).collect() } - withSQLConf( + withSparkSQLConf( "spark.sql.sources.v2.bucketing.enabled" -> "true", "spark.sql.autoBroadcastJoinThreshold" -> "-1") { val df = spark.sql(query) @@ -122,7 +122,11 @@ class BucketedTableQueryTest extends PaimonSparkTestBase with AdaptiveSparkPlanH spark.sql( "CREATE TABLE t5 (id INT, c STRING) TBLPROPERTIES ('primary-key' = 'id', 'bucket'='10')") spark.sql("INSERT INTO t5 VALUES (1, 'x1')") - checkAnswerAndShuffleSorts("SELECT * FROM t1 JOIN t5 on t1.id = t5.id", 2, 2) + if (gteqSpark4_0) { + checkAnswerAndShuffleSorts("SELECT * FROM t1 JOIN t5 on t1.id = t5.id", 0, 0) + } else { + checkAnswerAndShuffleSorts("SELECT * FROM t1 JOIN t5 on t1.id = t5.id", 2, 2) + } // one more bucket keys spark.sql( @@ -152,16 +156,16 @@ class BucketedTableQueryTest extends PaimonSparkTestBase with AdaptiveSparkPlanH checkAnswerAndShuffleSorts("SELECT id, max(c) FROM t1 GROUP BY id", 0, 0) checkAnswerAndShuffleSorts("SELECT c, count(*) FROM t1 GROUP BY c", 1, 0) checkAnswerAndShuffleSorts("SELECT c, max(c) FROM t1 GROUP BY c", 1, 2) - checkAnswerAndShuffleSorts("select sum(c) OVER (PARTITION BY id ORDER BY c) from t1", 0, 1) + checkAnswerAndShuffleSorts("select max(c) OVER (PARTITION BY id ORDER BY c) from t1", 0, 1) // TODO: it is a Spark issue for `WindowExec` which would required partition-by + and order-by // without do distinct.. - checkAnswerAndShuffleSorts("select sum(c) OVER (PARTITION BY id ORDER BY id) from t1", 0, 1) + checkAnswerAndShuffleSorts("select max(c) OVER (PARTITION BY id ORDER BY id) from t1", 0, 1) checkAnswerAndShuffleSorts("select sum(id) OVER (PARTITION BY c ORDER BY id) from t1", 1, 1) - withSQLConf("spark.sql.requireAllClusterKeysForDistribution" -> "false") { + withSparkSQLConf("spark.sql.requireAllClusterKeysForDistribution" -> "false") { checkAnswerAndShuffleSorts("SELECT id, c, count(*) FROM t1 GROUP BY id, c", 0, 0) } - withSQLConf("spark.sql.requireAllClusterKeysForDistribution" -> "true") { + withSparkSQLConf("spark.sql.requireAllClusterKeysForDistribution" -> "true") { checkAnswerAndShuffleSorts("SELECT id, c, count(*) FROM t1 GROUP BY id, c", 1, 0) } } diff --git a/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/DDLTestBase.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/DDLTestBase.scala similarity index 83% rename from paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/DDLTestBase.scala rename to paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/DDLTestBase.scala index f1215ede4eeb..3ed2c98306fb 100644 --- a/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/DDLTestBase.scala +++ b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/DDLTestBase.scala @@ -27,7 +27,7 @@ import org.apache.spark.SparkException import org.apache.spark.sql.Row import org.junit.jupiter.api.Assertions -import java.sql.Timestamp +import java.sql.{Date, Timestamp} import java.time.LocalDateTime abstract class DDLTestBase extends PaimonSparkTestBase { @@ -161,7 +161,7 @@ abstract class DDLTestBase extends PaimonSparkTestBase { test("Paimon DDL: create table without using paimon") { withTable("paimon_tbl") { sql("CREATE TABLE paimon_tbl (id int)") - assert(loadTable("paimon_tbl").options().get("provider").equals("paimon")) + assert(!loadTable("paimon_tbl").options().containsKey("provider")) } } @@ -238,21 +238,21 @@ abstract class DDLTestBase extends PaimonSparkTestBase { |USING PAIMON |""".stripMargin) - withSQLConf("spark.sql.legacy.charVarcharAsString" -> "true") { + withSparkSQLConf("spark.sql.legacy.charVarcharAsString" -> "true") { sql("INSERT INTO paimon_tbl VALUES (1, 'ab')") } - withSQLConf("spark.sql.legacy.charVarcharAsString" -> "false") { + withSparkSQLConf("spark.sql.legacy.charVarcharAsString" -> "false") { sql("INSERT INTO paimon_tbl VALUES (2, 'ab')") } if (gteqSpark3_4) { - withSQLConf("spark.sql.readSideCharPadding" -> "true") { + withSparkSQLConf("spark.sql.readSideCharPadding" -> "true") { checkAnswer( spark.sql("SELECT c FROM paimon_tbl ORDER BY id"), Row("ab ") :: Row("ab ") :: Nil) } - withSQLConf("spark.sql.readSideCharPadding" -> "false") { + withSparkSQLConf("spark.sql.readSideCharPadding" -> "false") { checkAnswer( spark.sql("SELECT c FROM paimon_tbl ORDER BY id"), Row("ab") :: Row("ab ") :: Nil) @@ -270,7 +270,8 @@ abstract class DDLTestBase extends PaimonSparkTestBase { format => Seq(true, false).foreach { datetimeJava8APIEnabled => - withSQLConf("spark.sql.datetime.java8API.enabled" -> datetimeJava8APIEnabled.toString) { + withSparkSQLConf( + "spark.sql.datetime.java8API.enabled" -> datetimeJava8APIEnabled.toString) { withTimeZone("Asia/Shanghai") { withTable("paimon_tbl") { // Spark support create table with timestamp_ntz since 3.4 @@ -352,7 +353,7 @@ abstract class DDLTestBase extends PaimonSparkTestBase { .column("ts", DataTypes.TIMESTAMP_WITH_LOCAL_TIME_ZONE()) .column("ts_ntz", DataTypes.TIMESTAMP()) .build - catalog.createTable(identifier, schema, false) + paimonCatalog.createTable(identifier, schema, false) sql( s"INSERT INTO paimon_tbl VALUES (timestamp'2024-01-01 00:00:00', timestamp_ntz'2024-01-01 00:00:00')") @@ -370,7 +371,7 @@ abstract class DDLTestBase extends PaimonSparkTestBase { // Due to previous design, read timestamp ltz type with spark 3.3 and below will cause problems, // skip testing it if (gteqSpark3_4) { - val table = catalog.getTable(identifier) + val table = paimonCatalog.getTable(identifier) val builder = table.newReadBuilder.withProjection(Array[Int](0, 1)) val splits = builder.newScan().plan().splits() builder.newRead @@ -405,7 +406,7 @@ abstract class DDLTestBase extends PaimonSparkTestBase { // Due to previous design, read timestamp ltz type with spark 3.3 and below will cause problems, // skip testing it if (gteqSpark3_4) { - val table = catalog.getTable(identifier) + val table = paimonCatalog.getTable(identifier) val builder = table.newReadBuilder.withProjection(Array[Int](0, 1)) val splits = builder.newScan().plan().splits() builder.newRead @@ -423,14 +424,15 @@ abstract class DDLTestBase extends PaimonSparkTestBase { } } } finally { - catalog.dropTable(identifier, true) + paimonCatalog.dropTable(identifier, true) } } test("Paimon DDL: select table with timestamp and timestamp_ntz with filter") { Seq(true, false).foreach { datetimeJava8APIEnabled => - withSQLConf("spark.sql.datetime.java8API.enabled" -> datetimeJava8APIEnabled.toString) { + withSparkSQLConf( + "spark.sql.datetime.java8API.enabled" -> datetimeJava8APIEnabled.toString) { withTable("paimon_tbl") { // Spark support create table with timestamp_ntz since 3.4 if (gteqSpark3_4) { @@ -495,4 +497,79 @@ abstract class DDLTestBase extends PaimonSparkTestBase { }.getMessage assert(error.contains("Unsupported partition transform")) } + + test("Fix partition column generate wrong partition spec") { + Seq(true, false).foreach { + legacyPartName => + withTable("p_t") { + spark.sql(s""" + |CREATE TABLE p_t ( + | id BIGINT, + | c1 STRING + |) using paimon + |PARTITIONED BY (day binary) + |tblproperties('partition.legacy-name'='$legacyPartName'); + |""".stripMargin) + + if (legacyPartName) { + spark.sql("insert into table p_t values(1, 'a', cast('2021' as binary))") + intercept[Exception] { + spark.sql("SELECT * FROM p_t").collect() + } + } else { + spark.sql("insert into table p_t values(1, 'a', cast('2021' as binary))") + checkAnswer(spark.sql("SELECT * FROM p_t"), Row(1, "a", "2021".getBytes)) + val path = spark.sql("SELECT __paimon_file_path FROM p_t").collect() + assert(path.length == 1) + assert(path.head.getString(0).contains("/day=2021/")) + } + } + + withTable("p_t") { + spark.sql(s""" + |CREATE TABLE p_t ( + | id BIGINT, + | c1 STRING + |) using paimon + |PARTITIONED BY (day date) + |tblproperties('partition.legacy-name'='$legacyPartName'); + |""".stripMargin) + + spark.sql("insert into table p_t values(1, 'a', cast('2021-01-01' as date))") + checkAnswer(spark.sql("SELECT * FROM p_t"), Row(1, "a", Date.valueOf("2021-01-01"))) + + val path = spark.sql("SELECT __paimon_file_path FROM p_t").collect() + assert(path.length == 1) + if (legacyPartName) { + assert(path.head.getString(0).contains("/day=18628/")) + } else { + assert(path.head.getString(0).contains("/day=2021-01-01/")) + } + } + } + } + + test("Paimon DDL: create and drop external / managed table") { + withTempDir { + tbLocation => + withTable("external_tbl", "managed_tbl") { + // create external table + val error = intercept[UnsupportedOperationException] { + sql( + s"CREATE TABLE external_tbl (id INT) USING paimon LOCATION '${tbLocation.getCanonicalPath}'") + }.getMessage + assert(error.contains("not support")) + + // create managed table + sql("CREATE TABLE managed_tbl (id INT) USING paimon") + val table = loadTable("managed_tbl") + val fileIO = table.fileIO() + val tableLocation = table.location() + + // drop managed table + sql("DROP TABLE managed_tbl") + assert(!fileIO.exists(tableLocation)) + } + } + } } diff --git a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/DDLWithHiveCatalogTestBase.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/DDLWithHiveCatalogTestBase.scala new file mode 100644 index 000000000000..1189f1f2906b --- /dev/null +++ b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/DDLWithHiveCatalogTestBase.scala @@ -0,0 +1,485 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.sql + +import org.apache.paimon.hive.HiveMetastoreClient +import org.apache.paimon.spark.PaimonHiveTestBase +import org.apache.paimon.table.FileStoreTable + +import org.apache.spark.sql.{Row, SparkSession} +import org.junit.jupiter.api.Assertions + +abstract class DDLWithHiveCatalogTestBase extends PaimonHiveTestBase { + + test("Paimon DDL with hive catalog: create database with location and comment") { + Seq(sparkCatalogName, paimonHiveCatalogName).foreach { + catalogName => + spark.sql(s"USE $catalogName") + withTempDir { + dBLocation => + withDatabase("paimon_db") { + val comment = "this is a test comment" + spark.sql( + s"CREATE DATABASE paimon_db LOCATION '${dBLocation.getCanonicalPath}' COMMENT '$comment'") + Assertions.assertEquals(getDatabaseLocation("paimon_db"), dBLocation.getCanonicalPath) + Assertions.assertEquals(getDatabaseComment("paimon_db"), comment) + + withTable("paimon_db.paimon_tbl") { + spark.sql(s""" + |CREATE TABLE paimon_db.paimon_tbl (id STRING, name STRING, pt STRING) + |USING PAIMON + |TBLPROPERTIES ('primary-key' = 'id') + |""".stripMargin) + Assertions.assertEquals( + getTableLocation("paimon_db.paimon_tbl"), + s"${dBLocation.getCanonicalPath}/paimon_tbl") + + val fileStoreTable = getPaimonScan("SELECT * FROM paimon_db.paimon_tbl").table + .asInstanceOf[FileStoreTable] + Assertions.assertEquals("paimon_tbl", fileStoreTable.name()) + Assertions.assertEquals("paimon_db.paimon_tbl", fileStoreTable.fullName()) + } + } + } + } + } + + test("Paimon DDL with hive catalog: drop partition for paimon table sparkCatalogName") { + Seq(paimonHiveCatalogName).foreach { + catalogName => + spark.sql(s"USE $catalogName") + withTempDir { + dBLocation => + withDatabase("paimon_db") { + val comment = "this is a test comment" + spark.sql( + s"CREATE DATABASE paimon_db LOCATION '${dBLocation.getCanonicalPath}' COMMENT '$comment'") + Assertions.assertEquals(getDatabaseLocation("paimon_db"), dBLocation.getCanonicalPath) + Assertions.assertEquals(getDatabaseComment("paimon_db"), comment) + + withTable("paimon_db.paimon_tbl") { + spark.sql(s""" + |CREATE TABLE paimon_db.paimon_tbl (id STRING, name STRING, pt STRING) + |USING PAIMON + |PARTITIONED BY (name, pt) + |TBLPROPERTIES('metastore.partitioned-table' = 'true') + |""".stripMargin) + Assertions.assertEquals( + getTableLocation("paimon_db.paimon_tbl"), + s"${dBLocation.getCanonicalPath}/paimon_tbl") + spark.sql("insert into paimon_db.paimon_tbl select '1', 'n', 'cc'") + spark.sql("insert into paimon_db.paimon_tbl select '1', 'n1', 'aa'") + spark.sql("insert into paimon_db.paimon_tbl select '1', 'n2', 'bb'") + + spark.sql("show partitions paimon_db.paimon_tbl") + checkAnswer( + spark.sql("show partitions paimon_db.paimon_tbl"), + Row("name=n/pt=cc") :: Row("name=n1/pt=aa") :: Row("name=n2/pt=bb") :: Nil) + spark.sql( + "alter table paimon_db.paimon_tbl drop partition (name='n1', `pt`='aa'), partition (name='n2', `pt`='bb')") + spark.sql("show partitions paimon_db.paimon_tbl") + checkAnswer( + spark.sql("show partitions paimon_db.paimon_tbl"), + Row("name=n/pt=cc") :: Nil) + + } + + // disable metastore.partitioned-table + withTable("paimon_db.paimon_tbl2") { + spark.sql(s""" + |CREATE TABLE paimon_db.paimon_tbl2 (id STRING, name STRING, pt STRING) + |USING PAIMON + |PARTITIONED BY (name, pt) + |TBLPROPERTIES('metastore.partitioned-table' = 'false') + |""".stripMargin) + Assertions.assertEquals( + getTableLocation("paimon_db.paimon_tbl2"), + s"${dBLocation.getCanonicalPath}/paimon_tbl2") + spark.sql("insert into paimon_db.paimon_tbl2 select '1', 'n', 'cc'") + spark.sql("insert into paimon_db.paimon_tbl2 select '1', 'n1', 'aa'") + spark.sql("insert into paimon_db.paimon_tbl2 select '1', 'n2', 'bb'") + + spark.sql("show partitions paimon_db.paimon_tbl2") + checkAnswer( + spark.sql("show partitions paimon_db.paimon_tbl2"), + Row("name=n/pt=cc") :: Row("name=n1/pt=aa") :: Row("name=n2/pt=bb") :: Nil) + spark.sql( + "alter table paimon_db.paimon_tbl2 drop partition (name='n1', `pt`='aa'), partition (name='n2', `pt`='bb')") + spark.sql("show partitions paimon_db.paimon_tbl2") + checkAnswer( + spark.sql("show partitions paimon_db.paimon_tbl2"), + Row("name=n/pt=cc") :: Nil) + + } + } + } + } + } + + test("Paimon DDL with hive catalog: create partition for paimon table sparkCatalogName") { + Seq(paimonHiveCatalogName).foreach { + catalogName => + spark.sql(s"USE $catalogName") + withTempDir { + dBLocation => + withDatabase("paimon_db") { + val comment = "this is a test comment" + spark.sql( + s"CREATE DATABASE paimon_db LOCATION '${dBLocation.getCanonicalPath}' COMMENT '$comment'") + Assertions.assertEquals(getDatabaseLocation("paimon_db"), dBLocation.getCanonicalPath) + Assertions.assertEquals(getDatabaseComment("paimon_db"), comment) + + withTable("paimon_db.paimon_tbl") { + spark.sql(s""" + |CREATE TABLE paimon_db.paimon_tbl (id STRING, name STRING, pt STRING) + |USING PAIMON + |PARTITIONED BY (name, pt) + |TBLPROPERTIES('metastore.partitioned-table' = 'true') + |""".stripMargin) + Assertions.assertEquals( + getTableLocation("paimon_db.paimon_tbl"), + s"${dBLocation.getCanonicalPath}/paimon_tbl") + spark.sql("insert into paimon_db.paimon_tbl select '1', 'n', 'cc'") + + spark.sql("alter table paimon_db.paimon_tbl add partition(name='cc', `pt`='aa') ") + } + + // disable metastore.partitioned-table + withTable("paimon_db.paimon_tbl2") { + spark.sql(s""" + |CREATE TABLE paimon_db.paimon_tbl2 (id STRING, name STRING, pt STRING) + |USING PAIMON + |PARTITIONED BY (name, pt) + |TBLPROPERTIES('metastore.partitioned-table' = 'false') + |""".stripMargin) + Assertions.assertEquals( + getTableLocation("paimon_db.paimon_tbl2"), + s"${dBLocation.getCanonicalPath}/paimon_tbl2") + spark.sql("insert into paimon_db.paimon_tbl2 select '1', 'n', 'cc'") + + spark.sql("alter table paimon_db.paimon_tbl2 add partition(name='cc', `pt`='aa') ") + } + } + } + } + } + + test("Paimon DDL with hive catalog: create database with props") { + Seq(sparkCatalogName, paimonHiveCatalogName).foreach { + catalogName => + spark.sql(s"USE $catalogName") + withDatabase("paimon_db") { + spark.sql(s"CREATE DATABASE paimon_db WITH DBPROPERTIES ('k1' = 'v1', 'k2' = 'v2')") + val props = getDatabaseProps("paimon_db") + Assertions.assertEquals(props("k1"), "v1") + Assertions.assertEquals(props("k2"), "v2") + Assertions.assertTrue(getDatabaseOwner("paimon_db").nonEmpty) + } + } + } + + test("Paimon DDL with hive catalog: set default database") { + var reusedSpark = spark + + Seq("paimon", sparkCatalogName, paimonHiveCatalogName).foreach { + catalogName => + { + val dbName = s"${catalogName}_default_db" + val tblName = s"${dbName}_tbl" + + reusedSpark.sql(s"use $catalogName") + reusedSpark.sql(s"create database $dbName") + reusedSpark.sql(s"use $dbName") + reusedSpark.sql(s"create table $tblName (id int, name string, dt string) using paimon") + reusedSpark.stop() + + reusedSpark = SparkSession + .builder() + .master("local[2]") + .config(sparkConf) + .config("spark.sql.defaultCatalog", catalogName) + .config(s"spark.sql.catalog.$catalogName.defaultDatabase", dbName) + .getOrCreate() + + if (catalogName.equals(sparkCatalogName) && !gteqSpark3_4) { + checkAnswer(reusedSpark.sql("show tables").select("tableName"), Nil) + reusedSpark.sql(s"use $dbName") + } + checkAnswer(reusedSpark.sql("show tables").select("tableName"), Row(tblName) :: Nil) + + reusedSpark.sql(s"drop table $tblName") + } + } + + // Since we created a new sparkContext, we need to stop it and reset the default sparkContext + reusedSpark.stop() + reset() + } + + test("Paimon DDL with hive catalog: drop database cascade which contains paimon table") { + // Spark supports DROP DATABASE CASCADE since 3.3 + if (gteqSpark3_3) { + Seq(sparkCatalogName, paimonHiveCatalogName).foreach { + catalogName => + spark.sql(s"USE $catalogName") + spark.sql(s"CREATE DATABASE paimon_db") + spark.sql(s"USE paimon_db") + spark.sql(s"CREATE TABLE paimon_tbl (id int, name string, dt string) using paimon") + // Only spark_catalog supports create other table + if (catalogName.equals(sparkCatalogName)) { + spark.sql(s"CREATE TABLE parquet_tbl (id int, name string, dt string) using parquet") + spark.sql(s"CREATE VIEW parquet_tbl_view AS SELECT * FROM parquet_tbl") + } + spark.sql(s"CREATE VIEW paimon_tbl_view AS SELECT * FROM paimon_tbl") + spark.sql(s"USE default") + spark.sql(s"DROP DATABASE paimon_db CASCADE") + } + } + } + + test("Paimon DDL with hive catalog: sync partitions to HMS") { + Seq(sparkCatalogName, paimonHiveCatalogName).foreach { + catalogName => + val dbName = "default" + val tblName = "t" + spark.sql(s"USE $catalogName.$dbName") + withTable(tblName) { + spark.sql(s""" + |CREATE TABLE $tblName (id INT, pt INT) + |USING PAIMON + |TBLPROPERTIES ('metastore.partitioned-table' = 'true') + |PARTITIONED BY (pt) + |""".stripMargin) + + val metastoreClient = loadTable(dbName, tblName) + .catalogEnvironment() + .metastoreClientFactory() + .create() + .asInstanceOf[HiveMetastoreClient] + .client() + + spark.sql(s"INSERT INTO $tblName VALUES (1, 1), (2, 2), (3, 3)") + // check partitions in paimon + checkAnswer( + spark.sql(s"show partitions $tblName"), + Seq(Row("pt=1"), Row("pt=2"), Row("pt=3"))) + // check partitions in HMS + assert(metastoreClient.listPartitions(dbName, tblName, 100).size() == 3) + + spark.sql(s"INSERT INTO $tblName VALUES (4, 3), (5, 4)") + checkAnswer( + spark.sql(s"show partitions $tblName"), + Seq(Row("pt=1"), Row("pt=2"), Row("pt=3"), Row("pt=4"))) + assert(metastoreClient.listPartitions(dbName, tblName, 100).size() == 4) + + spark.sql(s"ALTER TABLE $tblName DROP PARTITION (pt=1)") + checkAnswer( + spark.sql(s"show partitions $tblName"), + Seq(Row("pt=2"), Row("pt=3"), Row("pt=4"))) + assert(metastoreClient.listPartitions(dbName, tblName, 100).size() == 3) + } + } + } + + test("Paimon DDL with hive catalog: create and drop external / managed table") { + Seq(sparkCatalogName, paimonHiveCatalogName).foreach { + catalogName => + spark.sql(s"USE $catalogName") + withTempDir { + tbLocation => + withDatabase("paimon_db") { + spark.sql(s"CREATE DATABASE IF NOT EXISTS paimon_db") + spark.sql(s"USE paimon_db") + withTable("external_tbl", "managed_tbl") { + val expertTbLocation = tbLocation.getCanonicalPath + // create external table + spark.sql( + s"CREATE TABLE external_tbl (id INT) USING paimon LOCATION '$expertTbLocation'") + spark.sql("INSERT INTO external_tbl VALUES (1)") + checkAnswer(spark.sql("SELECT * FROM external_tbl"), Row(1)) + val table = loadTable("paimon_db", "external_tbl") + val fileIO = table.fileIO() + val actualTbLocation = table.location() + assert(actualTbLocation.toString.split(':').apply(1).equals(expertTbLocation)) + + // drop external table + spark.sql("DROP TABLE external_tbl") + assert(fileIO.exists(actualTbLocation)) + + // create external table again using the same location + spark.sql( + s"CREATE TABLE external_tbl (id INT) USING paimon LOCATION '$expertTbLocation'") + checkAnswer(spark.sql("SELECT * FROM external_tbl"), Row(1)) + assert(getActualTableLocation("paimon_db", "external_tbl").equals(expertTbLocation)) + + // create managed table + spark.sql(s"CREATE TABLE managed_tbl (id INT) USING paimon") + val managedTbLocation = loadTable("paimon_db", "managed_tbl").location() + + // drop managed table + spark.sql("DROP TABLE managed_tbl") + assert(!fileIO.exists(managedTbLocation)) + } + } + } + } + } + + test("Paimon DDL with hive catalog: rename external / managed table") { + Seq(sparkCatalogName, paimonHiveCatalogName).foreach { + catalogName => + spark.sql(s"USE $catalogName") + withTempDir { + tbLocation => + withDatabase("paimon_db") { + spark.sql(s"CREATE DATABASE paimon_db") + spark.sql(s"USE paimon_db") + withTable( + "external_tbl", + "managed_tbl", + "external_tbl_renamed", + "managed_tbl_renamed") { + val expertTbLocation = tbLocation.getCanonicalPath + // create external table + spark.sql( + s"CREATE TABLE external_tbl (id INT) USING paimon LOCATION '$expertTbLocation'") + spark.sql("INSERT INTO external_tbl VALUES (1)") + val actualTbLocation = loadTable("paimon_db", "external_tbl").location() + assert(actualTbLocation.toString.split(':').apply(1).equals(expertTbLocation)) + + // rename external table, location should not change + spark.sql("ALTER TABLE external_tbl RENAME TO external_tbl_renamed") + checkAnswer(spark.sql("SELECT * FROM external_tbl_renamed"), Row(1)) + assert( + getActualTableLocation("paimon_db", "external_tbl_renamed").equals( + expertTbLocation)) + + // create managed table + spark.sql(s"CREATE TABLE managed_tbl (id INT) USING paimon") + spark.sql("INSERT INTO managed_tbl VALUES (1)") + val managedTbLocation = loadTable("paimon_db", "managed_tbl").location() + + // rename managed table, location should change + spark.sql("ALTER TABLE managed_tbl RENAME TO managed_tbl_renamed") + checkAnswer(spark.sql("SELECT * FROM managed_tbl_renamed"), Row(1)) + assert( + !getActualTableLocation("paimon_db", "managed_tbl_renamed").equals( + managedTbLocation.toString)) + } + } + } + } + } + + test("Paimon DDL with hive catalog: create external table without schema") { + Seq(sparkCatalogName, paimonHiveCatalogName).foreach { + catalogName => + spark.sql(s"USE $catalogName") + withTempDir { + tbLocation => + withDatabase("paimon_db") { + spark.sql(s"CREATE DATABASE IF NOT EXISTS paimon_db") + spark.sql(s"USE paimon_db") + withTable("t1", "t2", "t3", "t4", "t5") { + val expertTbLocation = tbLocation.getCanonicalPath + spark.sql(s""" + |CREATE TABLE t1 (id INT, pt INT) USING paimon + |PARTITIONED BY (pt) + |TBLPROPERTIES('primary-key' = 'id', 'k1' = 'v1') + |LOCATION '$expertTbLocation' + |""".stripMargin) + spark.sql("INSERT INTO t1 VALUES (1, 1)") + + // create table without schema + spark.sql(s"CREATE TABLE t2 USING paimon LOCATION '$expertTbLocation'") + checkAnswer(spark.sql("SELECT * FROM t2"), Row(1, 1)) + assert(getActualTableLocation("paimon_db", "t2").equals(expertTbLocation)) + + // create table with wrong schema + intercept[Exception] { + spark.sql( + s"CREATE TABLE t3 (fake_col INT) USING paimon LOCATION '$expertTbLocation'") + } + + // create table with exists props + spark.sql( + s"CREATE TABLE t4 USING paimon TBLPROPERTIES ('k1' = 'v1') LOCATION '$expertTbLocation'") + checkAnswer(spark.sql("SELECT * FROM t4"), Row(1, 1)) + assert(getActualTableLocation("paimon_db", "t4").equals(expertTbLocation)) + + // create table with new props + intercept[Exception] { + spark.sql( + s"CREATE TABLE t5 USING paimon TBLPROPERTIES ('k2' = 'v2') LOCATION '$expertTbLocation'") + } + } + } + } + } + } + + def getDatabaseProp(dbName: String, propertyName: String): String = { + spark + .sql(s"DESC DATABASE EXTENDED $dbName") + .filter(s"info_name == '$propertyName'") + .head() + .getAs[String]("info_value") + } + + def getDatabaseLocation(dbName: String): String = + getDatabaseProp(dbName, "Location").split(":")(1) + + def getDatabaseComment(dbName: String): String = getDatabaseProp(dbName, "Comment") + + def getDatabaseOwner(dbName: String): String = getDatabaseProp(dbName, "Owner") + + def getDatabaseProps(dbName: String): Map[String, String] = { + val dbPropsStr = getDatabaseProp(dbName, "Properties") + val pattern = "\\(([^,]+),([^)]+)\\)".r + pattern + .findAllIn(dbPropsStr.drop(1).dropRight(1)) + .matchData + .map { + m => + val key = m.group(1).trim + val value = m.group(2).trim + (key, value) + } + .toMap + } + + def getTableLocation(tblName: String): String = { + val tablePropsStr = spark + .sql(s"DESC TABLE EXTENDED $tblName") + .filter("col_name == 'Table Properties'") + .head() + .getAs[String]("data_type") + val tableProps = tablePropsStr + .substring(1, tablePropsStr.length - 1) + .split(",") + .map(_.split("=")) + .map { case Array(key, value) => (key, value) } + .toMap + tableProps("path").split(":")(1) + } + + def getActualTableLocation(dbName: String, tblName: String): String = { + loadTable(dbName, tblName).location().toString.split(':').apply(1) + } +} diff --git a/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/DataFrameWriteTest.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/DataFrameWriteTest.scala similarity index 98% rename from paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/DataFrameWriteTest.scala rename to paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/DataFrameWriteTest.scala index 3f6e81da018c..edd092c85ce8 100644 --- a/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/DataFrameWriteTest.scala +++ b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/DataFrameWriteTest.scala @@ -20,6 +20,7 @@ package org.apache.paimon.spark.sql import org.apache.paimon.spark.PaimonSparkTestBase +import org.apache.spark.SparkConf import org.apache.spark.sql.Row import org.apache.spark.sql.types.DecimalType import org.junit.jupiter.api.Assertions @@ -27,6 +28,11 @@ import org.junit.jupiter.api.Assertions import java.sql.{Date, Timestamp} class DataFrameWriteTest extends PaimonSparkTestBase { + + override protected def sparkConf: SparkConf = { + super.sparkConf.set("spark.sql.catalog.paimon.cache-enabled", "false") + } + import testImplicits._ test("Paimon: DataFrameWrite.saveAsTable") { @@ -473,7 +479,11 @@ class DataFrameWriteTest extends PaimonSparkTestBase { .writeTo("t") .overwrite($"c1" === ($"c2" + 1)) }.getMessage - assert(msg3.contains("cannot translate expression to source filter")) + if (gteqSpark4_0) { + assert(msg3.contains("Table does not support overwrite by expression")) + } else { + assert(msg3.contains("cannot translate expression to source filter")) + } val msg4 = intercept[Exception] { spark diff --git a/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/DeleteFromTableTestBase.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/DeleteFromTableTestBase.scala similarity index 100% rename from paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/DeleteFromTableTestBase.scala rename to paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/DeleteFromTableTestBase.scala diff --git a/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/DeletionVectorTest.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/DeletionVectorTest.scala similarity index 95% rename from paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/DeletionVectorTest.scala rename to paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/DeletionVectorTest.scala index e944429e4218..ec5526f20e1d 100644 --- a/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/DeletionVectorTest.scala +++ b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/DeletionVectorTest.scala @@ -25,13 +25,13 @@ import org.apache.paimon.spark.{PaimonSparkTestBase, PaimonSplitScan} import org.apache.paimon.spark.schema.PaimonMetadataColumn import org.apache.paimon.table.FileStoreTable -import org.apache.spark.paimon.Utils import org.apache.spark.sql.Row import org.apache.spark.sql.execution.{QueryExecution, SparkPlan} import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper import org.apache.spark.sql.execution.columnar.InMemoryTableScanExec import org.apache.spark.sql.execution.datasources.v2.{BatchScanExec, DataSourceV2Relation} import org.apache.spark.sql.functions.lit +import org.apache.spark.sql.paimon.Utils import org.apache.spark.sql.util.QueryExecutionListener import org.junit.jupiter.api.Assertions @@ -631,6 +631,43 @@ class DeletionVectorTest extends PaimonSparkTestBase with AdaptiveSparkPlanHelpe ) } + test("Paimon deletionVector: get cardinality") { + sql(s""" + |CREATE TABLE T (id INT) + |TBLPROPERTIES ( + | 'deletion-vectors.enabled' = 'true', + | 'bucket-key' = 'id', + | 'bucket' = '1' + |) + |""".stripMargin) + + sql("INSERT INTO T SELECT /*+ REPARTITION(1) */ id FROM range (1, 50000)") + sql("DELETE FROM T WHERE id >= 111 and id <= 444") + + val fileStore = loadTable("T").store() + val indexManifest = fileStore.snapshotManager().latestSnapshot().indexManifest() + val entry = fileStore.newIndexFileHandler().readManifest(indexManifest).get(0) + val dvMeta = entry.indexFile().deletionVectorMetas().values().iterator().next() + + assert(dvMeta.cardinality() == 334) + } + + test("Paimon deletionVector: delete from non-pk table with data file path") { + sql(s""" + |CREATE TABLE T (id INT) + |TBLPROPERTIES ( + | 'deletion-vectors.enabled' = 'true', + | 'bucket-key' = 'id', + | 'bucket' = '1', + | 'data-file.path-directory' = 'data' + |) + |""".stripMargin) + + sql("INSERT INTO T SELECT /*+ REPARTITION(1) */ id FROM range (1, 50000)") + sql("DELETE FROM T WHERE id >= 111 and id <= 444") + checkAnswer(sql("SELECT count(*) FROM T"), Row(49665)) + } + private def getPathName(path: String): String = { new Path(path).getName } diff --git a/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/DescribeTableTest.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/DescribeTableTest.scala similarity index 51% rename from paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/DescribeTableTest.scala rename to paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/DescribeTableTest.scala index 528dcd3cd107..ae538fa48c4e 100644 --- a/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/DescribeTableTest.scala +++ b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/DescribeTableTest.scala @@ -27,6 +27,76 @@ import java.util.Objects class DescribeTableTest extends PaimonSparkTestBase { + test("Paimon show: show table extended") { + val testDB = "test_show" + withDatabase(testDB) { + spark.sql("CREATE TABLE s1 (id INT)") + + spark.sql(s"CREATE DATABASE $testDB") + spark.sql(s"USE $testDB") + spark.sql("CREATE TABLE s2 (id INT, pt STRING) PARTITIONED BY (pt)") + spark.sql("CREATE TABLE s3 (id INT, pt1 STRING, pt2 STRING) PARTITIONED BY (pt1, pt2)") + + spark.sql("INSERT INTO s2 VALUES (1, '2024'), (2, '2024'), (3, '2025'), (4, '2026')") + spark.sql(""" + |INSERT INTO s3 + |VALUES + |(1, '2024', '11'), (2, '2024', '12'), (3, '2025', '11'), (4, '2025', '12') + |""".stripMargin) + + // SHOW TABL EXTENDED will give four columns: namespace, tableName, isTemporary, information. + checkAnswer( + sql(s"SHOW TABLE EXTENDED IN $dbName0 LIKE '*'") + .select("namespace", "tableName", "isTemporary"), + Row("test", "s1", false)) + checkAnswer( + sql(s"SHOW TABLE EXTENDED IN $testDB LIKE '*'") + .select("namespace", "tableName", "isTemporary"), + Row(testDB, "s2", false) :: Row(testDB, "s3", false) :: Nil + ) + + // check table s1 + val res1 = spark.sql(s"SHOW TABLE EXTENDED IN $testDB LIKE 's2'").select("information") + Assertions.assertEquals(1, res1.count()) + val information1 = res1 + .collect() + .head + .getString(0) + .split("\n") + .map { + line => + val kv = line.split(": ", 2) + kv(0) -> kv(1) + } + .toMap + Assertions.assertEquals(information1("Catalog"), "paimon") + Assertions.assertEquals(information1("Namespace"), testDB) + Assertions.assertEquals(information1("Table"), "s2") + Assertions.assertEquals(information1("Provider"), "paimon") + Assertions.assertEquals(information1("Location"), loadTable(testDB, "s2").location().toString) + + // check table s2 partition info + val error1 = intercept[Exception] { + spark.sql(s"SHOW TABLE EXTENDED IN $testDB LIKE 's2' PARTITION(pt='2022')") + }.getMessage + assert(error1.contains("PARTITIONS_NOT_FOUND")) + + val error2 = intercept[Exception] { + spark.sql(s"SHOW TABLE EXTENDED IN $testDB LIKE 's3' PARTITION(pt1='2024')") + }.getMessage + assert(error2.contains("Partition spec is invalid")) + + val res2 = + spark.sql(s"SHOW TABLE EXTENDED IN $testDB LIKE 's3' PARTITION(pt1 = '2024', pt2 = 11)") + checkAnswer( + res2.select("namespace", "tableName", "isTemporary"), + Row(testDB, "s3", false) + ) + Assertions.assertTrue( + res2.select("information").collect().head.getString(0).contains("Partition Values")) + } + } + test(s"Paimon describe: describe table comment") { var comment = "test comment" spark.sql(s""" diff --git a/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/DisableUnnecessaryPaimonBucketedScanSuite.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/DisableUnnecessaryPaimonBucketedScanSuite.scala similarity index 97% rename from paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/DisableUnnecessaryPaimonBucketedScanSuite.scala rename to paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/DisableUnnecessaryPaimonBucketedScanSuite.scala index 70339bd7cac3..f47d40285aa9 100644 --- a/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/DisableUnnecessaryPaimonBucketedScanSuite.scala +++ b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/DisableUnnecessaryPaimonBucketedScanSuite.scala @@ -48,13 +48,13 @@ class DisableUnnecessaryPaimonBucketedScanSuite assert(bucketedScan.length == expectedNumBucketedScan, query) } - withSQLConf("spark.sql.sources.v2.bucketing.enabled" -> "true") { - withSQLConf("spark.sql.sources.bucketing.autoBucketedScan.enabled" -> "true") { + withSparkSQLConf("spark.sql.sources.v2.bucketing.enabled" -> "true") { + withSparkSQLConf("spark.sql.sources.bucketing.autoBucketedScan.enabled" -> "true") { val df = sql(query) val result = df.collect() checkNumBucketedScan(df, expectedNumScanWithAutoScanEnabled) - withSQLConf("spark.sql.sources.bucketing.autoBucketedScan.enabled" -> "false") { + withSparkSQLConf("spark.sql.sources.bucketing.autoBucketedScan.enabled" -> "false") { val expected = sql(query) checkAnswer(expected, result) checkNumBucketedScan(expected, expectedNumScanWithAutoScanDisabled) diff --git a/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/DynamicBucketTableTest.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/DynamicBucketTableTest.scala similarity index 100% rename from paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/DynamicBucketTableTest.scala rename to paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/DynamicBucketTableTest.scala diff --git a/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/InsertOverwriteTableTestBase.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/InsertOverwriteTableTestBase.scala similarity index 88% rename from paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/InsertOverwriteTableTestBase.scala rename to paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/InsertOverwriteTableTestBase.scala index 674b45fda68b..977b74707069 100644 --- a/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/InsertOverwriteTableTestBase.scala +++ b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/InsertOverwriteTableTestBase.scala @@ -346,7 +346,7 @@ abstract class InsertOverwriteTableTestBase extends PaimonSparkTestBase { spark.sql("SELECT * FROM T ORDER BY a, b"), Row(1, 3, "3") :: Row(2, 4, "4") :: Nil) - withSQLConf("spark.sql.sources.partitionOverwriteMode" -> "dynamic") { + withSparkSQLConf("spark.sql.sources.partitionOverwriteMode" -> "dynamic") { // dynamic overwrite the a=1 partition spark.sql("INSERT OVERWRITE T VALUES (1, 5, '5'), (1, 7, '7')") checkAnswer( @@ -387,7 +387,7 @@ abstract class InsertOverwriteTableTestBase extends PaimonSparkTestBase { "ptv2", 22) :: Nil) - withSQLConf("spark.sql.sources.partitionOverwriteMode" -> "dynamic") { + withSparkSQLConf("spark.sql.sources.partitionOverwriteMode" -> "dynamic") { // dynamic overwrite the pt2=22 partition spark.sql( "INSERT OVERWRITE T PARTITION (pt2 = 22) VALUES (3, 'c2', 'ptv1'), (4, 'd2', 'ptv3')") @@ -508,4 +508,56 @@ abstract class InsertOverwriteTableTestBase extends PaimonSparkTestBase { ) :: Nil ) } + + test("Paimon Insert: insert with column list") { + sql("CREATE TABLE T (name String, student_id INT) PARTITIONED BY (address STRING)") + + // insert with a column list + sql("INSERT INTO T (name, student_id, address) VALUES ('a', '1', 'Hangzhou')") + // Since Spark 3.4, INSERT INTO commands with explicit column lists comprising fewer columns than the target + // table will automatically add the corresponding default values for the remaining columns (or NULL for any column + // lacking an explicitly-assigned default value). In Spark 3.3 or earlier, these commands would have failed. + // See https://issues.apache.org/jira/browse/SPARK-42521 + if (gteqSpark3_4) { + sql("INSERT INTO T (name) VALUES ('b')") + sql("INSERT INTO T (address, name) VALUES ('Hangzhou', 'c')") + } else { + sql("INSERT INTO T (name, student_id, address) VALUES ('b', null, null)") + sql("INSERT INTO T (name, student_id, address) VALUES ('c', null, 'Hangzhou')") + } + + // insert with both a partition spec and a column list + if (gteqSpark3_4) { + sql("INSERT INTO T PARTITION (address='Beijing') (name) VALUES ('d')") + } else { + sql("INSERT INTO T PARTITION (address='Beijing') (name, student_id) VALUES ('d', null)") + } + sql("INSERT INTO T PARTITION (address='Hangzhou') (student_id, name) VALUES (5, 'e')") + + checkAnswer( + sql("SELECT * FROM T ORDER BY name"), + Seq( + Row("a", 1, "Hangzhou"), + Row("b", null, null), + Row("c", null, "Hangzhou"), + Row("d", null, "Beijing"), + Row("e", 5, "Hangzhou")) + ) + + // insert overwrite with a column list + if (gteqSpark3_4) { + sql("INSERT OVERWRITE T (name, address) VALUES ('f', 'Shanghai')") + } else { + sql("INSERT OVERWRITE T (name, student_id, address) VALUES ('f', null, 'Shanghai')") + } + checkAnswer(sql("SELECT * FROM T ORDER BY name"), Row("f", null, "Shanghai")) + + // insert overwrite with both a partition spec and a column list + if (gteqSpark3_4) { + sql("INSERT OVERWRITE T PARTITION (address='Shanghai') (name) VALUES ('g')") + } else { + sql("INSERT OVERWRITE T PARTITION (address='Shanghai') (name, student_id) VALUES ('g', null)") + } + checkAnswer(sql("SELECT * FROM T ORDER BY name"), Row("g", null, "Shanghai")) + } } diff --git a/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/LookupCompactionTest.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/LookupCompactionTest.scala similarity index 100% rename from paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/LookupCompactionTest.scala rename to paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/LookupCompactionTest.scala diff --git a/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/MergeIntoNotMatchedBySourceTest.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/MergeIntoNotMatchedBySourceTest.scala similarity index 100% rename from paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/MergeIntoNotMatchedBySourceTest.scala rename to paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/MergeIntoNotMatchedBySourceTest.scala diff --git a/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/MergeIntoTableTestBase.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/MergeIntoTableTestBase.scala similarity index 95% rename from paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/MergeIntoTableTestBase.scala rename to paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/MergeIntoTableTestBase.scala index 8973ea93d8a0..bcd84fdc11da 100644 --- a/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/MergeIntoTableTestBase.scala +++ b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/MergeIntoTableTestBase.scala @@ -113,6 +113,35 @@ abstract class MergeIntoTableTestBase extends PaimonSparkTestBase with PaimonTab } } + test(s"Paimon MergeInto: update + insert with data file path") { + withTable("source", "target") { + + Seq((1, 100, "c11"), (3, 300, "c33")).toDF("a", "b", "c").createOrReplaceTempView("source") + + createTable( + "target", + "a INT, b INT, c STRING", + Seq("a"), + Seq(), + Map("data-file.path-directory" -> "data")) + spark.sql("INSERT INTO target values (1, 10, 'c1'), (2, 20, 'c2')") + + spark.sql(s""" + |MERGE INTO target + |USING source + |ON target.a = source.a + |WHEN MATCHED THEN + |UPDATE SET a = source.a, b = source.b, c = source.c + |WHEN NOT MATCHED + |THEN INSERT (a, b, c) values (a, b, c) + |""".stripMargin) + + checkAnswer( + spark.sql("SELECT * FROM target ORDER BY a, b"), + Seq(Row(1, 100, "c11"), Row(2, 20, "c2"), Row(3, 300, "c33"))) + } + } + test(s"Paimon MergeInto: delete + insert") { withTable("source", "target") { diff --git a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/ObjectTableTest.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/ObjectTableTest.scala new file mode 100644 index 000000000000..3a446e33d8c8 --- /dev/null +++ b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/ObjectTableTest.scala @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.sql + +import org.apache.paimon.fs.Path +import org.apache.paimon.fs.local.LocalFileIO +import org.apache.paimon.spark.PaimonSparkTestBase + +import org.apache.spark.sql.Row + +class ObjectTableTest extends PaimonSparkTestBase { + + test(s"Paimon object table") { + val objectLocation = new Path(tempDBDir + "/object-location") + val fileIO = LocalFileIO.create + + spark.sql(s""" + |CREATE TABLE T TBLPROPERTIES ( + | 'type' = 'object-table', + | 'object-location' = '$objectLocation' + |) + |""".stripMargin) + + // add new file + fileIO.overwriteFileUtf8(new Path(objectLocation, "f0"), "1,2,3") + spark.sql("CALL sys.refresh_object_table('test.T')") + checkAnswer( + spark.sql("SELECT name, length FROM T"), + Row("f0", 5L) :: Nil + ) + + // add new file + fileIO.overwriteFileUtf8(new Path(objectLocation, "f1"), "4,5,6") + spark.sql("CALL sys.refresh_object_table('test.T')") + checkAnswer( + spark.sql("SELECT name, length FROM T"), + Row("f0", 5L) :: Row("f1", 5L) :: Nil + ) + + // time travel + checkAnswer( + spark.sql("SELECT name, length FROM T VERSION AS OF 1"), + Row("f0", 5L) :: Nil + ) + } +} diff --git a/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/PaimonCompositePartitionKeyTestBase.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/PaimonCompositePartitionKeyTestBase.scala similarity index 100% rename from paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/PaimonCompositePartitionKeyTestBase.scala rename to paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/PaimonCompositePartitionKeyTestBase.scala diff --git a/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/PaimonFunctionTest.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/PaimonFunctionTest.scala similarity index 100% rename from paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/PaimonFunctionTest.scala rename to paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/PaimonFunctionTest.scala diff --git a/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/PaimonMetricTest.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/PaimonMetricTest.scala similarity index 100% rename from paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/PaimonMetricTest.scala rename to paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/PaimonMetricTest.scala diff --git a/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/PaimonOptimizationTestBase.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/PaimonOptimizationTestBase.scala similarity index 90% rename from paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/PaimonOptimizationTestBase.scala rename to paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/PaimonOptimizationTestBase.scala index 78e8905fa969..87f4c9448619 100644 --- a/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/PaimonOptimizationTestBase.scala +++ b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/PaimonOptimizationTestBase.scala @@ -20,6 +20,7 @@ package org.apache.paimon.spark.sql import org.apache.paimon.Snapshot.CommitKind import org.apache.paimon.spark.PaimonSparkTestBase +import org.apache.paimon.spark.catalyst.analysis.expressions.ExpressionHelper import org.apache.paimon.spark.catalyst.optimizer.MergePaimonScalarSubqueries import org.apache.spark.sql.Row @@ -27,11 +28,12 @@ import org.apache.spark.sql.catalyst.expressions.{Attribute, CreateNamedStruct, import org.apache.spark.sql.catalyst.plans.logical.{CTERelationDef, LogicalPlan, OneRowRelation, WithCTE} import org.apache.spark.sql.catalyst.rules.RuleExecutor import org.apache.spark.sql.functions._ +import org.apache.spark.sql.paimon.Utils import org.junit.jupiter.api.Assertions import scala.collection.immutable -abstract class PaimonOptimizationTestBase extends PaimonSparkTestBase { +abstract class PaimonOptimizationTestBase extends PaimonSparkTestBase with ExpressionHelper { import org.apache.spark.sql.catalyst.dsl.expressions._ import org.apache.spark.sql.catalyst.dsl.plans._ @@ -59,24 +61,25 @@ abstract class PaimonOptimizationTestBase extends PaimonSparkTestBase { |""".stripMargin) val optimizedPlan = Optimize.execute(query.queryExecution.analyzed) - val relation = createRelationV2("T") - val mergedSubquery = relation + val df = Utils.createDataFrame(spark, createRelationV2("T")) + val mergedSubquery = df .select( - count(Literal(1)).as("cnt"), - sum(col("a").expr).as("sum_a"), - avg(col("b").expr).as("avg_b") + toColumn(count(Literal(1))).as("cnt"), + toColumn(sum(toExpression(spark, col("a")))).as("sum_a"), + toColumn(avg(toExpression(spark, col("b"))).as("avg_b")) ) .select( - CreateNamedStruct( - Seq( - Literal("cnt"), - 'cnt, - Literal("sum_a"), - 'sum_a, - Literal("avg_b"), - 'avg_b - )).as("mergedValue")) - val analyzedMergedSubquery = mergedSubquery.analyze + toColumn( + CreateNamedStruct( + Seq( + Literal("cnt"), + 'cnt, + Literal("sum_a"), + 'sum_a, + Literal("avg_b"), + 'avg_b + )).as("mergedValue"))) + val analyzedMergedSubquery = mergedSubquery.queryExecution.analyzed val correctAnswer = WithCTE( OneRowRelation() .select( diff --git a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/PaimonOptionTest.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/PaimonOptionTest.scala new file mode 100644 index 000000000000..44df3e54ca72 --- /dev/null +++ b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/PaimonOptionTest.scala @@ -0,0 +1,205 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.sql + +import org.apache.paimon.spark.PaimonSparkTestBase +import org.apache.paimon.table.FileStoreTableFactory + +import org.apache.spark.sql.Row +import org.junit.jupiter.api.Assertions + +class PaimonOptionTest extends PaimonSparkTestBase { + + import testImplicits._ + + test("Paimon Option: create table with sql conf") { + withSparkSQLConf("spark.paimon.scan.snapshot-id" -> "2") { + sql("CREATE TABLE T (id INT)") + val table = loadTable("T") + // check options in schema file directly + val fileStoreTable = FileStoreTableFactory.create(table.fileIO(), table.location()) + Assertions.assertNull(fileStoreTable.options().get("scan.snapshot-id")) + } + } + + test("Paimon Option: create table by dataframe with sql conf") { + withSparkSQLConf("spark.paimon.scan.snapshot-id" -> "2") { + Seq((1L, "x1"), (2L, "x2")) + .toDF("a", "b") + .write + .format("paimon") + .mode("append") + .saveAsTable("T") + val table = loadTable("T") + // check options in schema file directly + val fileStoreTable = FileStoreTableFactory.create(table.fileIO(), table.location()) + Assertions.assertNull(fileStoreTable.options().get("scan.snapshot-id")) + } + } + + test("Paimon Option: query table with sql conf") { + sql("CREATE TABLE T (id INT)") + sql("INSERT INTO T VALUES 1") + sql("INSERT INTO T VALUES 2") + checkAnswer(sql("SELECT * FROM T ORDER BY id"), Row(1) :: Row(2) :: Nil) + val table = loadTable("T") + + // query with mutable option + withSparkSQLConf("spark.paimon.scan.snapshot-id" -> "1") { + checkAnswer(sql("SELECT * FROM T ORDER BY id"), Row(1)) + checkAnswer(spark.read.format("paimon").load(table.location().toString), Row(1)) + } + + // query with immutable option + withSparkSQLConf("spark.paimon.bucket" -> "1") { + assertThrows[UnsupportedOperationException] { + sql("SELECT * FROM T ORDER BY id") + } + assertThrows[UnsupportedOperationException] { + spark.read.format("paimon").load(table.location().toString) + } + } + } + + test("Paimon Table Options: query one table with sql conf and table options") { + sql("CREATE TABLE T (id INT)") + sql("INSERT INTO T VALUES 1") + sql("INSERT INTO T VALUES 2") + checkAnswer(sql("SELECT * FROM T ORDER BY id"), Row(1) :: Row(2) :: Nil) + val table = loadTable("T") + + // query with global options + withSparkSQLConf("spark.paimon.scan.snapshot-id" -> "1") { + checkAnswer(sql("SELECT * FROM T ORDER BY id"), Row(1)) + checkAnswer(spark.read.format("paimon").load(table.location().toString), Row(1)) + } + + // query with table options + withSparkSQLConf("spark.paimon.*.*.T.scan.snapshot-id" -> "1") { + checkAnswer(sql("SELECT * FROM T ORDER BY id"), Row(1)) + checkAnswer(spark.read.format("paimon").load(table.location().toString), Row(1)) + } + + // query with both global and table options + withSparkSQLConf( + "spark.paimon.scan.snapshot-id" -> "1", + "spark.paimon.*.*.T.scan.snapshot-id" -> "2") { + checkAnswer(sql("SELECT * FROM T ORDER BY id"), Row(1) :: Row(2) :: Nil) + checkAnswer( + spark.read.format("paimon").load(table.location().toString), + Row(1) :: Row(2) :: Nil) + } + } + + test("Paimon Table Options: query multiple tables with sql conf and table options") { + sql("CREATE TABLE T1 (id INT)") + sql("INSERT INTO T1 VALUES 1") + sql("INSERT INTO T1 VALUES 2") + + sql("CREATE TABLE T2 (id INT)") + sql("INSERT INTO T2 VALUES 1") + sql("INSERT INTO T2 VALUES 2") + checkAnswer( + sql("SELECT * FROM T1 join T2 on T1.id = T2.id ORDER BY T1.id"), + Row(1, 1) :: Row(2, 2) :: Nil) + val table1 = loadTable("T1") + val table2 = loadTable("T1") + + // query with global options + withSparkSQLConf("spark.paimon.scan.snapshot-id" -> "1") { + checkAnswer(sql("SELECT * FROM T1 join T2 on T1.id = T2.id ORDER BY T1.id"), Row(1, 1)) + checkAnswer( + spark.read + .format("paimon") + .load(table1.location().toString) + .join(spark.read.format("paimon").load(table2.location().toString), "id"), + Row(1) + ) + } + + // query with table options + withSparkSQLConf("spark.paimon.*.*.*.scan.snapshot-id" -> "1") { + checkAnswer(sql("SELECT * FROM T1 join T2 on T1.id = T2.id ORDER BY T1.id"), Row(1, 1)) + checkAnswer( + spark.read + .format("paimon") + .load(table1.location().toString) + .join(spark.read.format("paimon").load(table2.location().toString), "id"), + Row(1) + ) + } + + // query with both global and table options + withSparkSQLConf( + "spark.paimon.scan.snapshot-id" -> "1", + "spark.paimon.*.*.*.scan.snapshot-id" -> "2") { + checkAnswer( + sql("SELECT * FROM T1 join T2 on T1.id = T2.id ORDER BY T1.id"), + Row(1, 1) :: Row(2, 2) :: Nil) + checkAnswer( + spark.read + .format("paimon") + .load(table1.location().toString) + .join(spark.read.format("paimon").load(table2.location().toString), "id"), + Row(1) :: Row(2) :: Nil + ) + } + + withSparkSQLConf( + "spark.paimon.*.*.T1.scan.snapshot-id" -> "1", + "spark.paimon.*.*.T2.scan.snapshot-id" -> "1") { + checkAnswer(sql("SELECT * FROM T1 join T2 on T1.id = T2.id ORDER BY T1.id"), Row(1, 1)) + checkAnswer( + spark.read + .format("paimon") + .load(table1.location().toString) + .join(spark.read.format("paimon").load(table2.location().toString), "id"), + Row(1) + ) + } + + withSparkSQLConf( + "spark.paimon.*.*.T1.scan.snapshot-id" -> "1", + "spark.paimon.*.*.T2.scan.snapshot-id" -> "2") { + checkAnswer(sql("SELECT * FROM T1 join T2 on T1.id = T2.id ORDER BY T1.id"), Row(1, 1)) + checkAnswer( + spark.read + .format("paimon") + .load(table1.location().toString) + .join(spark.read.format("paimon").load(table2.location().toString), "id"), + Row(1) + ) + } + + withSparkSQLConf( + "spark.paimon.*.*.T1.scan.snapshot-id" -> "2", + "spark.paimon.*.*.T2.scan.snapshot-id" -> "2") { + checkAnswer( + sql("SELECT * FROM T1 join T2 on T1.id = T2.id ORDER BY T1.id"), + Row(1, 1) :: Row(2, 2) :: Nil) + checkAnswer( + spark.read + .format("paimon") + .load(table1.location().toString) + .join(spark.read.format("paimon").load(table2.location().toString), "id"), + Row(1) :: Row(2) :: Nil + ) + } + } +} diff --git a/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/PaimonPartitionManagementTest.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/PaimonPartitionManagementTest.scala similarity index 100% rename from paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/PaimonPartitionManagementTest.scala rename to paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/PaimonPartitionManagementTest.scala diff --git a/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/PaimonPushDownTest.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/PaimonPushDownTest.scala similarity index 58% rename from paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/PaimonPushDownTest.scala rename to paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/PaimonPushDownTest.scala index c55ed876d6b1..503f1c8e3e9d 100644 --- a/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/PaimonPushDownTest.scala +++ b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/PaimonPushDownTest.scala @@ -18,7 +18,7 @@ package org.apache.paimon.spark.sql -import org.apache.paimon.spark.{PaimonBatch, PaimonInputPartition, PaimonScan, PaimonSparkTestBase, SparkTable} +import org.apache.paimon.spark.{PaimonScan, PaimonSparkTestBase, SparkTable} import org.apache.paimon.table.source.DataSplit import org.apache.spark.sql.Row @@ -29,8 +29,6 @@ import org.apache.spark.sql.connector.read.{ScanBuilder, SupportsPushDownLimit} import org.apache.spark.sql.util.CaseInsensitiveStringMap import org.junit.jupiter.api.Assertions -import scala.collection.JavaConverters._ - class PaimonPushDownTest extends PaimonSparkTestBase { import testImplicits._ @@ -64,7 +62,7 @@ class PaimonPushDownTest extends PaimonSparkTestBase { checkAnswer(spark.sql(q), Row(1, "a", "p1") :: Nil) // case 2 - // filter "id = '1' or pt = 'p1'" can't push down completely, it still need to be evaluated after scanning + // filter "id = '1' or pt = 'p1'" can't push down completely, it still needs to be evaluated after scanning q = "SELECT * FROM T WHERE id = '1' or pt = 'p1'" Assertions.assertTrue(checkEqualToFilterExists(q, "pt", Literal("p1"))) checkAnswer(spark.sql(q), Row(1, "a", "p1") :: Row(2, "b", "p1") :: Nil) @@ -86,6 +84,22 @@ class PaimonPushDownTest extends PaimonSparkTestBase { checkAnswer(spark.sql(q), Row(1, "a", "p1") :: Row(2, "b", "p1") :: Row(3, "c", "p2") :: Nil) } + test("Paimon pushDown: limit for append-only tables with deletion vector") { + withTable("dv_test") { + spark.sql( + """ + |CREATE TABLE dv_test (c1 INT, c2 STRING) + |TBLPROPERTIES ('deletion-vectors.enabled' = 'true', 'source.split.target-size' = '1') + |""".stripMargin) + + spark.sql("insert into table dv_test values(1, 'a'),(2, 'b'),(3, 'c')") + assert(spark.sql("select * from dv_test limit 2").count() == 2) + + spark.sql("delete from dv_test where c1 = 1") + assert(spark.sql("select * from dv_test limit 2").count() == 2) + } + } + test("Paimon pushDown: limit for append-only tables") { spark.sql(s""" |CREATE TABLE T (a INT, b STRING, c STRING) @@ -105,7 +119,7 @@ class PaimonPushDownTest extends PaimonSparkTestBase { val dataSplitsWithoutLimit = scanBuilder.build().asInstanceOf[PaimonScan].getOriginSplits Assertions.assertTrue(dataSplitsWithoutLimit.length >= 2) - // It still return false even it can push down limit. + // It still returns false even it can push down limit. Assertions.assertFalse(scanBuilder.asInstanceOf[SupportsPushDownLimit].pushLimit(1)) val dataSplitsWithLimit = scanBuilder.build().asInstanceOf[PaimonScan].getOriginSplits Assertions.assertEquals(1, dataSplitsWithLimit.length) @@ -113,7 +127,7 @@ class PaimonPushDownTest extends PaimonSparkTestBase { Assertions.assertEquals(1, spark.sql("SELECT * FROM T LIMIT 1").count()) } - test("Paimon pushDown: limit for change-log tables") { + test("Paimon pushDown: limit for primary key table") { spark.sql(s""" |CREATE TABLE T (a INT, b STRING, c STRING) |TBLPROPERTIES ('primary-key'='a') @@ -125,8 +139,100 @@ class PaimonPushDownTest extends PaimonSparkTestBase { val scanBuilder = getScanBuilder() Assertions.assertTrue(scanBuilder.isInstanceOf[SupportsPushDownLimit]) - // Tables with primary keys can't support the push-down limit. + // Case 1: All dataSplits is rawConvertible. + val dataSplitsWithoutLimit = scanBuilder.build().asInstanceOf[PaimonScan].getOriginSplits + Assertions.assertEquals(4, dataSplitsWithoutLimit.length) + // All dataSplits is rawConvertible. + dataSplitsWithoutLimit.foreach( + splits => { + Assertions.assertTrue(splits.asInstanceOf[DataSplit].rawConvertible()) + }) + + // It still returns false even it can push down limit. Assertions.assertFalse(scanBuilder.asInstanceOf[SupportsPushDownLimit].pushLimit(1)) + val dataSplitsWithLimit = scanBuilder.build().asInstanceOf[PaimonScan].getOriginSplits + Assertions.assertEquals(1, dataSplitsWithLimit.length) + Assertions.assertEquals(1, spark.sql("SELECT * FROM T LIMIT 1").count()) + + Assertions.assertFalse(scanBuilder.asInstanceOf[SupportsPushDownLimit].pushLimit(2)) + val dataSplitsWithLimit1 = scanBuilder.build().asInstanceOf[PaimonScan].getOriginSplits + Assertions.assertEquals(2, dataSplitsWithLimit1.length) + Assertions.assertEquals(2, spark.sql("SELECT * FROM T LIMIT 2").count()) + + // Case 2: Update 2 rawConvertible dataSplits to convert to nonRawConvertible. + spark.sql("INSERT INTO T VALUES (1, 'a2', '11'), (2, 'b2', '22')") + val scanBuilder2 = getScanBuilder() + val dataSplitsWithoutLimit2 = scanBuilder2.build().asInstanceOf[PaimonScan].getOriginSplits + Assertions.assertEquals(4, dataSplitsWithoutLimit2.length) + // Now, we have 4 dataSplits, and 2 dataSplit is nonRawConvertible, 2 dataSplit is rawConvertible. + Assertions.assertEquals( + 2, + dataSplitsWithoutLimit2.count(split => { split.asInstanceOf[DataSplit].rawConvertible() })) + + // Return 2 dataSplits. + Assertions.assertFalse(scanBuilder2.asInstanceOf[SupportsPushDownLimit].pushLimit(2)) + val dataSplitsWithLimit2 = scanBuilder2.build().asInstanceOf[PaimonScan].getOriginSplits + Assertions.assertEquals(2, dataSplitsWithLimit2.length) + Assertions.assertEquals(2, spark.sql("SELECT * FROM T LIMIT 2").count()) + + // 2 dataSplits cannot meet the limit requirement, so need to scan all dataSplits. + Assertions.assertFalse(scanBuilder2.asInstanceOf[SupportsPushDownLimit].pushLimit(3)) + val dataSplitsWithLimit22 = scanBuilder2.build().asInstanceOf[PaimonScan].getOriginSplits + // Need to scan all dataSplits. + Assertions.assertEquals(4, dataSplitsWithLimit22.length) + Assertions.assertEquals(3, spark.sql("SELECT * FROM T LIMIT 3").count()) + + // Case 3: Update the remaining 2 rawConvertible dataSplits to make all dataSplits is nonRawConvertible. + spark.sql("INSERT INTO T VALUES (3, 'c', '11'), (4, 'd', '22')") + val scanBuilder3 = getScanBuilder() + val dataSplitsWithoutLimit3 = scanBuilder3.build().asInstanceOf[PaimonScan].getOriginSplits + Assertions.assertEquals(4, dataSplitsWithoutLimit3.length) + + // All dataSplits is nonRawConvertible. + dataSplitsWithoutLimit3.foreach( + splits => { + Assertions.assertFalse(splits.asInstanceOf[DataSplit].rawConvertible()) + }) + + Assertions.assertFalse(scanBuilder3.asInstanceOf[SupportsPushDownLimit].pushLimit(1)) + val dataSplitsWithLimit3 = scanBuilder3.build().asInstanceOf[PaimonScan].getOriginSplits + // Need to scan all dataSplits. + Assertions.assertEquals(4, dataSplitsWithLimit3.length) + Assertions.assertEquals(1, spark.sql("SELECT * FROM T LIMIT 1").count()) + } + + test("Paimon pushDown: limit for table with deletion vector") { + Seq(true, false).foreach( + deletionVectorsEnabled => { + Seq(true, false).foreach( + primaryKeyTable => { + withTable("T") { + sql(s""" + |CREATE TABLE T (id INT) + |TBLPROPERTIES ( + | 'deletion-vectors.enabled' = $deletionVectorsEnabled, + | '${if (primaryKeyTable) "primary-key" else "bucket-key"}' = 'id', + | 'bucket' = '10' + |) + |""".stripMargin) + + sql("INSERT INTO T SELECT id FROM range (1, 50000)") + sql("DELETE FROM T WHERE id % 13 = 0") + + val withoutLimit = getScanBuilder().build().asInstanceOf[PaimonScan].getOriginSplits + assert(withoutLimit.length == 10) + + val scanBuilder = getScanBuilder().asInstanceOf[SupportsPushDownLimit] + scanBuilder.pushLimit(1) + val withLimit = scanBuilder.build().asInstanceOf[PaimonScan].getOriginSplits + if (deletionVectorsEnabled || !primaryKeyTable) { + assert(withLimit.length == 1) + } else { + assert(withLimit.length == 10) + } + } + }) + }) } test("Paimon pushDown: runtime filter") { @@ -170,8 +276,7 @@ class PaimonPushDownTest extends PaimonSparkTestBase { } private def getScanBuilder(tableName: String = "T"): ScanBuilder = { - new SparkTable(loadTable(tableName)) - .newScanBuilder(CaseInsensitiveStringMap.empty()) + SparkTable(loadTable(tableName)).newScanBuilder(CaseInsensitiveStringMap.empty()) } private def checkFilterExists(sql: String): Boolean = { @@ -192,5 +297,4 @@ class PaimonPushDownTest extends PaimonSparkTestBase { case _ => false } } - } diff --git a/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/PaimonQueryTest.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/PaimonQueryTest.scala similarity index 65% rename from paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/PaimonQueryTest.scala rename to paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/PaimonQueryTest.scala index b08b342ca503..08f5275f01b5 100644 --- a/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/PaimonQueryTest.scala +++ b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/PaimonQueryTest.scala @@ -219,108 +219,117 @@ class PaimonQueryTest extends PaimonSparkTestBase { } test("Paimon Query: query nested cols") { - fileFormats.foreach { - fileFormat => - bucketModes.foreach { - bucketMode => - val bucketProp = if (bucketMode != -1) { - s", 'bucket-key'='name', 'bucket' = '$bucketMode' " - } else { - "" - } - withTable("students") { - sql(s""" - |CREATE TABLE students ( - | name STRING, - | course STRUCT, - | teacher STRUCT>, - | m MAP>, - | l ARRAY>, - | s STRUCT>>>>, - | m2 MAP, STRUCT> - |) USING paimon - |TBLPROPERTIES ('file.format'='$fileFormat' $bucketProp) - |""".stripMargin) - - sql(s""" - |INSERT INTO students VALUES ( - | 'Alice', - | STRUCT('Math', 85.0), - | STRUCT('John', STRUCT('Street 1', 'City 1')), - | MAP('k1', STRUCT('s1', 1, 1.0), 'k2', STRUCT('s11', 11, 11.0)), - | ARRAY(STRUCT('s1', 1, 1.0), STRUCT('s11', 11, 11.0)), - | STRUCT('a', MAP('k1', STRUCT('s1', 1, ARRAY(STRUCT('s1', 1, 1.0))), 'k3', STRUCT('s11', 11, ARRAY(STRUCT('s11', 11, 11.0))))), - | MAP(STRUCT('k1', 1, 1.0), STRUCT('s1', 1, 1.0), STRUCT('k2', 1, 1.0), STRUCT('s11', 11, 11.0))) - |""".stripMargin) - - sql(s""" - |INSERT INTO students VALUES ( - | 'Bob', - | STRUCT('Biology', 92.0), - | STRUCT('Jane', STRUCT('Street 2', 'City 2')), - | MAP('k2', STRUCT('s2', 2, 2.0)), - | ARRAY(STRUCT('s2', 2, 2.0), STRUCT('s22', 22, 22.0)), - | STRUCT('b', MAP('k2', STRUCT('s22', 22, ARRAY(STRUCT('s22', 22, 22.0))))), - | MAP(STRUCT('k2', 2, 2.0), STRUCT('s22', 22, 22.0))) - |""".stripMargin) - - sql(s""" - |INSERT INTO students VALUES ( - | 'Cathy', - | STRUCT('History', 95.0), - | STRUCT('Jane', STRUCT('Street 3', 'City 3')), - | MAP('k1', STRUCT('s3', 3, 3.0), 'k2', STRUCT('s33', 33, 33.0)), - | ARRAY(STRUCT('s3', 3, 3.0)), - | STRUCT('c', MAP('k1', STRUCT('s3', 3, ARRAY(STRUCT('s3', 3, 3.0))), 'k2', STRUCT('s33', 33, ARRAY(STRUCT('s33', 33, 33.0))))), - | MAP(STRUCT('k1', 3, 3.0), STRUCT('s3', 3, 3.0), STRUCT('k2', 3, 3.0), STRUCT('s33', 33, 33.0))) - |""".stripMargin) - - checkAnswer( - sql(s""" - |SELECT - | course.grade, name, teacher.address, course.course_name, - | m['k1'].d, m['k1'].s, - | l[1].d, l[1].s, - | s.s2['k2'].a[0].i, - | map_keys(m2).i - |FROM students ORDER BY name - |""".stripMargin), - Seq( - Row( - 85.0, - "Alice", - Row("Street 1", "City 1"), - "Math", - 1.0, - "s1", - 11.0, - "s11", - null, - Seq(1, 1)), - Row( - 92.0, - "Bob", - Row("Street 2", "City 2"), - "Biology", - null, - null, - 22.0, - "s22", - 22, - Seq(2)), - Row( - 95.0, - "Cathy", - Row("Street 3", "City 3"), - "History", - 3.0, - "s3", - null, - null, - 33, - Seq(3, 3)) - ) - ) + withPk.foreach { + hasPk => + fileFormats.foreach { + fileFormat => + bucketModes.foreach { + bucketMode => + val key = if (hasPk) "primary-key" else "bucket-key" + val props = if (bucketMode != -1) { + s", '$key'='name', 'bucket' = '$bucketMode' " + } else { + "" + } + withTable("students") { + sql(s""" + |CREATE TABLE students ( + | name STRING, + | course STRUCT, + | teacher STRUCT>, + | m MAP>, + | l ARRAY>, + | s STRUCT>>>>, + | m2 MAP, STRUCT> + |) USING paimon + |TBLPROPERTIES ('file.format'='$fileFormat' $props) + |""".stripMargin) + + sql(s""" + |INSERT INTO students VALUES ( + | 'Alice', + | STRUCT('Math', 85.0), + | STRUCT('John', STRUCT('Street 1', 'City 1')), + | MAP('k1', STRUCT('s1', 1, 1.0), 'k2', STRUCT('s11', 11, 11.0)), + | ARRAY(STRUCT('s1', 1, 1.0), STRUCT('s11', 11, 11.0)), + | STRUCT('a', MAP('k1', STRUCT('s1', 1, ARRAY(STRUCT('s1', 1, 1.0))), 'k3', STRUCT('s11', 11, ARRAY(STRUCT('s11', 11, 11.0))))), + | MAP(STRUCT('k1', 1, 1.0), STRUCT('s1', 1, 1.0), STRUCT('k2', 1, 1.0), STRUCT('s11', 11, 11.0))) + |""".stripMargin) + + sql( + s""" + |INSERT INTO students VALUES ( + | 'Bob', + | STRUCT('Biology', 92.0), + | STRUCT('Jane', STRUCT('Street 2', 'City 2')), + | MAP('k2', STRUCT('s2', 2, 2.0)), + | ARRAY(STRUCT('s2', 2, 2.0), STRUCT('s22', 22, 22.0)), + | STRUCT('b', MAP('k2', STRUCT('s22', 22, ARRAY(STRUCT('s22', 22, 22.0))))), + | MAP(STRUCT('k2', 2, 2.0), STRUCT('s22', 22, 22.0))) + |""".stripMargin) + + sql(s""" + |INSERT INTO students VALUES ( + | 'Cathy', + | STRUCT('History', 95.0), + | STRUCT('Jane', STRUCT('Street 3', 'City 3')), + | MAP('k1', STRUCT('s3', 3, 3.0), 'k2', STRUCT('s33', 33, 33.0)), + | ARRAY(STRUCT('s3', 3, 3.0)), + | STRUCT('c', MAP('k1', STRUCT('s3', 3, ARRAY(STRUCT('s3', 3, 3.0))), 'k2', STRUCT('s33', 33, ARRAY(STRUCT('s33', 33, 33.0))))), + | MAP(STRUCT('k1', 3, 3.0), STRUCT('s3', 3, 3.0), STRUCT('k2', 3, 3.0), STRUCT('s33', 33, 33.0))) + |""".stripMargin) + + // Since Spark 4.0, when `spark.sql.ansi.enabled` is `true` and `array[i]` does not exist, an exception + // will be thrown instead of returning null. Here, just disabled it and return null for test. + withSparkSQLConf("spark.sql.ansi.enabled" -> "false") { + checkAnswer( + sql(s""" + |SELECT + | course.grade, name, teacher.address, course.course_name, + | m['k1'].d, m['k1'].s, + | l[1].d, l[1].s, + | s.s2['k2'].a[0].i, + | map_keys(m2).i + |FROM students ORDER BY name + |""".stripMargin), + Seq( + Row( + 85.0, + "Alice", + Row("Street 1", "City 1"), + "Math", + 1.0, + "s1", + 11.0, + "s11", + null, + Seq(1, 1)), + Row( + 92.0, + "Bob", + Row("Street 2", "City 2"), + "Biology", + null, + null, + 22.0, + "s22", + 22, + Seq(2)), + Row( + 95.0, + "Cathy", + Row("Street 3", "City 3"), + "History", + 3.0, + "s3", + null, + null, + 33, + Seq(3, 3)) + ) + ) + } + } } } } diff --git a/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/PaimonShowColumnsTestBase.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/PaimonShowColumnsTestBase.scala similarity index 100% rename from paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/PaimonShowColumnsTestBase.scala rename to paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/PaimonShowColumnsTestBase.scala diff --git a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/PaimonSystemTableTest.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/PaimonSystemTableTest.scala new file mode 100644 index 000000000000..7baa57a54d90 --- /dev/null +++ b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/PaimonSystemTableTest.scala @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.sql + +import org.apache.paimon.spark.PaimonSparkTestBase + +import org.apache.spark.sql.Row + +class PaimonSystemTableTest extends PaimonSparkTestBase { + + test("system table: sort tags table") { + spark.sql(s""" + |CREATE TABLE T (id STRING, name STRING) + |USING PAIMON + |""".stripMargin) + + spark.sql(s"INSERT INTO T VALUES(1, 'a')") + + spark.sql("CALL paimon.sys.create_tag(table => 'test.T', tag => '2024-10-02')") + spark.sql("CALL paimon.sys.create_tag(table => 'test.T', tag => '2024-10-01')") + spark.sql("CALL paimon.sys.create_tag(table => 'test.T', tag => '2024-10-04')") + spark.sql("CALL paimon.sys.create_tag(table => 'test.T', tag => '2024-10-03')") + + checkAnswer( + spark.sql("select tag_name from `T$tags`"), + Row("2024-10-01") :: Row("2024-10-02") :: Row("2024-10-03") :: Row("2024-10-04") :: Nil) + } + + test("system table: sort partitions table") { + spark.sql(s""" + |CREATE TABLE T (a INT, b STRING,dt STRING,hh STRING) + |PARTITIONED BY (dt, hh) + |TBLPROPERTIES ('primary-key'='a,dt,hh', 'bucket' = '3') + |""".stripMargin) + + spark.sql("INSERT INTO T VALUES(1, 'a', '2024-10-10', '01')") + spark.sql("INSERT INTO T VALUES(3, 'c', '2024-10-10', '23')") + spark.sql("INSERT INTO T VALUES(2, 'b', '2024-10-10', '12')") + spark.sql("INSERT INTO T VALUES(5, 'f', '2024-10-09', '02')") + spark.sql("INSERT INTO T VALUES(4, 'd', '2024-10-09', '01')") + + checkAnswer(spark.sql("select count(*) from `T$partitions`"), Row(5) :: Nil) + checkAnswer( + spark.sql("select partition from `T$partitions`"), + Row("[2024-10-09, 01]") :: Row("[2024-10-09, 02]") :: Row("[2024-10-10, 01]") :: Row( + "[2024-10-10, 12]") :: Row("[2024-10-10, 23]") :: Nil + ) + } + + test("system table: sort buckets table") { + spark.sql(s""" + |CREATE TABLE T (a INT, b STRING,dt STRING,hh STRING) + |PARTITIONED BY (dt, hh) + |TBLPROPERTIES ('primary-key'='a,dt,hh', 'bucket' = '3') + |""".stripMargin) + + spark.sql("INSERT INTO T VALUES(1, 'a', '2024-10-10', '01')") + spark.sql("INSERT INTO T VALUES(2, 'b', '2024-10-10', '01')") + spark.sql("INSERT INTO T VALUES(3, 'c', '2024-10-10', '01')") + spark.sql("INSERT INTO T VALUES(4, 'd', '2024-10-10', '01')") + spark.sql("INSERT INTO T VALUES(5, 'f', '2024-10-10', '01')") + + checkAnswer(spark.sql("select count(*) from `T$partitions`"), Row(1) :: Nil) + checkAnswer( + spark.sql("select partition,bucket from `T$buckets`"), + Row("[2024-10-10, 01]", 0) :: Row("[2024-10-10, 01]", 1) :: Row("[2024-10-10, 01]", 2) :: Nil) + } + + test("system table: binlog table") { + sql(""" + |CREATE TABLE T (a INT, b INT) + |TBLPROPERTIES ('primary-key'='a', 'changelog-producer' = 'lookup', 'bucket' = '2') + |""".stripMargin) + + sql("INSERT INTO T VALUES (1, 2)") + sql("INSERT INTO T VALUES (1, 3)") + sql("INSERT INTO T VALUES (2, 2)") + + checkAnswer( + sql("SELECT * FROM `T$binlog`"), + Seq(Row("+I", Array(1), Array(3)), Row("+I", Array(2), Array(2))) + ) + } +} diff --git a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/PaimonTagDdlTestBase.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/PaimonTagDdlTestBase.scala new file mode 100644 index 000000000000..5ad687b4da0f --- /dev/null +++ b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/PaimonTagDdlTestBase.scala @@ -0,0 +1,145 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.sql + +import org.apache.paimon.spark.PaimonSparkTestBase + +import org.apache.spark.sql.Row + +abstract class PaimonTagDdlTestBase extends PaimonSparkTestBase { + test("Tag ddl: show tags syntax") { + spark.sql("""CREATE TABLE T (id INT, name STRING) + |USING PAIMON + |TBLPROPERTIES ('primary-key'='id')""".stripMargin) + + spark.sql("insert into T values(1, 'a')") + + spark.sql("alter table T create tag `2024-10-12`") + spark.sql("alter table T create tag `2024-10-11`") + spark.sql("alter table T create tag `2024-10-13`") + + checkAnswer( + spark.sql("show tags T"), + Row("2024-10-11") :: Row("2024-10-12") :: Row("2024-10-13") :: Nil) + } + + test("Tag ddl: alter table t crete tag syntax") { + spark.sql("""CREATE TABLE T (id INT, name STRING) + |USING PAIMON + |TBLPROPERTIES ('primary-key'='id')""".stripMargin) + + spark.sql("insert into T values(1, 'a')") + spark.sql("insert into T values(2, 'b')") + spark.sql("insert into T values(3, 'c')") + val table = loadTable("T") + assertResult(3)(table.snapshotManager().snapshotCount()) + + spark.sql("alter table T create tag `tag-1`") + spark.sql("alter table T create tag `tag-2` RETAIN 2 DAYS") + spark.sql("alter table T create tag `tag-3` as of version 1") + spark.sql("alter table T create tag `tag-4` as of version 2 RETAIN 3 HOURS") + assertResult(4)(spark.sql("show tags T").count()) + + checkAnswer( + spark.sql("select tag_name,snapshot_id,time_retained from `T$tags`"), + Row("tag-1", 3, null) :: Row("tag-2", 3, "PT48H") :: Row("tag-3", 1, null) :: Row( + "tag-4", + 2, + "PT3H") :: Nil + ) + + // not update tag with 'if not exists' syntax + spark.sql("alter table T create tag if not exists `tag-1` RETAIN 10 HOURS") + checkAnswer( + spark.sql("select tag_name,snapshot_id,time_retained from `T$tags` where tag_name='tag-1'"), + Row("tag-1", 3, null)) + } + + test("Tag ddl: alter table t create or replace tag syntax") { + spark.sql("""CREATE TABLE T (id INT, name STRING) + |USING PAIMON + |TBLPROPERTIES ('primary-key'='id')""".stripMargin) + + spark.sql("insert into T values(1, 'a')") + spark.sql("insert into T values(2, 'b')") + assertResult(2)(loadTable("T").snapshotManager().snapshotCount()) + + // test 'replace' syntax + spark.sql("alter table T create tag `tag-1` as of version 1") + spark.sql("alter table T replace tag `tag-1` as of version 2 RETAIN 1 HOURS") + checkAnswer( + spark.sql("select tag_name,snapshot_id,time_retained from `T$tags`"), + Row("tag-1", 2, "PT1H") :: Nil) + + // test 'create or replace' syntax + // tag-2 not exist, create it + spark.sql("alter table T create or replace tag `tag-2` as of version 1") + checkAnswer( + spark.sql("select tag_name,snapshot_id,time_retained from `T$tags` where tag_name = 'tag-2'"), + Row("tag-2", 1, null) :: Nil) + // tag-2 exists, replace it + spark.sql("alter table T create or replace tag `tag-2` as of version 2 RETAIN 1 HOURS") + checkAnswer( + spark.sql("select tag_name,snapshot_id,time_retained from `T$tags` where tag_name = 'tag-2'"), + Row("tag-2", 2, "PT1H") :: Nil) + } + + test("Tag ddl: alter table t delete tag syntax") { + spark.sql("""CREATE TABLE T (id INT, name STRING) + |USING PAIMON + |TBLPROPERTIES ('primary-key'='id')""".stripMargin) + + spark.sql("insert into T values(1, 'a')") + assertResult(1)(loadTable("T").snapshotManager().snapshotCount()) + + spark.sql("alter table T create tag `2024-10-12`") + spark.sql("alter table T create tag `2024-10-15`") + spark.sql("alter table T create tag `2024-10-13`") + spark.sql("alter table T create tag `2024-10-14`") + checkAnswer( + spark.sql("show tags T"), + Row("2024-10-12") :: Row("2024-10-13") :: Row("2024-10-14") :: Row("2024-10-15") :: Nil) + + spark.sql("alter table T delete tag `2024-10-12`") + checkAnswer( + spark.sql("show tags T"), + Row("2024-10-13") :: Row("2024-10-14") :: Row("2024-10-15") :: Nil) + + spark.sql("alter table T delete tag `2024-10-13, 2024-10-14`") + checkAnswer(spark.sql("show tags T"), Row("2024-10-15") :: Nil) + + spark.sql("alter table T delete tag if EXISTS `2024-10-18`") + checkAnswer(spark.sql("show tags T"), Row("2024-10-15") :: Nil) + } + + test("Tag ddl: alter table t rename tag syntax") { + spark.sql("""CREATE TABLE T (id INT, name STRING) + |USING PAIMON + |TBLPROPERTIES ('primary-key'='id')""".stripMargin) + + spark.sql("insert into T values(1, 'a')") + assertResult(1)(loadTable("T").snapshotManager().snapshotCount()) + + spark.sql("alter table T create tag `tag-1`") + checkAnswer(spark.sql("show tags T"), Row("tag-1")) + + spark.sql("alter table T rename tag `tag-1` to `tag-2`") + checkAnswer(spark.sql("show tags T"), Row("tag-2")) + } +} diff --git a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/PaimonViewTestBase.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/PaimonViewTestBase.scala new file mode 100644 index 000000000000..00f5566ed47a --- /dev/null +++ b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/PaimonViewTestBase.scala @@ -0,0 +1,158 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.sql + +import org.apache.paimon.spark.PaimonHiveTestBase + +import org.apache.spark.sql.Row + +abstract class PaimonViewTestBase extends PaimonHiveTestBase { + + test("Paimon View: create and drop view") { + Seq(sparkCatalogName, paimonHiveCatalogName).foreach { + catalogName => + { + sql(s"USE $catalogName") + withDatabase("test_db") { + sql("CREATE DATABASE test_db") + sql("USE test_db") + withTable("t") { + withView("v1") { + sql("CREATE TABLE t (id INT) USING paimon") + sql("INSERT INTO t VALUES (1), (2)") + + sql("CREATE VIEW v1 AS SELECT * FROM t") + checkAnswer(sql("SHOW VIEWS"), Seq(Row("test_db", "v1", false))) + checkAnswer(sql("SELECT * FROM v1"), Seq(Row(1), Row(2))) + checkAnswer( + sql("SELECT * FROM v1 WHERE id >= (SELECT max(id) FROM v1)"), + Seq(Row(2))) + + // test drop view + sql("DROP VIEW IF EXISTS v1") + checkAnswer(sql("SHOW VIEWS"), Seq()) + sql("CREATE VIEW v1 AS SELECT * FROM t WHERE id > 1") + checkAnswer(sql("SHOW VIEWS"), Seq(Row("test_db", "v1", false))) + checkAnswer(sql("SELECT * FROM v1"), Seq(Row(2))) + + // test create or replace view + intercept[Exception] { + sql("CREATE VIEW v1 AS SELECT * FROM t WHERE id < 2") + } + sql("CREATE OR REPLACE VIEW v1 AS SELECT * FROM t WHERE id < 2") + checkAnswer(sql("SELECT * FROM v1"), Seq(Row(1))) + } + } + } + } + } + } + + test("Paimon View: show views") { + Seq(sparkCatalogName, paimonHiveCatalogName).foreach { + catalogName => + { + sql(s"USE $catalogName") + withDatabase("test_db") { + sql("CREATE DATABASE test_db") + sql("USE test_db") + withTable("t") { + withView("va", "vab", "vc") { + sql("CREATE TABLE t (id INT) USING paimon") + sql("CREATE VIEW va AS SELECT * FROM t") + sql("CREATE VIEW vab AS SELECT * FROM t") + sql("CREATE VIEW vc AS SELECT * FROM t") + checkAnswer( + sql("SHOW VIEWS"), + Seq( + Row("test_db", "va", false), + Row("test_db", "vab", false), + Row("test_db", "vc", false))) + checkAnswer( + sql("SHOW VIEWS LIKE 'va*'"), + Seq(Row("test_db", "va", false), Row("test_db", "vab", false))) + } + } + } + } + } + } + + test("Paimon View: show create view") { + sql(s"USE $paimonHiveCatalogName") + withDatabase("test_db") { + sql("CREATE DATABASE test_db") + sql("USE test_db") + withTable("t") { + withView("v") { + sql("CREATE TABLE t (id INT, c STRING) USING paimon") + sql(""" + |CREATE VIEW v + |COMMENT 'test comment' + |TBLPROPERTIES ('k1' = 'v1') + |AS SELECT * FROM t + |""".stripMargin) + + val s = sql("SHOW CREATE TABLE v").collectAsList().get(0).get(0).toString + val r = """ + |CREATE VIEW test_db.v \( + | id, + | c\) + |COMMENT 'test comment' + |TBLPROPERTIES \( + | 'k1' = 'v1', + | 'transient_lastDdlTime' = '\d+'\) + |AS + |SELECT \* FROM t + |""".stripMargin.replace("\n", "").r + assert(r.findFirstIn(s.replace("\n", "")).isDefined) + } + } + } + } + + test("Paimon View: describe [extended] view") { + sql(s"USE $paimonHiveCatalogName") + withDatabase("test_db") { + sql("CREATE DATABASE test_db") + sql("USE test_db") + withTable("t") { + withView("v") { + sql("CREATE TABLE t (id INT, c STRING) USING paimon") + sql(""" + |CREATE VIEW v + |COMMENT 'test comment' + |TBLPROPERTIES ('k1' = 'v1') + |AS SELECT * FROM t + |""".stripMargin) + + checkAnswer(sql("DESC TABLE v"), Seq(Row("id", "INT", null), Row("c", "STRING", null))) + + val rows = sql("DESC TABLE EXTENDED v").collectAsList() + assert(rows.get(3).toString().equals("[# Detailed View Information,,]")) + assert(rows.get(4).toString().equals("[Name,test_db.v,]")) + assert(rows.get(5).toString().equals("[Comment,test comment,]")) + assert(rows.get(6).toString().equals("[View Text,SELECT * FROM t,]")) + assert(rows.get(7).toString().equals("[View Query Output Columns,[id, c],]")) + assert(rows.get(8).toString().contains("[View Properties,[k1=v1")) + } + } + } + } +} diff --git a/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/PushDownAggregatesTest.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/PushDownAggregatesTest.scala similarity index 70% rename from paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/PushDownAggregatesTest.scala rename to paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/PushDownAggregatesTest.scala index 667b64e28f61..78c02644a7ce 100644 --- a/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/PushDownAggregatesTest.scala +++ b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/PushDownAggregatesTest.scala @@ -43,6 +43,8 @@ class PushDownAggregatesTest extends PaimonSparkTestBase with AdaptiveSparkPlanH if (numAggregates == 0) { assert(collect(df.queryExecution.executedPlan) { case scan: LocalTableScanExec => scan + // For compatibility with Spark3.x + case e if e.getClass.getName == "org.apache.spark.sql.execution.EmptyRelationExec" => e }.size == 1) } } @@ -115,22 +117,58 @@ class PushDownAggregatesTest extends PaimonSparkTestBase with AdaptiveSparkPlanH } } - test("Push down aggregate - primary table") { - withTable("T") { - spark.sql("CREATE TABLE T (c1 INT, c2 STRING) TBLPROPERTIES ('primary-key' = 'c1')") - runAndCheckAggregate("SELECT COUNT(*) FROM T", Row(0) :: Nil, 2) - spark.sql("INSERT INTO T VALUES(1, 'x'), (2, 'x'), (3, 'x'), (3, 'x')") - runAndCheckAggregate("SELECT COUNT(*) FROM T", Row(3) :: Nil, 2) - } + test("Push down aggregate - primary key table with deletion vector") { + Seq(true, false).foreach( + deletionVectorsEnabled => { + withTable("T") { + spark.sql(s""" + |CREATE TABLE T (c1 INT, c2 STRING) + |TBLPROPERTIES ( + |'primary-key' = 'c1', + |'deletion-vectors.enabled' = $deletionVectorsEnabled + |) + |""".stripMargin) + runAndCheckAggregate("SELECT COUNT(*) FROM T", Row(0) :: Nil, 0) + + spark.sql("INSERT INTO T VALUES(1, 'x'), (2, 'x'), (3, 'x'), (3, 'x')") + runAndCheckAggregate("SELECT COUNT(*) FROM T", Row(3) :: Nil, 0) + + spark.sql("INSERT INTO T VALUES(1, 'x_1')") + if (deletionVectorsEnabled) { + runAndCheckAggregate("SELECT COUNT(*) FROM T", Row(3) :: Nil, 0) + } else { + runAndCheckAggregate("SELECT COUNT(*) FROM T", Row(3) :: Nil, 2) + } + } + }) } - test("Push down aggregate - enable deletion vector") { - withTable("T") { - spark.sql( - "CREATE TABLE T (c1 INT, c2 STRING) TBLPROPERTIES('deletion-vectors.enabled' = 'true')") - runAndCheckAggregate("SELECT COUNT(*) FROM T", Row(0) :: Nil, 2) - spark.sql("INSERT INTO T VALUES(1, 'x'), (2, 'x'), (3, 'x'), (3, 'x')") - runAndCheckAggregate("SELECT COUNT(*) FROM T", Row(4) :: Nil, 2) - } + test("Push down aggregate - table with deletion vector") { + Seq(true, false).foreach( + deletionVectorsEnabled => { + Seq(true, false).foreach( + primaryKeyTable => { + withTable("T") { + sql(s""" + |CREATE TABLE T (id INT) + |TBLPROPERTIES ( + | 'deletion-vectors.enabled' = $deletionVectorsEnabled, + | '${if (primaryKeyTable) "primary-key" else "bucket-key"}' = 'id', + | 'bucket' = '1' + |) + |""".stripMargin) + + sql("INSERT INTO T SELECT id FROM range (0, 5000)") + runAndCheckAggregate("SELECT COUNT(*) FROM T", Seq(Row(5000)), 0) + + sql("DELETE FROM T WHERE id > 100 and id <= 400") + if (deletionVectorsEnabled || !primaryKeyTable) { + runAndCheckAggregate("SELECT COUNT(*) FROM T", Seq(Row(4700)), 0) + } else { + runAndCheckAggregate("SELECT COUNT(*) FROM T", Seq(Row(4700)), 2) + } + } + }) + }) } } diff --git a/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/SparkVersionSupport.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/SparkVersionSupport.scala similarity index 95% rename from paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/SparkVersionSupport.scala rename to paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/SparkVersionSupport.scala index fed73ba0f9e2..647b4cfdcab7 100644 --- a/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/SparkVersionSupport.scala +++ b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/SparkVersionSupport.scala @@ -28,4 +28,6 @@ trait SparkVersionSupport { lazy val gteqSpark3_4: Boolean = sparkVersion >= "3.4" lazy val gteqSpark3_5: Boolean = sparkVersion >= "3.5" + + lazy val gteqSpark4_0: Boolean = sparkVersion >= "4.0" } diff --git a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/TableValuedFunctionsTest.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/TableValuedFunctionsTest.scala new file mode 100644 index 000000000000..b9c187b83a25 --- /dev/null +++ b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/TableValuedFunctionsTest.scala @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.sql + +import org.apache.paimon.spark.PaimonHiveTestBase + +import org.apache.spark.sql.{DataFrame, Row} + +class TableValuedFunctionsTest extends PaimonHiveTestBase { + + withPk.foreach { + hasPk => + bucketModes.foreach { + bucket => + test(s"incremental query: hasPk: $hasPk, bucket: $bucket") { + Seq("paimon", sparkCatalogName, paimonHiveCatalogName).foreach { + catalogName => + sql(s"use $catalogName") + + withTable("t") { + val prop = if (hasPk) { + s"'primary-key'='a,b', 'bucket' = '$bucket' " + } else if (bucket != -1) { + s"'bucket-key'='b', 'bucket' = '$bucket' " + } else { + "'write-only'='true'" + } + + spark.sql(s""" + |CREATE TABLE t (a INT, b INT, c STRING) + |USING paimon + |TBLPROPERTIES ($prop) + |PARTITIONED BY (a) + |""".stripMargin) + + spark.sql("INSERT INTO t values (1, 1, '1'), (2, 2, '2')") + spark.sql("INSERT INTO t VALUES (1, 3, '3'), (2, 4, '4')") + spark.sql("INSERT INTO t VALUES (1, 5, '5'), (1, 7, '7')") + + checkAnswer( + incrementalDF("t", 0, 1).orderBy("a", "b"), + Row(1, 1, "1") :: Row(2, 2, "2") :: Nil) + checkAnswer( + spark.sql( + "SELECT * FROM paimon_incremental_query('t', '0', '1') ORDER BY a, b"), + Row(1, 1, "1") :: Row(2, 2, "2") :: Nil) + + checkAnswer( + incrementalDF("t", 1, 2).orderBy("a", "b"), + Row(1, 3, "3") :: Row(2, 4, "4") :: Nil) + checkAnswer( + spark.sql( + "SELECT * FROM paimon_incremental_query('t', '1', '2') ORDER BY a, b"), + Row(1, 3, "3") :: Row(2, 4, "4") :: Nil) + + checkAnswer( + incrementalDF("t", 2, 3).orderBy("a", "b"), + Row(1, 5, "5") :: Row(1, 7, "7") :: Nil) + checkAnswer( + spark.sql( + "SELECT * FROM paimon_incremental_query('t', '2', '3') ORDER BY a, b"), + Row(1, 5, "5") :: Row(1, 7, "7") :: Nil) + + checkAnswer( + incrementalDF("t", 1, 3).orderBy("a", "b"), + Row(1, 3, "3") :: Row(1, 5, "5") :: Row(1, 7, "7") :: Row(2, 4, "4") :: Nil + ) + checkAnswer( + spark.sql( + "SELECT * FROM paimon_incremental_query('t', '1', '3') ORDER BY a, b"), + Row(1, 3, "3") :: Row(1, 5, "5") :: Row(1, 7, "7") :: Row(2, 4, "4") :: Nil) + } + } + } + } + } + + private def incrementalDF(tableIdent: String, start: Int, end: Int): DataFrame = { + spark.read + .format("paimon") + .option("incremental-between", s"$start,$end") + .table(tableIdent) + } +} diff --git a/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/UpdateTableTestBase.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/UpdateTableTestBase.scala similarity index 99% rename from paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/UpdateTableTestBase.scala rename to paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/UpdateTableTestBase.scala index d7222a1970a2..5beaea59548f 100644 --- a/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/UpdateTableTestBase.scala +++ b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/UpdateTableTestBase.scala @@ -328,7 +328,6 @@ abstract class UpdateTableTestBase extends PaimonSparkTestBase { "INSERT INTO T VALUES (1, map(1, 'a'), '11'), (2, map(2, 'b'), '22'), (3, map(3, 'c'), '33')") assertThatThrownBy(() => spark.sql("UPDATE T SET m.key = 11 WHERE id = 1")) - .hasMessageContaining("Unsupported update expression") spark.sql("UPDATE T SET m = map(11, 'a_new') WHERE id = 1") val rows = spark.sql("SELECT * FROM T ORDER BY id").collectAsList() diff --git a/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/WithTableOptions.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/WithTableOptions.scala similarity index 100% rename from paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/WithTableOptions.scala rename to paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/WithTableOptions.scala diff --git a/paimon-spark/paimon-spark-common/src/test/scala/org/apache/spark/paimon/Utils.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/spark/sql/paimon/Utils.scala similarity index 74% rename from paimon-spark/paimon-spark-common/src/test/scala/org/apache/spark/paimon/Utils.scala rename to paimon-spark/paimon-spark-ut/src/test/scala/org/apache/spark/sql/paimon/Utils.scala index 5ea2dd861e19..03f1c7706efb 100644 --- a/paimon-spark/paimon-spark-common/src/test/scala/org/apache/spark/paimon/Utils.scala +++ b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/spark/sql/paimon/Utils.scala @@ -16,9 +16,10 @@ * limitations under the License. */ -package org.apache.spark.paimon +package org.apache.spark.sql.paimon -import org.apache.spark.sql.SparkSession +import org.apache.spark.sql.{DataFrame, Dataset, SparkSession} +import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan import org.apache.spark.util.{Utils => SparkUtils} import java.io.File @@ -28,9 +29,14 @@ import java.io.File */ object Utils { - def createTempDir: File = SparkUtils.createTempDir() + def createTempDir: File = SparkUtils.createTempDir(System.getProperty("java.io.tmpdir"), "spark") def waitUntilEventEmpty(spark: SparkSession): Unit = { spark.sparkContext.listenerBus.waitUntilEmpty() } + + def createDataFrame(sparkSession: SparkSession, plan: LogicalPlan): DataFrame = { + Dataset.ofRows(sparkSession, plan) + } + } diff --git a/paimon-spark/paimon-spark3-common/pom.xml b/paimon-spark/paimon-spark3-common/pom.xml new file mode 100644 index 000000000000..5fd869f1b393 --- /dev/null +++ b/paimon-spark/paimon-spark3-common/pom.xml @@ -0,0 +1,73 @@ + + + + 4.0.0 + + + org.apache.paimon + paimon-spark + 1.0-SNAPSHOT + + + jar + + paimon-spark3-common + Paimon : Spark3 : Common + + + ${paimon-spark-common.spark.version} + + + + + org.apache.paimon + paimon-spark-common_${scala.binary.version} + ${project.version} + + + + + + + org.apache.maven.plugins + maven-shade-plugin + + + shade-paimon + package + + shade + + + + + org.apache.paimon:paimon-bundle + org.apache.paimon:paimon-spark-common_${scala.binary.version} + + + + + + + + + \ No newline at end of file diff --git a/paimon-spark/paimon-spark3-common/src/main/resources/META-INF/services/org.apache.spark.sql.paimon.shims.SparkShim b/paimon-spark/paimon-spark3-common/src/main/resources/META-INF/services/org.apache.spark.sql.paimon.shims.SparkShim new file mode 100644 index 000000000000..b79ef54f6e30 --- /dev/null +++ b/paimon-spark/paimon-spark3-common/src/main/resources/META-INF/services/org.apache.spark.sql.paimon.shims.SparkShim @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +org.apache.spark.sql.paimon.shims.Spark3Shim \ No newline at end of file diff --git a/paimon-spark/paimon-spark3-common/src/main/scala/org/apache/paimon/spark/catalyst/analysis/Spark3ResolutionRules.scala b/paimon-spark/paimon-spark3-common/src/main/scala/org/apache/paimon/spark/catalyst/analysis/Spark3ResolutionRules.scala new file mode 100644 index 000000000000..924df2d1e320 --- /dev/null +++ b/paimon-spark/paimon-spark3-common/src/main/scala/org/apache/paimon/spark/catalyst/analysis/Spark3ResolutionRules.scala @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.catalyst.analysis + +import org.apache.paimon.spark.commands.{PaimonShowTablePartitionCommand, PaimonShowTablesExtendedCommand} + +import org.apache.spark.sql.SparkSession +import org.apache.spark.sql.catalyst.SQLConfHelper +import org.apache.spark.sql.catalyst.analysis.{PartitionSpec, ResolvedNamespace, UnresolvedPartitionSpec} +import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, ShowTableExtended} +import org.apache.spark.sql.catalyst.rules.Rule +import org.apache.spark.sql.connector.catalog.Identifier + +case class Spark3ResolutionRules(session: SparkSession) + extends Rule[LogicalPlan] + with SQLConfHelper { + + import org.apache.spark.sql.connector.catalog.PaimonCatalogImplicits._ + + override def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperatorsDown { + case ShowTableExtended( + ResolvedNamespace(catalog, ns), + pattern, + partitionSpec @ (None | Some(UnresolvedPartitionSpec(_, _))), + output) => + partitionSpec + .map { + spec: PartitionSpec => + val table = Identifier.of(ns.toArray, pattern) + val resolvedSpec = + PaimonResolvePartitionSpec.resolve(catalog.asTableCatalog, table, spec) + PaimonShowTablePartitionCommand(output, catalog.asTableCatalog, table, resolvedSpec) + } + .getOrElse { + PaimonShowTablesExtendedCommand(catalog.asTableCatalog, ns, pattern, output) + } + + } + +} diff --git a/paimon-spark/paimon-spark3-common/src/main/scala/org/apache/paimon/spark/catalyst/parser/extensions/PaimonSpark3SqlExtensionsParser.scala b/paimon-spark/paimon-spark3-common/src/main/scala/org/apache/paimon/spark/catalyst/parser/extensions/PaimonSpark3SqlExtensionsParser.scala new file mode 100644 index 000000000000..07481b6f639f --- /dev/null +++ b/paimon-spark/paimon-spark3-common/src/main/scala/org/apache/paimon/spark/catalyst/parser/extensions/PaimonSpark3SqlExtensionsParser.scala @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.catalyst.parser.extensions + +import org.apache.spark.sql.catalyst.parser.ParserInterface +import org.apache.spark.sql.catalyst.parser.extensions.AbstractPaimonSparkSqlExtensionsParser + +class PaimonSpark3SqlExtensionsParser(override val delegate: ParserInterface) + extends AbstractPaimonSparkSqlExtensionsParser(delegate) {} diff --git a/paimon-spark/paimon-spark3-common/src/main/scala/org/apache/paimon/spark/commands/PaimonShowTablePartitionCommand.scala b/paimon-spark/paimon-spark3-common/src/main/scala/org/apache/paimon/spark/commands/PaimonShowTablePartitionCommand.scala new file mode 100644 index 000000000000..32f94985859c --- /dev/null +++ b/paimon-spark/paimon-spark3-common/src/main/scala/org/apache/paimon/spark/commands/PaimonShowTablePartitionCommand.scala @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.commands + +import org.apache.paimon.spark.leafnode.PaimonLeafRunnableCommand + +import org.apache.spark.sql.{Row, SparkSession} +import org.apache.spark.sql.catalyst.analysis.ResolvedPartitionSpec +import org.apache.spark.sql.catalyst.catalog.ExternalCatalogUtils.escapePathName +import org.apache.spark.sql.catalyst.expressions.{Attribute, ToPrettyString} +import org.apache.spark.sql.catalyst.expressions.Literal +import org.apache.spark.sql.connector.catalog.{Identifier, SupportsPartitionManagement, TableCatalog} +import org.apache.spark.sql.connector.catalog.PaimonCatalogImplicits._ +import org.apache.spark.sql.execution.datasources.v2.DataSourceV2Implicits._ + +import scala.collection.JavaConverters._ +import scala.collection.mutable + +case class PaimonShowTablePartitionCommand( + override val output: Seq[Attribute], + catalog: TableCatalog, + tableIndent: Identifier, + partSpec: ResolvedPartitionSpec) + extends PaimonLeafRunnableCommand { + override def run(sparkSession: SparkSession): Seq[Row] = { + val rows = new mutable.ArrayBuffer[Row]() + val table = catalog.loadTable(tableIndent) + val information = getTablePartitionDetails(tableIndent, table.asPartitionable, partSpec) + rows += Row(tableIndent.namespace.quoted, tableIndent.name(), false, s"$information\n") + + rows.toSeq + } + + private def getTablePartitionDetails( + tableIdent: Identifier, + partitionTable: SupportsPartitionManagement, + partSpec: ResolvedPartitionSpec): String = { + val results = new mutable.LinkedHashMap[String, String]() + + // "Partition Values" + val partitionSchema = partitionTable.partitionSchema() + val (names, ident) = (partSpec.names, partSpec.ident) + val partitionIdentifiers = partitionTable.listPartitionIdentifiers(names.toArray, ident) + if (partitionIdentifiers.isEmpty) { + val part = ident + .toSeq(partitionSchema) + .zip(partitionSchema.map(_.name)) + .map(kv => s"${kv._2}" + s" = ${kv._1}") + .mkString(", ") + throw new RuntimeException( + s""" + |[PARTITIONS_NOT_FOUND] The partition(s) PARTITION ($part) cannot be found in table ${tableIdent.toString}. + |Verify the partition specification and table name. + |""".stripMargin) + } + assert(partitionIdentifiers.length == 1) + val row = partitionIdentifiers.head + val len = partitionSchema.length + val partitions = new Array[String](len) + val timeZoneId = conf.sessionLocalTimeZone + for (i <- 0 until len) { + val dataType = partitionSchema(i).dataType + val partValueUTF8String = + ToPrettyString(Literal(row.get(i, dataType), dataType), Some(timeZoneId)).eval(null) + val partValueStr = if (partValueUTF8String == null) "null" else partValueUTF8String.toString + partitions(i) = escapePathName(partitionSchema(i).name) + "=" + escapePathName(partValueStr) + } + val partitionValues = partitions.mkString("[", ", ", "]") + results.put("Partition Values", s"$partitionValues") + + // TODO "Partition Parameters", "Created Time", "Last Access", "Partition Statistics" + + results + .map { + case (key, value) => + if (value.isEmpty) key else s"$key: $value" + } + .mkString("", "\n", "") + } +} diff --git a/paimon-spark/paimon-spark3-common/src/main/scala/org/apache/paimon/spark/commands/PaimonShowTablesExtendedCommand.scala b/paimon-spark/paimon-spark3-common/src/main/scala/org/apache/paimon/spark/commands/PaimonShowTablesExtendedCommand.scala new file mode 100644 index 000000000000..b393982e25d3 --- /dev/null +++ b/paimon-spark/paimon-spark3-common/src/main/scala/org/apache/paimon/spark/commands/PaimonShowTablesExtendedCommand.scala @@ -0,0 +1,123 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.commands + +import org.apache.paimon.spark.leafnode.PaimonLeafRunnableCommand + +import org.apache.spark.sql.{Row, SparkSession} +import org.apache.spark.sql.catalyst.catalog.CatalogTableType +import org.apache.spark.sql.catalyst.catalog.CatalogTypes.TablePartitionSpec +import org.apache.spark.sql.catalyst.expressions.Attribute +import org.apache.spark.sql.catalyst.util.{QuotingUtils, StringUtils} +import org.apache.spark.sql.connector.catalog.{Identifier, PaimonCatalogUtils, SupportsPartitionManagement, Table, TableCatalog} +import org.apache.spark.sql.connector.catalog.PaimonCatalogImplicits._ +import org.apache.spark.sql.execution.datasources.v2.DataSourceV2Implicits._ + +import scala.collection.JavaConverters._ +import scala.collection.mutable + +case class PaimonShowTablesExtendedCommand( + catalog: TableCatalog, + namespace: Seq[String], + pattern: String, + override val output: Seq[Attribute], + isExtended: Boolean = false, + partitionSpec: Option[TablePartitionSpec] = None) + extends PaimonLeafRunnableCommand { + + override def run(spark: SparkSession): Seq[Row] = { + val rows = new mutable.ArrayBuffer[Row]() + + val tables = catalog.listTables(namespace.toArray) + tables.map { + tableIdent: Identifier => + if (StringUtils.filterPattern(Seq(tableIdent.name()), pattern).nonEmpty) { + val table = catalog.loadTable(tableIdent) + val information = getTableDetails(catalog.name, tableIdent, table) + rows += Row(tableIdent.namespace().quoted, tableIdent.name(), false, s"$information\n") + } + } + + // TODO: view + + rows.toSeq + } + + private def getTableDetails(catalogName: String, identifier: Identifier, table: Table): String = { + val results = new mutable.LinkedHashMap[String, String]() + + results.put("Catalog", catalogName) + results.put("Namespace", identifier.namespace().quoted) + results.put("Table", identifier.name()) + val tableType = if (table.properties().containsKey(TableCatalog.PROP_EXTERNAL)) { + CatalogTableType.EXTERNAL + } else { + CatalogTableType.MANAGED + } + results.put("Type", tableType.name) + + PaimonCatalogUtils.TABLE_RESERVED_PROPERTIES + .filterNot(_ == TableCatalog.PROP_EXTERNAL) + .foreach( + propKey => { + if (table.properties.containsKey(propKey)) { + results.put(propKey.capitalize, table.properties.get(propKey)) + } + }) + + val properties: Seq[String] = + conf + .redactOptions(table.properties.asScala.toMap) + .toList + .filter(kv => !PaimonCatalogUtils.TABLE_RESERVED_PROPERTIES.contains(kv._1)) + .sortBy(_._1) + .map { case (key, value) => key + "=" + value } + if (!table.properties().isEmpty) { + results.put("Table Properties", properties.mkString("[", ", ", "]")) + } + + // Partition Provider & Partition Columns + if (supportsPartitions(table) && table.asPartitionable.partitionSchema().nonEmpty) { + results.put("Partition Provider", "Catalog") + results.put( + "Partition Columns", + table.asPartitionable + .partitionSchema() + .map(field => QuotingUtils.quoteIdentifier(field.name)) + .mkString("[", ", ", "]")) + } + + if (table.schema().nonEmpty) { + results.put("Schema", table.schema().treeString) + } + + results + .map { + case (key, value) => + if (value.isEmpty) key else s"$key: $value" + } + .mkString("", "\n", "") + } + + private def supportsPartitions(table: Table): Boolean = table match { + case _: SupportsPartitionManagement => true + case _ => false + } + +} diff --git a/paimon-spark/paimon-spark3-common/src/main/scala/org/apache/paimon/spark/data/Spark3ArrayData.scala b/paimon-spark/paimon-spark3-common/src/main/scala/org/apache/paimon/spark/data/Spark3ArrayData.scala new file mode 100644 index 000000000000..cb393d928dcb --- /dev/null +++ b/paimon-spark/paimon-spark3-common/src/main/scala/org/apache/paimon/spark/data/Spark3ArrayData.scala @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.data + +import org.apache.paimon.types.DataType + +class Spark3ArrayData(override val elementType: DataType) extends AbstractSparkArrayData {} diff --git a/paimon-spark/paimon-spark3-common/src/main/scala/org/apache/paimon/spark/data/Spark3InternalRow.scala b/paimon-spark/paimon-spark3-common/src/main/scala/org/apache/paimon/spark/data/Spark3InternalRow.scala new file mode 100644 index 000000000000..9c9a1c6bac95 --- /dev/null +++ b/paimon-spark/paimon-spark3-common/src/main/scala/org/apache/paimon/spark/data/Spark3InternalRow.scala @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.data + +import org.apache.paimon.spark.AbstractSparkInternalRow +import org.apache.paimon.types.RowType + +class Spark3InternalRow(rowType: RowType) extends AbstractSparkInternalRow(rowType) {} diff --git a/paimon-spark/paimon-spark3-common/src/main/scala/org/apache/spark/sql/paimon/shims/Spark3Shim.scala b/paimon-spark/paimon-spark3-common/src/main/scala/org/apache/spark/sql/paimon/shims/Spark3Shim.scala new file mode 100644 index 000000000000..f508e2605cbc --- /dev/null +++ b/paimon-spark/paimon-spark3-common/src/main/scala/org/apache/spark/sql/paimon/shims/Spark3Shim.scala @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.paimon.shims + +import org.apache.paimon.spark.catalyst.analysis.Spark3ResolutionRules +import org.apache.paimon.spark.catalyst.parser.extensions.PaimonSpark3SqlExtensionsParser +import org.apache.paimon.spark.data.{Spark3ArrayData, Spark3InternalRow, SparkArrayData, SparkInternalRow} +import org.apache.paimon.types.{DataType, RowType} + +import org.apache.spark.sql.{Column, SparkSession} +import org.apache.spark.sql.catalyst.expressions.{Attribute, Expression} +import org.apache.spark.sql.catalyst.parser.ParserInterface +import org.apache.spark.sql.catalyst.plans.logical.{Aggregate, LogicalPlan} +import org.apache.spark.sql.catalyst.rules.Rule +import org.apache.spark.sql.connector.catalog.{Identifier, Table, TableCatalog} +import org.apache.spark.sql.connector.expressions.Transform +import org.apache.spark.sql.types.StructType + +import java.util.{Map => JMap} + +class Spark3Shim extends SparkShim { + + override def createSparkParser(delegate: ParserInterface): ParserInterface = { + new PaimonSpark3SqlExtensionsParser(delegate) + } + + override def createCustomResolution(spark: SparkSession): Rule[LogicalPlan] = { + Spark3ResolutionRules(spark) + } + + override def createSparkInternalRow(rowType: RowType): SparkInternalRow = { + new Spark3InternalRow(rowType) + } + + override def createSparkArrayData(elementType: DataType): SparkArrayData = { + new Spark3ArrayData(elementType) + } + + override def supportsHashAggregate( + aggregateBufferAttributes: Seq[Attribute], + groupingExpression: Seq[Expression]): Boolean = { + Aggregate.supportsHashAggregate(aggregateBufferAttributes) + } + + override def createTable( + tableCatalog: TableCatalog, + ident: Identifier, + schema: StructType, + partitions: Array[Transform], + properties: JMap[String, String]): Table = { + tableCatalog.createTable(ident, schema, partitions, properties) + } + + override def column(expr: Expression): Column = new Column(expr) + + override def convertToExpression(spark: SparkSession, column: Column): Expression = column.expr + +} diff --git a/paimon-spark/paimon-spark4-common/pom.xml b/paimon-spark/paimon-spark4-common/pom.xml new file mode 100644 index 000000000000..d160b984fa05 --- /dev/null +++ b/paimon-spark/paimon-spark4-common/pom.xml @@ -0,0 +1,94 @@ + + + + 4.0.0 + + + org.apache.paimon + paimon-spark + 1.0-SNAPSHOT + + + jar + + paimon-spark4-common + Paimon : Spark4 : Common + + + ${paimon-spark-common.spark.version} + + + + + org.apache.paimon + paimon-spark-common_${scala.binary.version} + ${project.version} + + + + org.apache.spark + spark-sql-api_2.13 + ${spark.version} + + + log4j + log4j + + + org.slf4j + slf4j-log4j12 + + + org.apache.logging.log4j + log4j-slf4j2-impl + + + + + + + + + + org.apache.maven.plugins + maven-shade-plugin + + + shade-paimon + package + + shade + + + + + org.apache.paimon:paimon-bundle + org.apache.paimon:paimon-spark-common_${scala.binary.version} + + + + + + + + + diff --git a/paimon-spark/paimon-spark4-common/src/main/resources/META-INF/services/org.apache.spark.sql.paimon.shims.SparkShim b/paimon-spark/paimon-spark4-common/src/main/resources/META-INF/services/org.apache.spark.sql.paimon.shims.SparkShim new file mode 100644 index 000000000000..b0df8c67cf9a --- /dev/null +++ b/paimon-spark/paimon-spark4-common/src/main/resources/META-INF/services/org.apache.spark.sql.paimon.shims.SparkShim @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +org.apache.spark.sql.paimon.shims.Spark4Shim \ No newline at end of file diff --git a/paimon-spark/paimon-spark4-common/src/main/scala/org/apache/paimon/spark/catalyst/analysis/Spark4ResolutionRules.scala b/paimon-spark/paimon-spark4-common/src/main/scala/org/apache/paimon/spark/catalyst/analysis/Spark4ResolutionRules.scala new file mode 100644 index 000000000000..461cbd0c938a --- /dev/null +++ b/paimon-spark/paimon-spark4-common/src/main/scala/org/apache/paimon/spark/catalyst/analysis/Spark4ResolutionRules.scala @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.catalyst.analysis + +import org.apache.spark.sql.SparkSession +import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan +import org.apache.spark.sql.catalyst.rules.Rule + +case class Spark4ResolutionRules(session: SparkSession) extends Rule[LogicalPlan] { + override def apply(plan: LogicalPlan): LogicalPlan = plan +} diff --git a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/flink/table/catalog/CatalogMaterializedTable.java b/paimon-spark/paimon-spark4-common/src/main/scala/org/apache/paimon/spark/catalyst/parser/extensions/PaimonSpark4SqlExtensionsParser.scala similarity index 63% rename from paimon-flink/paimon-flink-cdc/src/main/java/org/apache/flink/table/catalog/CatalogMaterializedTable.java rename to paimon-spark/paimon-spark4-common/src/main/scala/org/apache/paimon/spark/catalyst/parser/extensions/PaimonSpark4SqlExtensionsParser.scala index 6eabd1db7f38..ef1f5763d27b 100644 --- a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/flink/table/catalog/CatalogMaterializedTable.java +++ b/paimon-spark/paimon-spark4-common/src/main/scala/org/apache/paimon/spark/catalyst/parser/extensions/PaimonSpark4SqlExtensionsParser.scala @@ -16,19 +16,13 @@ * limitations under the License. */ -package org.apache.flink.table.catalog; +package org.apache.paimon.spark.catalyst.parser.extensions -/** - * Dummy placeholder to resolve compatibility issue of CatalogMaterializedTable(introduced in flink - * 1.20). - */ -public interface CatalogMaterializedTable extends CatalogBaseTable { - /** Dummy LogicalRefreshMode placeholder. */ - enum LogicalRefreshMode {} +import org.apache.spark.sql.catalyst.parser.{CompoundBody, ParserInterface} +import org.apache.spark.sql.catalyst.parser.extensions.AbstractPaimonSparkSqlExtensionsParser - /** Dummy RefreshMode placeholder. */ - enum RefreshMode {} +class PaimonSpark4SqlExtensionsParser(override val delegate: ParserInterface) + extends AbstractPaimonSparkSqlExtensionsParser(delegate) { - /** Dummy RefreshStatus placeholder. */ - enum RefreshStatus {} + def parseScript(sqlScriptText: String): CompoundBody = delegate.parseScript(sqlScriptText) } diff --git a/paimon-common/src/main/java/org/apache/paimon/lineage/TableLineageEntity.java b/paimon-spark/paimon-spark4-common/src/main/scala/org/apache/paimon/spark/data/Spark4ArrayData.scala similarity index 72% rename from paimon-common/src/main/java/org/apache/paimon/lineage/TableLineageEntity.java rename to paimon-spark/paimon-spark4-common/src/main/scala/org/apache/paimon/spark/data/Spark4ArrayData.scala index c4312c4eb080..be319c0a9c23 100644 --- a/paimon-common/src/main/java/org/apache/paimon/lineage/TableLineageEntity.java +++ b/paimon-spark/paimon-spark4-common/src/main/scala/org/apache/paimon/spark/data/Spark4ArrayData.scala @@ -16,17 +16,14 @@ * limitations under the License. */ -package org.apache.paimon.lineage; +package org.apache.paimon.spark.data -import org.apache.paimon.data.Timestamp; +import org.apache.paimon.types.DataType -/** Table lineage entity with database, table and job for table source and sink lineage. */ -public interface TableLineageEntity { - String getDatabase(); +import org.apache.spark.unsafe.types.VariantVal - String getTable(); +class Spark4ArrayData(override val elementType: DataType) extends AbstractSparkArrayData { - String getJob(); + override def getVariant(ordinal: Int): VariantVal = throw new UnsupportedOperationException - Timestamp getCreateTime(); } diff --git a/paimon-spark/paimon-spark4-common/src/main/scala/org/apache/paimon/spark/data/Spark4InternalRow.scala b/paimon-spark/paimon-spark4-common/src/main/scala/org/apache/paimon/spark/data/Spark4InternalRow.scala new file mode 100644 index 000000000000..54b0f420ea93 --- /dev/null +++ b/paimon-spark/paimon-spark4-common/src/main/scala/org/apache/paimon/spark/data/Spark4InternalRow.scala @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.data + +import org.apache.paimon.spark.AbstractSparkInternalRow +import org.apache.paimon.types.RowType + +import org.apache.spark.unsafe.types.VariantVal + +class Spark4InternalRow(rowType: RowType) extends AbstractSparkInternalRow(rowType) { + override def getVariant(i: Int): VariantVal = throw new UnsupportedOperationException +} diff --git a/paimon-spark/paimon-spark4-common/src/main/scala/org/apache/spark/sql/paimon/shims/Spark4Shim.scala b/paimon-spark/paimon-spark4-common/src/main/scala/org/apache/spark/sql/paimon/shims/Spark4Shim.scala new file mode 100644 index 000000000000..eefddafdbfb8 --- /dev/null +++ b/paimon-spark/paimon-spark4-common/src/main/scala/org/apache/spark/sql/paimon/shims/Spark4Shim.scala @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.paimon.shims + +import org.apache.paimon.spark.catalyst.analysis.Spark4ResolutionRules +import org.apache.paimon.spark.catalyst.parser.extensions.PaimonSpark4SqlExtensionsParser +import org.apache.paimon.spark.data.{Spark4ArrayData, Spark4InternalRow, SparkArrayData, SparkInternalRow} +import org.apache.paimon.types.{DataType, RowType} + +import org.apache.spark.sql.{Column, SparkSession} +import org.apache.spark.sql.catalyst.expressions.{Attribute, Expression} +import org.apache.spark.sql.catalyst.parser.ParserInterface +import org.apache.spark.sql.catalyst.plans.logical.{Aggregate, LogicalPlan} +import org.apache.spark.sql.catalyst.rules.Rule +import org.apache.spark.sql.connector.catalog.{CatalogV2Util, Identifier, Table, TableCatalog} +import org.apache.spark.sql.connector.expressions.Transform +import org.apache.spark.sql.internal.ExpressionUtils +import org.apache.spark.sql.types.StructType + +import java.util.{Map => JMap} + +class Spark4Shim extends SparkShim { + + override def createSparkParser(delegate: ParserInterface): ParserInterface = { + new PaimonSpark4SqlExtensionsParser(delegate) + } + + override def createCustomResolution(spark: SparkSession): Rule[LogicalPlan] = { + Spark4ResolutionRules(spark) + } + + override def createSparkInternalRow(rowType: RowType): SparkInternalRow = { + new Spark4InternalRow(rowType) + } + + override def createSparkArrayData(elementType: DataType): SparkArrayData = { + new Spark4ArrayData(elementType) + } + + def supportsHashAggregate( + aggregateBufferAttributes: Seq[Attribute], + groupingExpression: Seq[Expression]): Boolean = { + Aggregate.supportsHashAggregate(aggregateBufferAttributes, groupingExpression) + } + + def createTable( + tableCatalog: TableCatalog, + ident: Identifier, + schema: StructType, + partitions: Array[Transform], + properties: JMap[String, String]): Table = { + val columns = CatalogV2Util.structTypeToV2Columns(schema) + tableCatalog.createTable(ident, columns, partitions, properties) + } + + def column(expr: Expression): Column = ExpressionUtils.column(expr) + + def convertToExpression(spark: SparkSession, column: Column): Expression = + spark.expression(column) +} diff --git a/paimon-spark/pom.xml b/paimon-spark/pom.xml index c06b3a3d4e66..61ecd20a0500 100644 --- a/paimon-spark/pom.xml +++ b/paimon-spark/pom.xml @@ -23,8 +23,8 @@ under the License. 4.0.0 - paimon-parent org.apache.paimon + paimon-parent 1.0-SNAPSHOT @@ -39,20 +39,217 @@ under the License. paimon-spark-common - paimon-spark-3.5 - paimon-spark-3.4 - paimon-spark-3.3 - paimon-spark-3.2 + paimon-spark-ut + + + + org.apache.spark + spark-sql_${scala.binary.version} + + + log4j + log4j + + + org.slf4j + slf4j-log4j12 + + + org.apache.logging.log4j + log4j-slf4j2-impl + + + org.apache.orc + orc-core + + + org.apache.orc + orc-mapreduce + + + org.apache.parquet + parquet-column + + + org.apache.parquet + parquet-hadoop + + + + + + org.apache.spark + spark-core_${scala.binary.version} + + + log4j + log4j + + + org.slf4j + slf4j-log4j12 + + + + org.apache.logging.log4j + log4j-slf4j2-impl + + + org.apache.orc + orc-core + + + org.apache.orc + orc-mapreduce + + + org.apache.parquet + parquet-column + + + com.google.protobuf + protobuf-java + + + + + + org.apache.spark + spark-hive_${scala.binary.version} + + + log4j + log4j + + + org.slf4j + slf4j-log4j12 + + + + + + org.apache.paimon + paimon-bundle + ${project.version} + + + + + + org.apache.spark + spark-sql_${scala.binary.version} + tests + test + + + log4j + log4j + + + org.slf4j + slf4j-log4j12 + + + org.apache.logging.log4j + log4j-slf4j2-impl + + + + + + org.apache.spark + spark-catalyst_${scala.binary.version} + tests + test + + + log4j + log4j + + + org.slf4j + slf4j-log4j12 + + + + + + org.apache.spark + spark-core_${scala.binary.version} + tests + test + + + log4j + log4j + + + org.slf4j + slf4j-log4j12 + + + org.apache.logging.log4j + log4j-slf4j2-impl + + + + + + + + org.scala-lang + scala-library + ${scala.version} + + + + org.scala-lang + scala-reflect + ${scala.version} + + + + org.scala-lang + scala-compiler + ${scala.version} + + + + + + org.scalatest + scalatest_${scala.binary.version} + 3.1.0 + test + + org.apache.paimon - paimon-bundle + paimon-common ${project.version} + test-jar + test - + + org.apache.paimon + paimon-hive-common + ${project.version} + test + + + + org.apache.paimon + paimon-hive-common + ${project.version} + tests + test-jar + test + org.apache.paimon @@ -81,6 +278,12 @@ under the License. aws-java-sdk-core ${aws.version} test + + + com.fasterxml.jackson.core + * + + @@ -90,4 +293,90 @@ under the License. test + + + + + + net.alchim31.maven + scala-maven-plugin + ${scala-maven-plugin.version} + + + + scala-compile-first + process-resources + + add-source + compile + + + + + + scala-test-compile + process-test-resources + + testCompile + + + + + ${scala.version} + false + + -nobootcp + -target:jvm-${target.java.version} + + + + + + + org.scalatest + scalatest-maven-plugin + ${scalatest-maven-plugin.version} + + ${project.build.directory}/surefire-reports + . + -ea -Xmx4g -Xss4m -XX:MaxMetaspaceSize=2g -XX:ReservedCodeCacheSize=${CodeCacheSize} ${extraJavaTestArgs} -Dio.netty.tryReflectionSetAccessible=true + PaimonTestSuite.txt + + once + true + + + + test + + test + + + + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + + + net.alchim31.maven + scala-maven-plugin + + + + org.scalatest + scalatest-maven-plugin + + + diff --git a/pom.xml b/pom.xml index 1f0d47860124..dbef98af06b2 100644 --- a/pom.xml +++ b/pom.xml @@ -69,6 +69,7 @@ under the License. paimon-test-utils paimon-arrow tools/ci/paimon-ci-tools + paimon-open-api @@ -80,8 +81,12 @@ under the License. 4.1.100.Final 4.9.3 2.8.5 - 2.12.15 2.12 + 2.12 + 2.12.15 + 2.13.14 + ${scala212.version} + ${scala212.version} 1.1.8.4 0.27 1.8.0 @@ -97,6 +102,10 @@ under the License. 1C true 1.19.1 + 1.6.1 + 1.13.1 + 1.9.2 + 3.19.6 @@ -107,6 +116,12 @@ under the License. 3.0.1-1.18 8.0.27 + + paimon-spark3-common + 3.5.3 + 3.3 + 3.3.0 + 1.5.5-11 3.0.11 3.4.6 @@ -130,11 +145,13 @@ under the License. --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED + --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false + -Dio.netty.tryReflectionSetAccessible=true @@ -338,6 +355,55 @@ under the License. + + + spark3 + + paimon-spark/paimon-spark3-common + paimon-spark/paimon-spark-3.5 + paimon-spark/paimon-spark-3.4 + paimon-spark/paimon-spark-3.3 + paimon-spark/paimon-spark-3.2 + + + 2.12 + ${scala212.version} + 3.5.3 + paimon-spark3-common + + 3.3 + 3.3.0 + + + true + + spark3 + + + + + + spark4 + + paimon-spark/paimon-spark4-common + paimon-spark/paimon-spark-4.0 + + + 17 + 4.13.1 + 2.13 + ${scala213.version} + 4.0.0-preview2 + paimon-spark4-common + 4.0 + 4.0.0-preview2 + + + + spark4 + + + @@ -463,6 +529,7 @@ under the License. release/** paimon-common/src/main/antlr4/** + paimon-core/src/test/resources/compatibility/** @@ -493,7 +560,7 @@ under the License. true - -Xms256m -Xmx2048m -Dmvn.forkNumber=${surefire.forkNumber} -XX:+UseG1GC + -Xms256m -Xmx2048m -Dmvn.forkNumber=${surefire.forkNumber} -XX:+UseG1GC ${extraJavaTestArgs} @@ -533,7 +600,6 @@ under the License. - org.apache.maven.plugins maven-enforcer-plugin @@ -615,10 +681,10 @@ under the License. - org.apache.flink:flink-table-planner_${scala.binary.version} + org.apache.flink:flink-table-planner_${flink.scala.binary.version} - org.apache.flink:flink-table-planner_${scala.binary.version}:*:*:test + org.apache.flink:flink-table-planner_${flink.scala.binary.version}:*:*:test Direct dependencies on flink-table-planner are not allowed. diff --git a/tools/maven/checkstyle.xml b/tools/maven/checkstyle.xml index d5db52cb03df..80e785353526 100644 --- a/tools/maven/checkstyle.xml +++ b/tools/maven/checkstyle.xml @@ -74,7 +74,7 @@ This file is based on the checkstyle file of Apache Beam. --> - +