diff --git a/.github/workflows/e2e-tests-1.19-jdk11.yml b/.github/workflows/e2e-tests-1.19-jdk11.yml index bc917f453f0f1..5a809aeae3227 100644 --- a/.github/workflows/e2e-tests-1.19-jdk11.yml +++ b/.github/workflows/e2e-tests-1.19-jdk11.yml @@ -53,6 +53,6 @@ jobs: . .github/workflows/utils.sh jvm_timezone=$(random_timezone) echo "JVM timezone is set to $jvm_timezone" - mvn -T 1C -B test -pl paimon-e2e-tests -Duser.timezone=$jvm_timezone + mvn -T 1C -B test -pl paimon-e2e-tests -Duser.timezone=$jvm_timezone -Pflink-1.19 env: MAVEN_OPTS: -Xmx4096m \ No newline at end of file diff --git a/.github/workflows/e2e-tests-1.19.yml b/.github/workflows/e2e-tests-1.19.yml index b451d6385a9f4..b9444b89fd2be 100644 --- a/.github/workflows/e2e-tests-1.19.yml +++ b/.github/workflows/e2e-tests-1.19.yml @@ -52,6 +52,6 @@ jobs: . .github/workflows/utils.sh jvm_timezone=$(random_timezone) echo "JVM timezone is set to $jvm_timezone" - mvn -T 1C -B test -pl paimon-e2e-tests -Duser.timezone=$jvm_timezone + mvn -T 1C -B test -pl paimon-e2e-tests -Duser.timezone=$jvm_timezone -Pflink-1.19 env: MAVEN_OPTS: -Xmx4096m \ No newline at end of file diff --git a/.github/workflows/e2e-tests-1.20-jdk11.yml b/.github/workflows/e2e-tests-1.20-jdk11.yml new file mode 100644 index 0000000000000..968ea74b9c54a --- /dev/null +++ b/.github/workflows/e2e-tests-1.20-jdk11.yml @@ -0,0 +1,58 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +name: End to End Tests Flink 1.20 on JDK 11 + +on: + issue_comment: + types: [created, edited, deleted] + + # daily run + schedule: + - cron: "0 0 * * *" + +env: + JDK_VERSION: 11 + +jobs: + build: + if: | + github.event_name == 'schedule' || + (contains(github.event.comment.html_url, '/pull/') && contains(github.event.comment.body, '/jdk11')) + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v2 + - name: Set up JDK ${{ env.JDK_VERSION }} + uses: actions/setup-java@v2 + with: + java-version: ${{ env.JDK_VERSION }} + distribution: 'adopt' + - name: Build Flink 1.20 + run: mvn -T 1C -B clean install -DskipTests + - name: Test Flink 1.20 + timeout-minutes: 60 + run: | + # run tests with random timezone to find out timezone related bugs + . .github/workflows/utils.sh + jvm_timezone=$(random_timezone) + echo "JVM timezone is set to $jvm_timezone" + mvn -T 1C -B test -pl paimon-e2e-tests -Duser.timezone=$jvm_timezone + env: + MAVEN_OPTS: -Xmx4096m \ No newline at end of file diff --git a/.github/workflows/e2e-tests-1.20.yml b/.github/workflows/e2e-tests-1.20.yml new file mode 100644 index 0000000000000..a511a87d4f550 --- /dev/null +++ b/.github/workflows/e2e-tests-1.20.yml @@ -0,0 +1,57 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +name: End to End Tests Flink 1.20 + +on: + push: + pull_request: + paths-ignore: + - 'docs/**' + - '**/*.md' + +env: + JDK_VERSION: 8 + +concurrency: + group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.number || github.run_id }} + cancel-in-progress: true + +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v2 + - name: Set up JDK ${{ env.JDK_VERSION }} + uses: actions/setup-java@v2 + with: + java-version: ${{ env.JDK_VERSION }} + distribution: 'adopt' + - name: Build Flink 1.20 + run: mvn -T 1C -B clean install -DskipTests + - name: Test Flink 1.20 + timeout-minutes: 60 + run: | + # run tests with random timezone to find out timezone related bugs + . .github/workflows/utils.sh + jvm_timezone=$(random_timezone) + echo "JVM timezone is set to $jvm_timezone" + mvn -T 1C -B test -pl paimon-e2e-tests -Duser.timezone=$jvm_timezone + env: + MAVEN_OPTS: -Xmx4096m diff --git a/.github/workflows/unitcase-flink-jdk11.yml b/.github/workflows/unitcase-flink-jdk11.yml index 135dc5718f23d..42dfe8442ed54 100644 --- a/.github/workflows/unitcase-flink-jdk11.yml +++ b/.github/workflows/unitcase-flink-jdk11.yml @@ -53,7 +53,7 @@ jobs: jvm_timezone=$(random_timezone) echo "JVM timezone is set to $jvm_timezone" test_modules="" - for suffix in 1.15 1.16 1.17 1.18 1.19 common; do + for suffix in 1.15 1.16 1.17 1.18 1.19 1.20 common; do test_modules+="org.apache.paimon:paimon-flink-${suffix}," done test_modules="${test_modules%,}" diff --git a/.github/workflows/utitcase-flink.yml b/.github/workflows/utitcase-flink.yml index c7455e8143b47..c39f4a7c4829f 100644 --- a/.github/workflows/utitcase-flink.yml +++ b/.github/workflows/utitcase-flink.yml @@ -52,7 +52,7 @@ jobs: jvm_timezone=$(random_timezone) echo "JVM timezone is set to $jvm_timezone" test_modules="" - for suffix in 1.15 1.16 1.17 1.18 1.19 common; do + for suffix in 1.15 1.16 1.17 1.18 1.19 1.20 common; do test_modules+="org.apache.paimon:paimon-flink-${suffix}," done test_modules="${test_modules%,}" diff --git a/docs/content/engines/overview.md b/docs/content/engines/overview.md index f80cf6271bb9b..264071d994f60 100644 --- a/docs/content/engines/overview.md +++ b/docs/content/engines/overview.md @@ -30,13 +30,13 @@ under the License. | Engine | Version | Batch Read | Batch Write | Create Table | Alter Table | Streaming Write | Streaming Read | Batch Overwrite | DELETE & UPDATE | MERGE INTO | Time Travel | |:-------------------------------------------------------------------------------:|:-------------:|:-----------:|:-----------:|:-------------:|:-------------:|:----------------:|:----------------:|:---------------:|:------------------:|:-----------:|:-----------:| -| Flink | 1.15 - 1.19 | ✅ | ✅ | ✅ | ✅(1.17+) | ✅ | ✅ | ✅ | ✅(1.17+) | ❌ | ✅ | +| Flink | 1.15 - 1.20 | ✅ | ✅ | ✅ | ✅(1.17+) | ✅ | ✅ | ✅ | ✅(1.17+) | ❌ | ✅ | | Spark | 3.1 - 3.5 | ✅ | ✅(3.2+) | ✅ | ✅ | ✅(3.3+) | ✅(3.3+) | ✅(3.2+) | ✅(3.2+) | ✅(3.2+) | ✅(3.3+) | | Hive | 2.1 - 3.1 | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | | Trino | 420 - 439 | ✅ | ✅(427+) | ✅(427+) | ✅(427+) | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | | Presto | 0.236 - 0.280 | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | | [StarRocks](https://docs.starrocks.io/docs/data_source/catalog/paimon_catalog/) | 3.1+ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | -| [Doris](https://doris.apache.org/docs/lakehouse/datalake-analytics/paimon) | 2.0.6+ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | +| [Doris](https://doris.apache.org/docs/lakehouse/datalake-analytics/paimon) | 2.0.6+ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ## Streaming Engines diff --git a/docs/content/flink/quick-start.md b/docs/content/flink/quick-start.md index 9980c6c159d42..62559065ec9a9 100644 --- a/docs/content/flink/quick-start.md +++ b/docs/content/flink/quick-start.md @@ -30,7 +30,7 @@ This documentation is a guide for using Paimon in Flink. ## Jars -Paimon currently supports Flink 1.19, 1.18, 1.17, 1.16, 1.15. We recommend the latest Flink version for a better experience. +Paimon currently supports Flink 1.20, 1.19, 1.18, 1.17, 1.16, 1.15. We recommend the latest Flink version for a better experience. Download the jar file with corresponding version. @@ -39,6 +39,7 @@ Download the jar file with corresponding version. | Version | Type | Jar | |--------------|-------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| Flink 1.20 | Bundled Jar | [paimon-flink-1.20-{{< version >}}.jar](https://repo.maven.apache.org/maven2/org/apache/paimon/paimon-flink-1.20/{{< version >}}/paimon-flink-1.20-{{< version >}}.jar) | | Flink 1.19 | Bundled Jar | [paimon-flink-1.19-{{< version >}}.jar](https://repo.maven.apache.org/maven2/org/apache/paimon/paimon-flink-1.19/{{< version >}}/paimon-flink-1.19-{{< version >}}.jar) | | Flink 1.18 | Bundled Jar | [paimon-flink-1.18-{{< version >}}.jar](https://repo.maven.apache.org/maven2/org/apache/paimon/paimon-flink-1.18/{{< version >}}/paimon-flink-1.18-{{< version >}}.jar) | | Flink 1.17 | Bundled Jar | [paimon-flink-1.17-{{< version >}}.jar](https://repo.maven.apache.org/maven2/org/apache/paimon/paimon-flink-1.17/{{< version >}}/paimon-flink-1.17-{{< version >}}.jar) | @@ -52,6 +53,7 @@ Download the jar file with corresponding version. | Version | Type | Jar | |--------------|-------------|-------------------------------------------------------------------------------------------------------------------------------------------| +| Flink 1.20 | Bundled Jar | [paimon-flink-1.20-{{< version >}}.jar](https://repository.apache.org/snapshots/org/apache/paimon/paimon-flink-1.20/{{< version >}}/) | | Flink 1.19 | Bundled Jar | [paimon-flink-1.19-{{< version >}}.jar](https://repository.apache.org/snapshots/org/apache/paimon/paimon-flink-1.19/{{< version >}}/) | | Flink 1.18 | Bundled Jar | [paimon-flink-1.18-{{< version >}}.jar](https://repository.apache.org/snapshots/org/apache/paimon/paimon-flink-1.18/{{< version >}}/) | | Flink 1.17 | Bundled Jar | [paimon-flink-1.17-{{< version >}}.jar](https://repository.apache.org/snapshots/org/apache/paimon/paimon-flink-1.17/{{< version >}}/) | diff --git a/docs/content/project/download.md b/docs/content/project/download.md index 04d391e7aab59..f9189132a53b0 100644 --- a/docs/content/project/download.md +++ b/docs/content/project/download.md @@ -34,6 +34,7 @@ This documentation is a guide for downloading Paimon Jars. | Version | Jar | |------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| Flink 1.20 | [paimon-flink-1.20-{{< version >}}.jar](https://repository.apache.org/snapshots/org/apache/paimon/paimon-flink-1.20/{{< version >}}/) | | Flink 1.19 | [paimon-flink-1.19-{{< version >}}.jar](https://repository.apache.org/snapshots/org/apache/paimon/paimon-flink-1.19/{{< version >}}/) | | Flink 1.18 | [paimon-flink-1.18-{{< version >}}.jar](https://repository.apache.org/snapshots/org/apache/paimon/paimon-flink-1.18/{{< version >}}/) | | Flink 1.17 | [paimon-flink-1.17-{{< version >}}.jar](https://repository.apache.org/snapshots/org/apache/paimon/paimon-flink-1.17/{{< version >}}/) | @@ -63,6 +64,7 @@ This documentation is a guide for downloading Paimon Jars. | Version | Jar | |------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| Flink 1.20 | [paimon-flink-1.20-{{< version >}}.jar](https://repo.maven.apache.org/maven2/org/apache/paimon/paimon-flink-1.20/{{< version >}}/paimon-flink-1.20-{{< version >}}.jar) | | Flink 1.19 | [paimon-flink-1.19-{{< version >}}.jar](https://repo.maven.apache.org/maven2/org/apache/paimon/paimon-flink-1.19/{{< version >}}/paimon-flink-1.19-{{< version >}}.jar) | | Flink 1.18 | [paimon-flink-1.18-{{< version >}}.jar](https://repo.maven.apache.org/maven2/org/apache/paimon/paimon-flink-1.18/{{< version >}}/paimon-flink-1.18-{{< version >}}.jar) | | Flink 1.17 | [paimon-flink-1.17-{{< version >}}.jar](https://repo.maven.apache.org/maven2/org/apache/paimon/paimon-flink-1.17/{{< version >}}/paimon-flink-1.17-{{< version >}}.jar) | diff --git a/paimon-e2e-tests/pom.xml b/paimon-e2e-tests/pom.xml index 5b99a868ec138..1b439bea20e11 100644 --- a/paimon-e2e-tests/pom.xml +++ b/paimon-e2e-tests/pom.xml @@ -34,7 +34,7 @@ under the License. 2.8.3-10.0 2.4.2 - flink-sql-connector-hive-2.3.9_${scala.binary.version} + flink-sql-connector-hive-2.3.10_${scala.binary.version} @@ -275,11 +275,21 @@ under the License. + + flink-1.19 + + 1.19 + 1.19.1 + flink-sql-connector-hive-2.3.9_${scala.binary.version} + + + flink-1.18 1.18 1.18.1 + flink-sql-connector-hive-2.3.9_${scala.binary.version} @@ -289,6 +299,7 @@ under the License. 1.17 1.17.2 ${test.flink.version} + flink-sql-connector-hive-2.3.9_${scala.binary.version} @@ -298,6 +309,7 @@ under the License. 1.16 1.16.3 ${test.flink.version} + flink-sql-connector-hive-2.3.9_${scala.binary.version} diff --git a/paimon-flink/paimon-flink-1.20/pom.xml b/paimon-flink/paimon-flink-1.20/pom.xml new file mode 100644 index 0000000000000..ebee64a66ad0b --- /dev/null +++ b/paimon-flink/paimon-flink-1.20/pom.xml @@ -0,0 +1,85 @@ + + + + 4.0.0 + + + org.apache.paimon + paimon-flink + 0.9-SNAPSHOT + + + jar + + paimon-flink-1.20 + Paimon : Flink : 1.20 + + + 1.20.0 + + + + + org.apache.paimon + paimon-flink-common + ${project.version} + + + + org.apache.paimon + paimon-flink-cdc + ${project.version} + + + * + * + + + + + + + + + org.apache.maven.plugins + maven-shade-plugin + + + shade-paimon + package + + shade + + + + + org.apache.paimon:paimon-flink-common + org.apache.paimon:paimon-flink-cdc + + + + + + + + + diff --git a/paimon-flink/paimon-flink-common/pom.xml b/paimon-flink/paimon-flink-common/pom.xml index 54ef1516acd0f..8e24d26a281ff 100644 --- a/paimon-flink/paimon-flink-common/pom.xml +++ b/paimon-flink/paimon-flink-common/pom.xml @@ -34,7 +34,7 @@ under the License. Paimon : Flink : Common - 1.19.1 + 1.20.0 diff --git a/paimon-flink/pom.xml b/paimon-flink/pom.xml index cffafe1b5d49d..c9281d0c0bf89 100644 --- a/paimon-flink/pom.xml +++ b/paimon-flink/pom.xml @@ -40,6 +40,7 @@ under the License. paimon-flink-1.17 paimon-flink-1.18 paimon-flink-1.19 + paimon-flink-1.20 paimon-flink-action paimon-flink-cdc @@ -168,6 +169,7 @@ under the License. paimon-flink-1.17 paimon-flink-1.18 paimon-flink-1.19 + paimon-flink-1.20 paimon-flink-cdc diff --git a/paimon-hive/paimon-hive-connector-2.3/pom.xml b/paimon-hive/paimon-hive-connector-2.3/pom.xml index 3454940299035..34c6c2d18f664 100644 --- a/paimon-hive/paimon-hive-connector-2.3/pom.xml +++ b/paimon-hive/paimon-hive-connector-2.3/pom.xml @@ -34,7 +34,7 @@ under the License. jar - 2.3.9 + 2.3.10 diff --git a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/FlinkGenericCatalogITCase.java b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/FlinkGenericCatalogITCase.java index bd9a8bd1f28ba..3e8760dc1a7ff 100644 --- a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/FlinkGenericCatalogITCase.java +++ b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/FlinkGenericCatalogITCase.java @@ -33,7 +33,7 @@ import org.apache.flink.table.api.internal.TableEnvironmentImpl; import org.apache.flink.table.catalog.hive.HiveCatalog; import org.apache.flink.table.catalog.hive.client.HiveShimLoader; -import org.apache.flink.test.util.AbstractTestBase; +import org.apache.flink.test.util.AbstractTestBaseJUnit4; import org.apache.flink.types.Row; import org.apache.flink.util.CloseableIterator; import org.apache.hadoop.hive.conf.HiveConf; @@ -51,7 +51,7 @@ /** IT cases for using Flink {@code FlinkGenericCatalog}. */ @RunWith(PaimonEmbeddedHiveRunner.class) -public class FlinkGenericCatalogITCase extends AbstractTestBase { +public class FlinkGenericCatalogITCase extends AbstractTestBaseJUnit4 { @Rule public TemporaryFolder folder = new TemporaryFolder(); diff --git a/paimon-hive/pom.xml b/paimon-hive/pom.xml index 6f3a1254d1f68..3456cd7597ed3 100644 --- a/paimon-hive/pom.xml +++ b/paimon-hive/pom.xml @@ -45,7 +45,7 @@ under the License. - 2.3.9 + 2.3.10 4.0.0 0.9.8 1.12.319 diff --git a/pom.xml b/pom.xml index b9f67a5ef6f22..75a7cd098a95a 100644 --- a/pom.xml +++ b/pom.xml @@ -100,8 +100,8 @@ under the License. **/*Test.* - 1.19 - 1.19.1 + 1.20 + 1.20.0 3.0.1-1.18