From da4fbb957898da0b929a81f9273b26fd25b48617 Mon Sep 17 00:00:00 2001 From: yunfengzhou-hub Date: Mon, 5 Aug 2024 14:05:23 +0800 Subject: [PATCH] [flink] Bump flink version to 1.20 (#3884) --- .github/workflows/e2e-tests-1.19-jdk11.yml | 2 +- .github/workflows/e2e-tests-1.19.yml | 2 +- .github/workflows/e2e-tests-1.20-jdk11.yml | 58 +++++++++++++ .github/workflows/e2e-tests-1.20.yml | 57 +++++++++++++ .github/workflows/unitcase-flink-jdk11.yml | 2 +- .github/workflows/utitcase-flink.yml | 2 +- docs/content/engines/overview.md | 4 +- docs/content/flink/quick-start.md | 4 +- docs/content/project/download.md | 2 + paimon-e2e-tests/pom.xml | 14 ++- paimon-flink/paimon-flink-1.20/pom.xml | 85 +++++++++++++++++++ paimon-flink/paimon-flink-common/pom.xml | 2 +- paimon-flink/pom.xml | 2 + paimon-hive/paimon-hive-connector-2.3/pom.xml | 2 +- .../hive/FlinkGenericCatalogITCase.java | 4 +- paimon-hive/pom.xml | 2 +- pom.xml | 4 +- 17 files changed, 233 insertions(+), 15 deletions(-) create mode 100644 .github/workflows/e2e-tests-1.20-jdk11.yml create mode 100644 .github/workflows/e2e-tests-1.20.yml create mode 100644 paimon-flink/paimon-flink-1.20/pom.xml diff --git a/.github/workflows/e2e-tests-1.19-jdk11.yml b/.github/workflows/e2e-tests-1.19-jdk11.yml index bc917f453f0f..5a809aeae322 100644 --- a/.github/workflows/e2e-tests-1.19-jdk11.yml +++ b/.github/workflows/e2e-tests-1.19-jdk11.yml @@ -53,6 +53,6 @@ jobs: . .github/workflows/utils.sh jvm_timezone=$(random_timezone) echo "JVM timezone is set to $jvm_timezone" - mvn -T 1C -B test -pl paimon-e2e-tests -Duser.timezone=$jvm_timezone + mvn -T 1C -B test -pl paimon-e2e-tests -Duser.timezone=$jvm_timezone -Pflink-1.19 env: MAVEN_OPTS: -Xmx4096m \ No newline at end of file diff --git a/.github/workflows/e2e-tests-1.19.yml b/.github/workflows/e2e-tests-1.19.yml index b451d6385a9f..b9444b89fd2b 100644 --- a/.github/workflows/e2e-tests-1.19.yml +++ b/.github/workflows/e2e-tests-1.19.yml @@ -52,6 +52,6 @@ jobs: . .github/workflows/utils.sh jvm_timezone=$(random_timezone) echo "JVM timezone is set to $jvm_timezone" - mvn -T 1C -B test -pl paimon-e2e-tests -Duser.timezone=$jvm_timezone + mvn -T 1C -B test -pl paimon-e2e-tests -Duser.timezone=$jvm_timezone -Pflink-1.19 env: MAVEN_OPTS: -Xmx4096m \ No newline at end of file diff --git a/.github/workflows/e2e-tests-1.20-jdk11.yml b/.github/workflows/e2e-tests-1.20-jdk11.yml new file mode 100644 index 000000000000..968ea74b9c54 --- /dev/null +++ b/.github/workflows/e2e-tests-1.20-jdk11.yml @@ -0,0 +1,58 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +name: End to End Tests Flink 1.20 on JDK 11 + +on: + issue_comment: + types: [created, edited, deleted] + + # daily run + schedule: + - cron: "0 0 * * *" + +env: + JDK_VERSION: 11 + +jobs: + build: + if: | + github.event_name == 'schedule' || + (contains(github.event.comment.html_url, '/pull/') && contains(github.event.comment.body, '/jdk11')) + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v2 + - name: Set up JDK ${{ env.JDK_VERSION }} + uses: actions/setup-java@v2 + with: + java-version: ${{ env.JDK_VERSION }} + distribution: 'adopt' + - name: Build Flink 1.20 + run: mvn -T 1C -B clean install -DskipTests + - name: Test Flink 1.20 + timeout-minutes: 60 + run: | + # run tests with random timezone to find out timezone related bugs + . .github/workflows/utils.sh + jvm_timezone=$(random_timezone) + echo "JVM timezone is set to $jvm_timezone" + mvn -T 1C -B test -pl paimon-e2e-tests -Duser.timezone=$jvm_timezone + env: + MAVEN_OPTS: -Xmx4096m \ No newline at end of file diff --git a/.github/workflows/e2e-tests-1.20.yml b/.github/workflows/e2e-tests-1.20.yml new file mode 100644 index 000000000000..a511a87d4f55 --- /dev/null +++ b/.github/workflows/e2e-tests-1.20.yml @@ -0,0 +1,57 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +name: End to End Tests Flink 1.20 + +on: + push: + pull_request: + paths-ignore: + - 'docs/**' + - '**/*.md' + +env: + JDK_VERSION: 8 + +concurrency: + group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.number || github.run_id }} + cancel-in-progress: true + +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v2 + - name: Set up JDK ${{ env.JDK_VERSION }} + uses: actions/setup-java@v2 + with: + java-version: ${{ env.JDK_VERSION }} + distribution: 'adopt' + - name: Build Flink 1.20 + run: mvn -T 1C -B clean install -DskipTests + - name: Test Flink 1.20 + timeout-minutes: 60 + run: | + # run tests with random timezone to find out timezone related bugs + . .github/workflows/utils.sh + jvm_timezone=$(random_timezone) + echo "JVM timezone is set to $jvm_timezone" + mvn -T 1C -B test -pl paimon-e2e-tests -Duser.timezone=$jvm_timezone + env: + MAVEN_OPTS: -Xmx4096m diff --git a/.github/workflows/unitcase-flink-jdk11.yml b/.github/workflows/unitcase-flink-jdk11.yml index 135dc5718f23..42dfe8442ed5 100644 --- a/.github/workflows/unitcase-flink-jdk11.yml +++ b/.github/workflows/unitcase-flink-jdk11.yml @@ -53,7 +53,7 @@ jobs: jvm_timezone=$(random_timezone) echo "JVM timezone is set to $jvm_timezone" test_modules="" - for suffix in 1.15 1.16 1.17 1.18 1.19 common; do + for suffix in 1.15 1.16 1.17 1.18 1.19 1.20 common; do test_modules+="org.apache.paimon:paimon-flink-${suffix}," done test_modules="${test_modules%,}" diff --git a/.github/workflows/utitcase-flink.yml b/.github/workflows/utitcase-flink.yml index c7455e8143b4..c39f4a7c4829 100644 --- a/.github/workflows/utitcase-flink.yml +++ b/.github/workflows/utitcase-flink.yml @@ -52,7 +52,7 @@ jobs: jvm_timezone=$(random_timezone) echo "JVM timezone is set to $jvm_timezone" test_modules="" - for suffix in 1.15 1.16 1.17 1.18 1.19 common; do + for suffix in 1.15 1.16 1.17 1.18 1.19 1.20 common; do test_modules+="org.apache.paimon:paimon-flink-${suffix}," done test_modules="${test_modules%,}" diff --git a/docs/content/engines/overview.md b/docs/content/engines/overview.md index f80cf6271bb9..264071d994f6 100644 --- a/docs/content/engines/overview.md +++ b/docs/content/engines/overview.md @@ -30,13 +30,13 @@ under the License. | Engine | Version | Batch Read | Batch Write | Create Table | Alter Table | Streaming Write | Streaming Read | Batch Overwrite | DELETE & UPDATE | MERGE INTO | Time Travel | |:-------------------------------------------------------------------------------:|:-------------:|:-----------:|:-----------:|:-------------:|:-------------:|:----------------:|:----------------:|:---------------:|:------------------:|:-----------:|:-----------:| -| Flink | 1.15 - 1.19 | ✅ | ✅ | ✅ | ✅(1.17+) | ✅ | ✅ | ✅ | ✅(1.17+) | ❌ | ✅ | +| Flink | 1.15 - 1.20 | ✅ | ✅ | ✅ | ✅(1.17+) | ✅ | ✅ | ✅ | ✅(1.17+) | ❌ | ✅ | | Spark | 3.1 - 3.5 | ✅ | ✅(3.2+) | ✅ | ✅ | ✅(3.3+) | ✅(3.3+) | ✅(3.2+) | ✅(3.2+) | ✅(3.2+) | ✅(3.3+) | | Hive | 2.1 - 3.1 | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | | Trino | 420 - 439 | ✅ | ✅(427+) | ✅(427+) | ✅(427+) | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | | Presto | 0.236 - 0.280 | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | | [StarRocks](https://docs.starrocks.io/docs/data_source/catalog/paimon_catalog/) | 3.1+ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | -| [Doris](https://doris.apache.org/docs/lakehouse/datalake-analytics/paimon) | 2.0.6+ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | +| [Doris](https://doris.apache.org/docs/lakehouse/datalake-analytics/paimon) | 2.0.6+ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ## Streaming Engines diff --git a/docs/content/flink/quick-start.md b/docs/content/flink/quick-start.md index 9980c6c159d4..62559065ec9a 100644 --- a/docs/content/flink/quick-start.md +++ b/docs/content/flink/quick-start.md @@ -30,7 +30,7 @@ This documentation is a guide for using Paimon in Flink. ## Jars -Paimon currently supports Flink 1.19, 1.18, 1.17, 1.16, 1.15. We recommend the latest Flink version for a better experience. +Paimon currently supports Flink 1.20, 1.19, 1.18, 1.17, 1.16, 1.15. We recommend the latest Flink version for a better experience. Download the jar file with corresponding version. @@ -39,6 +39,7 @@ Download the jar file with corresponding version. | Version | Type | Jar | |--------------|-------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| Flink 1.20 | Bundled Jar | [paimon-flink-1.20-{{< version >}}.jar](https://repo.maven.apache.org/maven2/org/apache/paimon/paimon-flink-1.20/{{< version >}}/paimon-flink-1.20-{{< version >}}.jar) | | Flink 1.19 | Bundled Jar | [paimon-flink-1.19-{{< version >}}.jar](https://repo.maven.apache.org/maven2/org/apache/paimon/paimon-flink-1.19/{{< version >}}/paimon-flink-1.19-{{< version >}}.jar) | | Flink 1.18 | Bundled Jar | [paimon-flink-1.18-{{< version >}}.jar](https://repo.maven.apache.org/maven2/org/apache/paimon/paimon-flink-1.18/{{< version >}}/paimon-flink-1.18-{{< version >}}.jar) | | Flink 1.17 | Bundled Jar | [paimon-flink-1.17-{{< version >}}.jar](https://repo.maven.apache.org/maven2/org/apache/paimon/paimon-flink-1.17/{{< version >}}/paimon-flink-1.17-{{< version >}}.jar) | @@ -52,6 +53,7 @@ Download the jar file with corresponding version. | Version | Type | Jar | |--------------|-------------|-------------------------------------------------------------------------------------------------------------------------------------------| +| Flink 1.20 | Bundled Jar | [paimon-flink-1.20-{{< version >}}.jar](https://repository.apache.org/snapshots/org/apache/paimon/paimon-flink-1.20/{{< version >}}/) | | Flink 1.19 | Bundled Jar | [paimon-flink-1.19-{{< version >}}.jar](https://repository.apache.org/snapshots/org/apache/paimon/paimon-flink-1.19/{{< version >}}/) | | Flink 1.18 | Bundled Jar | [paimon-flink-1.18-{{< version >}}.jar](https://repository.apache.org/snapshots/org/apache/paimon/paimon-flink-1.18/{{< version >}}/) | | Flink 1.17 | Bundled Jar | [paimon-flink-1.17-{{< version >}}.jar](https://repository.apache.org/snapshots/org/apache/paimon/paimon-flink-1.17/{{< version >}}/) | diff --git a/docs/content/project/download.md b/docs/content/project/download.md index 04d391e7aab5..f9189132a53b 100644 --- a/docs/content/project/download.md +++ b/docs/content/project/download.md @@ -34,6 +34,7 @@ This documentation is a guide for downloading Paimon Jars. | Version | Jar | |------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| Flink 1.20 | [paimon-flink-1.20-{{< version >}}.jar](https://repository.apache.org/snapshots/org/apache/paimon/paimon-flink-1.20/{{< version >}}/) | | Flink 1.19 | [paimon-flink-1.19-{{< version >}}.jar](https://repository.apache.org/snapshots/org/apache/paimon/paimon-flink-1.19/{{< version >}}/) | | Flink 1.18 | [paimon-flink-1.18-{{< version >}}.jar](https://repository.apache.org/snapshots/org/apache/paimon/paimon-flink-1.18/{{< version >}}/) | | Flink 1.17 | [paimon-flink-1.17-{{< version >}}.jar](https://repository.apache.org/snapshots/org/apache/paimon/paimon-flink-1.17/{{< version >}}/) | @@ -63,6 +64,7 @@ This documentation is a guide for downloading Paimon Jars. | Version | Jar | |------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| Flink 1.20 | [paimon-flink-1.20-{{< version >}}.jar](https://repo.maven.apache.org/maven2/org/apache/paimon/paimon-flink-1.20/{{< version >}}/paimon-flink-1.20-{{< version >}}.jar) | | Flink 1.19 | [paimon-flink-1.19-{{< version >}}.jar](https://repo.maven.apache.org/maven2/org/apache/paimon/paimon-flink-1.19/{{< version >}}/paimon-flink-1.19-{{< version >}}.jar) | | Flink 1.18 | [paimon-flink-1.18-{{< version >}}.jar](https://repo.maven.apache.org/maven2/org/apache/paimon/paimon-flink-1.18/{{< version >}}/paimon-flink-1.18-{{< version >}}.jar) | | Flink 1.17 | [paimon-flink-1.17-{{< version >}}.jar](https://repo.maven.apache.org/maven2/org/apache/paimon/paimon-flink-1.17/{{< version >}}/paimon-flink-1.17-{{< version >}}.jar) | diff --git a/paimon-e2e-tests/pom.xml b/paimon-e2e-tests/pom.xml index 5b99a868ec13..1b439bea20e1 100644 --- a/paimon-e2e-tests/pom.xml +++ b/paimon-e2e-tests/pom.xml @@ -34,7 +34,7 @@ under the License. 2.8.3-10.0 2.4.2 - flink-sql-connector-hive-2.3.9_${scala.binary.version} + flink-sql-connector-hive-2.3.10_${scala.binary.version} @@ -275,11 +275,21 @@ under the License. + + flink-1.19 + + 1.19 + 1.19.1 + flink-sql-connector-hive-2.3.9_${scala.binary.version} + + + flink-1.18 1.18 1.18.1 + flink-sql-connector-hive-2.3.9_${scala.binary.version} @@ -289,6 +299,7 @@ under the License. 1.17 1.17.2 ${test.flink.version} + flink-sql-connector-hive-2.3.9_${scala.binary.version} @@ -298,6 +309,7 @@ under the License. 1.16 1.16.3 ${test.flink.version} + flink-sql-connector-hive-2.3.9_${scala.binary.version} diff --git a/paimon-flink/paimon-flink-1.20/pom.xml b/paimon-flink/paimon-flink-1.20/pom.xml new file mode 100644 index 000000000000..ebee64a66ad0 --- /dev/null +++ b/paimon-flink/paimon-flink-1.20/pom.xml @@ -0,0 +1,85 @@ + + + + 4.0.0 + + + org.apache.paimon + paimon-flink + 0.9-SNAPSHOT + + + jar + + paimon-flink-1.20 + Paimon : Flink : 1.20 + + + 1.20.0 + + + + + org.apache.paimon + paimon-flink-common + ${project.version} + + + + org.apache.paimon + paimon-flink-cdc + ${project.version} + + + * + * + + + + + + + + + org.apache.maven.plugins + maven-shade-plugin + + + shade-paimon + package + + shade + + + + + org.apache.paimon:paimon-flink-common + org.apache.paimon:paimon-flink-cdc + + + + + + + + + diff --git a/paimon-flink/paimon-flink-common/pom.xml b/paimon-flink/paimon-flink-common/pom.xml index 54ef1516acd0..8e24d26a281f 100644 --- a/paimon-flink/paimon-flink-common/pom.xml +++ b/paimon-flink/paimon-flink-common/pom.xml @@ -34,7 +34,7 @@ under the License. Paimon : Flink : Common - 1.19.1 + 1.20.0 diff --git a/paimon-flink/pom.xml b/paimon-flink/pom.xml index cffafe1b5d49..c9281d0c0bf8 100644 --- a/paimon-flink/pom.xml +++ b/paimon-flink/pom.xml @@ -40,6 +40,7 @@ under the License. paimon-flink-1.17 paimon-flink-1.18 paimon-flink-1.19 + paimon-flink-1.20 paimon-flink-action paimon-flink-cdc @@ -168,6 +169,7 @@ under the License. paimon-flink-1.17 paimon-flink-1.18 paimon-flink-1.19 + paimon-flink-1.20 paimon-flink-cdc diff --git a/paimon-hive/paimon-hive-connector-2.3/pom.xml b/paimon-hive/paimon-hive-connector-2.3/pom.xml index 345494029903..34c6c2d18f66 100644 --- a/paimon-hive/paimon-hive-connector-2.3/pom.xml +++ b/paimon-hive/paimon-hive-connector-2.3/pom.xml @@ -34,7 +34,7 @@ under the License. jar - 2.3.9 + 2.3.10 diff --git a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/FlinkGenericCatalogITCase.java b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/FlinkGenericCatalogITCase.java index bd9a8bd1f28b..3e8760dc1a7f 100644 --- a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/FlinkGenericCatalogITCase.java +++ b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/FlinkGenericCatalogITCase.java @@ -33,7 +33,7 @@ import org.apache.flink.table.api.internal.TableEnvironmentImpl; import org.apache.flink.table.catalog.hive.HiveCatalog; import org.apache.flink.table.catalog.hive.client.HiveShimLoader; -import org.apache.flink.test.util.AbstractTestBase; +import org.apache.flink.test.util.AbstractTestBaseJUnit4; import org.apache.flink.types.Row; import org.apache.flink.util.CloseableIterator; import org.apache.hadoop.hive.conf.HiveConf; @@ -51,7 +51,7 @@ /** IT cases for using Flink {@code FlinkGenericCatalog}. */ @RunWith(PaimonEmbeddedHiveRunner.class) -public class FlinkGenericCatalogITCase extends AbstractTestBase { +public class FlinkGenericCatalogITCase extends AbstractTestBaseJUnit4 { @Rule public TemporaryFolder folder = new TemporaryFolder(); diff --git a/paimon-hive/pom.xml b/paimon-hive/pom.xml index 6f3a1254d1f6..3456cd7597ed 100644 --- a/paimon-hive/pom.xml +++ b/paimon-hive/pom.xml @@ -45,7 +45,7 @@ under the License. - 2.3.9 + 2.3.10 4.0.0 0.9.8 1.12.319 diff --git a/pom.xml b/pom.xml index b9f67a5ef6f2..75a7cd098a95 100644 --- a/pom.xml +++ b/pom.xml @@ -100,8 +100,8 @@ under the License. **/*Test.* - 1.19 - 1.19.1 + 1.20 + 1.20.0 3.0.1-1.18