Merge branch 'master' into hive-client-pool

apache · Mar 25, 2024 · 46a76d9 · 46a76d9
2 parents 355641a + 1cf8b47
commit 46a76d9
Show file tree

Hide file tree

Showing 149 changed files with 6,480 additions and 1,065 deletions.
diff --git a/.asf.yaml b/.asf.yaml
@@ -18,7 +18,7 @@
 # See: https://cwiki.apache.org/confluence/display/INFRA/git+-+.asf.yaml+features
 
 github:
-  description: "Apache Paimon(incubating) is a lake format that enables building a Realtime Lakehouse Architecture with Flink and Spark for both streaming and batch operations."
+  description: "Apache Paimon is a lake format that enables building a Realtime Lakehouse Architecture with Flink and Spark for both streaming and batch operations."
   homepage: https://paimon.apache.org/
   labels:
     - paimon

diff --git a/.github/ISSUE_TEMPLATE/bug-report.yml b/.github/ISSUE_TEMPLATE/bug-report.yml
@@ -29,10 +29,10 @@ body:
     attributes:
       label: Search before asking
       description: >
-        Please search [issues](https://github.com/apache/incubator-paimon/issues) to check if your issue has already been reported.
+        Please search [issues](https://github.com/apache/paimon/issues) to check if your issue has already been reported.
       options:
         - label: >
-            I searched in the [issues](https://github.com/apache/incubator-paimon/issues) and found nothing similar.
+            I searched in the [issues](https://github.com/apache/paimon/issues) and found nothing similar.
           required: true
   - type: textarea
     attributes:

diff --git a/.github/ISSUE_TEMPLATE/feature.yml b/.github/ISSUE_TEMPLATE/feature.yml
@@ -29,10 +29,10 @@ body:
     attributes:
       label: Search before asking
       description: >
-        Please search [issues](https://github.com/apache/incubator-paimon/issues) to check if your issue has already been reported.
+        Please search [issues](https://github.com/apache/paimon/issues) to check if your issue has already been reported.
       options:
         - label: >
-            I searched in the [issues](https://github.com/apache/incubator-paimon/issues) and found nothing similar.
+            I searched in the [issues](https://github.com/apache/paimon/issues) and found nothing similar.
           required: true
   - type: textarea
     attributes:

diff --git a/.github/workflows/e2e-tests-1.18-jdk11.yml b/.github/workflows/e2e-tests-1.18-jdk11.yml
@@ -46,13 +46,13 @@ jobs:
           distribution: 'adopt'
       - name: Build Flink 1.18
         run: mvn -T 1C -B clean install -DskipTests
-      - name: Test Flink 1.17
+      - name: Test Flink 1.18
         timeout-minutes: 60
         run: |
           # run tests with random timezone to find out timezone related bugs
           . .github/workflows/utils.sh
           jvm_timezone=$(random_timezone)
           echo "JVM timezone is set to $jvm_timezone"
-          mvn -T 1C -B test -pl paimon-e2e-tests -Duser.timezone=$jvm_timezone
+          mvn -T 1C -B test -pl paimon-e2e-tests -Duser.timezone=$jvm_timezone -Pflink-1.18
         env:
           MAVEN_OPTS: -Xmx4096m
diff --git a/.github/workflows/e2e-tests-1.18.yml b/.github/workflows/e2e-tests-1.18.yml
@@ -52,6 +52,6 @@ jobs:
           . .github/workflows/utils.sh
           jvm_timezone=$(random_timezone)
           echo "JVM timezone is set to $jvm_timezone"
-          mvn -T 1C -B test -pl paimon-e2e-tests -Duser.timezone=$jvm_timezone
+          mvn -T 1C -B test -pl paimon-e2e-tests -Duser.timezone=$jvm_timezone -Pflink-1.18
         env:
           MAVEN_OPTS: -Xmx4096m
diff --git a/.github/workflows/e2e-tests-1.19-jdk11.yml b/.github/workflows/e2e-tests-1.19-jdk11.yml
@@ -0,0 +1,58 @@
+################################################################################
+#  Licensed to the Apache Software Foundation (ASF) under one
+#  or more contributor license agreements.  See the NOTICE file
+#  distributed with this work for additional information
+#  regarding copyright ownership.  The ASF licenses this file
+#  to you under the Apache License, Version 2.0 (the
+#  "License"); you may not use this file except in compliance
+#  with the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+name: End to End Tests Flink 1.19 on JDK 11
+
+on:
+  issue_comment:
+    types: [created, edited, deleted]
+
+  # daily run
+  schedule:
+    - cron: "0 0 * * *"
+
+env:
+  JDK_VERSION: 11
+
+jobs:
+  build:
+    if: |
+      github.event_name == 'schedule' ||
+      (contains(github.event.comment.html_url, '/pull/') && contains(github.event.comment.body, '/jdk11'))
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+      - name: Set up JDK ${{ env.JDK_VERSION }}
+        uses: actions/setup-java@v2
+        with:
+          java-version: ${{ env.JDK_VERSION }}
+          distribution: 'adopt'
+      - name: Build Flink 1.19
+        run: mvn -T 1C -B clean install -DskipTests
+      - name: Test Flink 1.19
+        timeout-minutes: 60
+        run: |
+          # run tests with random timezone to find out timezone related bugs
+          . .github/workflows/utils.sh
+          jvm_timezone=$(random_timezone)
+          echo "JVM timezone is set to $jvm_timezone"
+          mvn -T 1C -B test -pl paimon-e2e-tests -Duser.timezone=$jvm_timezone
+        env:
+          MAVEN_OPTS: -Xmx4096m
diff --git a/.github/workflows/e2e-tests-1.19.yml b/.github/workflows/e2e-tests-1.19.yml
@@ -0,0 +1,57 @@
+################################################################################
+#  Licensed to the Apache Software Foundation (ASF) under one
+#  or more contributor license agreements.  See the NOTICE file
+#  distributed with this work for additional information
+#  regarding copyright ownership.  The ASF licenses this file
+#  to you under the Apache License, Version 2.0 (the
+#  "License"); you may not use this file except in compliance
+#  with the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+name: End to End Tests Flink 1.19
+
+on:
+  push:
+  pull_request:
+    paths-ignore:
+      - 'docs/**'
+      - '**/*.md'
+
+env:
+  JDK_VERSION: 8
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.number || github.run_id }}
+  cancel-in-progress: true
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+      - name: Set up JDK ${{ env.JDK_VERSION }}
+        uses: actions/setup-java@v2
+        with:
+          java-version: ${{ env.JDK_VERSION }}
+          distribution: 'adopt'
+      - name: Build Flink 1.19
+        run: mvn -T 1C -B clean install -DskipTests
+      - name: Test Flink 1.19
+        timeout-minutes: 60
+        run: |
+          # run tests with random timezone to find out timezone related bugs
+          . .github/workflows/utils.sh
+          jvm_timezone=$(random_timezone)
+          echo "JVM timezone is set to $jvm_timezone"
+          mvn -T 1C -B test -pl paimon-e2e-tests -Duser.timezone=$jvm_timezone
+        env:
+          MAVEN_OPTS: -Xmx4096m
diff --git a/.github/workflows/publish_snapshot.yml b/.github/workflows/publish_snapshot.yml
@@ -33,7 +33,7 @@ concurrency:
 
 jobs:
   publish-snapshot:
-    if: github.repository == 'apache/incubator-paimon'
+    if: github.repository == 'apache/paimon'
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code

diff --git a/.github/workflows/unitcase-flink-jdk11.yml b/.github/workflows/unitcase-flink-jdk11.yml
@@ -52,6 +52,11 @@ jobs:
           . .github/workflows/utils.sh
           jvm_timezone=$(random_timezone)
           echo "JVM timezone is set to $jvm_timezone"
-          mvn -T 1C -B clean install -pl 'org.apache.paimon:paimon-flink-common' -Duser.timezone=$jvm_timezone
+          test_modules=""
+          for suffix in 1.15 1.16 1.17 1.18 1.19 common; do
+          test_modules+="org.apache.paimon:paimon-flink-${suffix},"
+          done
+          test_modules="${test_modules%,}"
+          mvn -T 1C -B clean install -pl "${test_modules}" -Duser.timezone=$jvm_timezone
         env:
           MAVEN_OPTS: -Xmx4096m
diff --git a/.github/workflows/utitcase-flink.yml b/.github/workflows/utitcase-flink.yml
@@ -52,7 +52,7 @@ jobs:
           jvm_timezone=$(random_timezone)
           echo "JVM timezone is set to $jvm_timezone"
           test_modules=""
-          for suffix in 1.15 1.16 1.17 1.18 common; do
+          for suffix in 1.15 1.16 1.17 1.18 1.19 common; do
             test_modules+="org.apache.paimon:paimon-flink-${suffix}," 
           done
           test_modules="${test_modules%,}"

diff --git a/.mvn/readme.txt b/.mvn/readme.txt
@@ -0,0 +1,4 @@
+The .mvn directory is needed to be able to use the ${maven.multiModuleProjectDirectory} property, since git cannot
+commit an empty directory, add this file.
+
+Once we do not use ${maven.multiModuleProjectDirectory}, we can remove this file and .mvn directory.
diff --git a/DISCLAIMER b/DISCLAIMER
diff --git a/LICENSE b/LICENSE
@@ -255,6 +255,14 @@ from http://iceberg.apache.org/ version 1.3.0
 paimon-hive/paimon-hive-common/src/test/resources/hive-schema-3.1.0.derby.sql
 from https://hive.apache.org/ version 3.1.0
 
+paimon-format/src/main/java/org/apache/orc/impl/PhysicalFsWriter.java
+paimon-format/src/main/java/org/apache/orc/impl/WriterImpl.java
+paimon-format/src/main/java/org/apache/orc/impl/ZstdCodec.java
+paimon-format/src/main/java/org/apache/orc/CompressionKind.java
+paimon-format/src/main/java/org/apache/orc/OrcConf.java
+paimon-format/src/main/java/org/apache/orc/OrcFile.java
+from https://orc.apache.org/ version 2.0
+
 MIT License
 -----------
 

diff --git a/NOTICE b/NOTICE
@@ -1,4 +1,4 @@
-Apache Paimon (incubating)
+Apache Paimon
 Copyright 2023-2024 The Apache Software Foundation
 
 This product includes software developed at

diff --git a/README.md b/README.md
@@ -3,7 +3,7 @@
 [![License](https://img.shields.io/badge/license-Apache%202-4EB1BA.svg)](https://www.apache.org/licenses/LICENSE-2.0.html)
 [![Get on Slack](https://img.shields.io/badge/slack-join-orange.svg)](https://the-asf.slack.com/archives/C053Q2NCW8G)
 
-Apache Paimon(incubating) is a lake format that enables building a Realtime Lakehouse Architecture with Flink and Spark 
+Apache Paimon is a lake format that enables building a Realtime Lakehouse Architecture with Flink and Spark 
 for both streaming and batch operations. Paimon innovatively combines lake format and LSM structure, bringing realtime 
 streaming updates into the lake architecture.
 
@@ -68,7 +68,7 @@ You can join the Paimon community on Slack. Paimon channel is in ASF Slack works
 
 ## Building
 
-JDK 8/11 is required for building the project.
+JDK 8/11 is required for building the project. Maven version >=3.3.1.
 
 - Run the `mvn clean install -DskipTests` command to build the project.
 - Run the `mvn spotless:apply` to format the project (both Java and Scala).

diff --git a/docs/config.toml b/docs/config.toml
@@ -44,13 +44,13 @@ pygmentsUseClasses = true
   Branch = "master"
 
   # The github repository for Apache Paimon
-  Repo = "//github.com/apache/incubator-paimon"
+  Repo = "//github.com/apache/paimon"
 
-  GithubRepo = "https://github.com/apache/incubator-paimon.git"
+  GithubRepo = "https://github.com/apache/paimon.git"
 
-  TrinoGithubRepo = "https://github.com/apache/incubator-paimon-trino.git"
+  TrinoGithubRepo = "https://github.com/apache/paimon-trino.git"
 
-  PrestoGithubRepo = "https://github.com/apache/incubator-paimon-presto.git"
+  PrestoGithubRepo = "https://github.com/apache/paimon-presto.git"
 
   # This suffix is appended to the Scala-dependent Maven artifact names
   ScalaVersion = "_2.12"

diff --git a/docs/content/_index.md b/docs/content/_index.md
@@ -24,7 +24,7 @@ under the License.
 
 # Apache Paimon
 
-Apache Paimon(incubating) is a lake format that enables building a Realtime Lakehouse Architecture with Flink and Spark 
+Apache Paimon is a lake format that enables building a Realtime Lakehouse Architecture with Flink and Spark 
 for both streaming and batch operations. Paimon innovatively combines lake format and LSM (Log-structured merge-tree) 
 structure, bringing realtime streaming updates into the lake architecture.
 

diff --git a/docs/content/concepts/overview.md b/docs/content/concepts/overview.md
@@ -26,7 +26,7 @@ under the License.
 
 # Overview
 
-Apache Paimon(incubating)'s Architecture:
+Apache Paimon's Architecture:
 
 {{< img src="/img/architecture.png">}}