[Feature-3332][Flink] Support Flink 1.19 (DataLinkDC#3414)

Co-authored-by: wenmo <[email protected]>
Zzm0809 · Apr 19, 2024 · 26c0f9d · 26c0f9d
1 parent b099de0
commit 26c0f9d
Show file tree

Hide file tree

Showing 58 changed files with 5,182 additions and 17 deletions.
diff --git a/.github/workflows/backend.yaml b/.github/workflows/backend.yaml
@@ -89,7 +89,7 @@ jobs:
       fail-fast: true
       matrix:
         jdk: [8, 11]
-        flink: [1.14, 1.15, 1.16, 1.17, 1.18]
+        flink: [1.14, 1.15, 1.16, 1.17, 1.18, 1.19]
 
     timeout-minutes: 30
     env:

diff --git a/.github/workflows/docker_build.yaml b/.github/workflows/docker_build.yaml
@@ -153,7 +153,7 @@ jobs:
       fail-fast: true
       matrix:
         url: [ registry.cn-hangzhou.aliyuncs.com ,docker.io ]
-        FLINK_VERSION: [1.14.6 , 1.15.4 , 1.16.3, 1.17.2, 1.18.1]
+        FLINK_VERSION: [1.14.6 , 1.15.4 , 1.16.3, 1.17.2, 1.18.1, 1.19.0]
         include:
           - url: registry.cn-hangzhou.aliyuncs.com
             namespace: dinky
@@ -169,6 +169,8 @@ jobs:
             FLINK_BIG_VERSION: 1.17
           - FLINK_VERSION: 1.18.1
             FLINK_BIG_VERSION: 1.18
+          - FLINK_VERSION: 1.19.0
+            FLINK_BIG_VERSION: 1.19
     steps:
       - uses: actions/checkout@v3
       - name: Move Dockerfile

diff --git a/dinky-app/dinky-app-1.19/pom.xml b/dinky-app/dinky-app-1.19/pom.xml
@@ -0,0 +1,76 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+        <groupId>org.dinky</groupId>
+        <artifactId>dinky-app</artifactId>
+        <version>${revision}</version>
+        <relativePath>../pom.xml</relativePath>
+    </parent>
+    <artifactId>dinky-app-1.19</artifactId>
+
+    <packaging>jar</packaging>
+
+    <name>Dinky : App 1.19</name>
+
+    <properties>
+        <mainClass>org.dinky.app.MainApp</mainClass>
+    </properties>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.dinky</groupId>
+            <artifactId>dinky-app-base</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.dinky</groupId>
+            <artifactId>dinky-client-1.19</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.dinky</groupId>
+            <artifactId>dinky-flink-1.19</artifactId>
+            <scope>${scope.runtime}</scope>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <resources>
+            <resource>
+                <directory>src/main/resources</directory>
+                <includes>
+                    <include>*.properties</include>
+                </includes>
+            </resource>
+        </resources>
+
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-assembly-plugin</artifactId>
+                <version>${maven-assembly-plugin.version}</version>
+                <configuration>
+                    <descriptorRefs>
+                        <descriptorRef>jar-with-dependencies</descriptorRef>
+                    </descriptorRefs>
+                    <archive>
+                        <manifest>
+                            <!-- 可以设置jar包的入口类(可选) -->
+                            <mainClass>org.dinky.app.MainApp</mainClass>
+                        </manifest>
+                    </archive>
+                    <outputDirectory>${project.parent.parent.basedir}/build/extends</outputDirectory>
+                </configuration>
+                <executions>
+                    <execution>
+                        <id>make-assembly</id>
+                        <goals>
+                            <goal>single</goal>
+                        </goals>
+                        <phase>package</phase>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+    </build>
+</project>
diff --git a/dinky-app/dinky-app-1.19/src/main/java/org/dinky/app/MainApp.java b/dinky-app/dinky-app-1.19/src/main/java/org/dinky/app/MainApp.java
@@ -0,0 +1,55 @@
+/*
+ *
+ *  Licensed to the Apache Software Foundation (ASF) under one or more
+ *  contributor license agreements.  See the NOTICE file distributed with
+ *  this work for additional information regarding copyright ownership.
+ *  The ASF licenses this file to You under the Apache License, Version 2.0
+ *  (the "License"); you may not use this file except in compliance with
+ *  the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ */
+
+package org.dinky.app;
+
+import org.dinky.app.constant.AppParamConstant;
+import org.dinky.app.db.DBUtil;
+import org.dinky.app.flinksql.Submitter;
+import org.dinky.data.app.AppParamConfig;
+import org.dinky.utils.JsonUtils;
+
+import org.apache.flink.api.java.utils.ParameterTool;
+
+import java.util.Base64;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * MainApp
+ *
+ * @since 2022/11/05
+ */
+public class MainApp {
+
+    private static final Logger log = LoggerFactory.getLogger(Submitter.class);
+
+    public static void main(String[] args) throws Exception {
+        log.info("=========================Start run dinky app job===============================");
+        ParameterTool parameters = ParameterTool.fromArgs(args);
+        boolean isEncrypt = parameters.getBoolean(AppParamConstant.isEncrypt, true);
+        String config = parameters.get(AppParamConstant.config);
+        config = isEncrypt ? new String(Base64.getDecoder().decode(config)) : config;
+        AppParamConfig appConfig = JsonUtils.toJavaBean(config, AppParamConfig.class);
+        log.info("dinky app is Ready to run, config is {}", appConfig);
+        DBUtil.init(appConfig);
+        Submitter.submit(appConfig);
+    }
+}
diff --git a/dinky-app/dinky-app-base/pom.xml b/dinky-app/dinky-app-base/pom.xml
@@ -303,6 +303,17 @@
                     <artifactId>dinky-flink-1.18</artifactId>
                     <scope>${scope.runtime}</scope>
                 </dependency>
+
+                <dependency>
+                    <groupId>org.dinky</groupId>
+                    <artifactId>dinky-client-1.19</artifactId>
+                    <scope>${scope.runtime}</scope>
+                </dependency>
+                <dependency>
+                    <groupId>org.dinky</groupId>
+                    <artifactId>dinky-flink-1.19</artifactId>
+                    <scope>${scope.runtime}</scope>
+                </dependency>
             </dependencies>
         </profile>
     </profiles>

diff --git a/dinky-app/pom.xml b/dinky-app/pom.xml
@@ -32,6 +32,7 @@
                 <module>dinky-app-1.16</module>
                 <module>dinky-app-1.17</module>
                 <module>dinky-app-1.18</module>
+                <module>dinky-app-1.19</module>
             </modules>
         </profile>
 

diff --git a/dinky-assembly/src/main/assembly/package.xml b/dinky-assembly/src/main/assembly/package.xml
@@ -108,6 +108,14 @@
                 <include>dinky-client-1.18-${project.version}.jar</include>
             </includes>
         </fileSet>
+        <fileSet>
+            <directory>${project.parent.basedir}/build/extends/</directory>
+            <outputDirectory>extends/flink1.19/dinky</outputDirectory>
+            <includes>
+                <include>dinky-catalog-mysql-1.19-${project.version}.jar</include>
+                <include>dinky-client-1.19-${project.version}.jar</include>
+            </includes>
+        </fileSet>
         <fileSet>
             <directory>${project.parent.basedir}/build/extends/</directory>
             <outputDirectory>extends</outputDirectory>

diff --git a/dinky-catalog/dinky-catalog-mysql/dinky-catalog-mysql-1.19/pom.xml b/dinky-catalog/dinky-catalog-mysql/dinky-catalog-mysql-1.19/pom.xml
@@ -0,0 +1,46 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+        <groupId>org.dinky</groupId>
+        <artifactId>dinky-catalog-mysql</artifactId>
+        <version>${revision}</version>
+        <relativePath>../pom.xml</relativePath>
+    </parent>
+    <artifactId>dinky-catalog-mysql-1.19</artifactId>
+
+    <packaging>jar</packaging>
+
+    <name>Dinky : Catalog : Mysql 1.19</name>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.dinky</groupId>
+            <artifactId>dinky-common</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.dinky</groupId>
+            <artifactId>dinky-flink-1.19</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>junit</groupId>
+            <artifactId>junit</artifactId>
+            <version>4.13.2</version>
+            <scope>test</scope>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-jar-plugin</artifactId>
+                <configuration>
+                    <!-- 指定打包的jar包输出路径 -->
+                    <outputDirectory>${project.parent.parent.parent.basedir}/build/extends</outputDirectory>
+                </configuration>
+            </plugin>
+        </plugins>
+    </build>
+</project>