[Optimization-2694][flink] Upgrade Flink and Flink CDC (#2695)

Co-authored-by: wenmo <[email protected]>
DataLinkDC · Dec 19, 2023 · 177330d · 177330d
1 parent 19d921f
commit 177330d
Show file tree

Hide file tree

Showing 10 changed files with 22 additions and 17 deletions.
diff --git a/dinky-admin/src/main/resources/mapper/JobInstanceMapper.xml b/dinky-admin/src/main/resources/mapper/JobInstanceMapper.xml
@@ -79,7 +79,6 @@
                  from (select max(ji.id) as id from dinky_job_instance ji group by ji.task_id) as a
         left join dinky_job_instance dji on dji.id = a.id
         left join dinky_history dh on dji.history_id = dh.id) b
-        group by b.batch;
     </select>
 
     <select id="listJobInstanceActive" resultType="org.dinky.data.model.job.JobInstance">

diff --git a/dinky-cdc/dinky-cdc-core/src/main/java/org/dinky/cdc/sqlserver/SqlServerCDCBuilder.java b/dinky-cdc/dinky-cdc-core/src/main/java/org/dinky/cdc/sqlserver/SqlServerCDCBuilder.java
@@ -34,8 +34,8 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import com.ververica.cdc.connectors.base.options.StartupOptions;
 import com.ververica.cdc.connectors.sqlserver.SqlServerSource;
-import com.ververica.cdc.connectors.sqlserver.table.StartupOptions;
 import com.ververica.cdc.debezium.JsonDebeziumDeserializationSchema;
 
 public class SqlServerCDCBuilder extends AbstractCDCBuilder implements CDCBuilder {

diff --git a/dinky-cdc/pom.xml b/dinky-cdc/pom.xml
@@ -47,5 +47,11 @@
                 <module>dinky-cdc-plus</module>
             </modules>
         </profile>
+        <profile>
+            <id>flink-1.18</id>
+            <modules>
+                <module>dinky-cdc-plus</module>
+            </modules>
+        </profile>
     </profiles>
 </project>
diff --git a/dinky-flink/dinky-flink-1.14/pom.xml b/dinky-flink/dinky-flink-1.14/pom.xml
@@ -34,7 +34,7 @@
         <commons.version>1.3.1</commons.version>
         <flink.guava.version>14.0</flink.guava.version>
         <flink.version>1.14.6</flink.version>
-        <flinkcdc.version>2.3.0</flinkcdc.version>
+        <flinkcdc.version>3.0.0</flinkcdc.version>
         <junit.version>4.12</junit.version>
     </properties>
 

diff --git a/dinky-flink/dinky-flink-1.15/pom.xml b/dinky-flink/dinky-flink-1.15/pom.xml
@@ -33,8 +33,8 @@
     <properties>
         <commons.version>1.3.1</commons.version>
         <flink.guava.version>15.0</flink.guava.version>
-        <flink.version>1.15.2</flink.version>
-        <flinkcdc.version>2.3.0</flinkcdc.version>
+        <flink.version>1.15.4</flink.version>
+        <flinkcdc.version>3.0.0</flinkcdc.version>
         <junit.version>4.12</junit.version>
     </properties>
 

diff --git a/dinky-flink/dinky-flink-1.16/pom.xml b/dinky-flink/dinky-flink-1.16/pom.xml
@@ -17,8 +17,8 @@
     <properties>
         <commons.version>1.3.1</commons.version>
         <flink.guava.version>16.0</flink.guava.version>
-        <flink.version>1.16.2</flink.version>
-        <flinkcdc.version>2.3.0</flinkcdc.version>
+        <flink.version>1.16.3</flink.version>
+        <flinkcdc.version>3.0.0</flinkcdc.version>
     </properties>
 
     <dependencies>

diff --git a/dinky-flink/dinky-flink-1.17/pom.xml b/dinky-flink/dinky-flink-1.17/pom.xml
@@ -16,9 +16,9 @@
 
     <properties>
         <commons.version>1.3.1</commons.version>
-        <flink.guava.version>17.0</flink.guava.version>
-        <flink.version>1.17.1</flink.version>
-        <flinkcdc.version>2.3.0</flinkcdc.version>
+        <flink.shaded.version>16.1</flink.shaded.version>
+        <flink.version>1.17.2</flink.version>
+        <flinkcdc.version>3.0.0</flinkcdc.version>
     </properties>
 
     <dependencies>
@@ -77,7 +77,7 @@
         <dependency>
             <groupId>org.apache.flink</groupId>
             <artifactId>flink-shaded-guava</artifactId>
-            <version>31.1-jre-${flink.guava.version}</version>
+            <version>30.1.1-jre-${flink.shaded.version}</version>
         </dependency>
         <dependency>
             <groupId>com.ververica</groupId>

diff --git a/dinky-flink/dinky-flink-1.18/pom.xml b/dinky-flink/dinky-flink-1.18/pom.xml
@@ -16,9 +16,9 @@
 
     <properties>
         <commons.version>1.3.1</commons.version>
-        <flink.guava.version>17.0</flink.guava.version>
+        <flink.shaded.version>17.0</flink.shaded.version>
         <flink.version>1.18.0</flink.version>
-        <flinkcdc.version>2.3.0</flinkcdc.version>
+        <flinkcdc.version>3.0.0</flinkcdc.version>
     </properties>
 
     <dependencies>
@@ -77,7 +77,7 @@
         <dependency>
             <groupId>org.apache.flink</groupId>
             <artifactId>flink-shaded-guava</artifactId>
-            <version>31.1-jre-${flink.guava.version}</version>
+            <version>31.1-jre-${flink.shaded.version}</version>
         </dependency>
         <dependency>
             <groupId>com.ververica</groupId>

diff --git a/pom.xml b/pom.xml
@@ -57,12 +57,12 @@
         <!--    jexl cannot be upgraded to 3.3, and static methods need to be used.(jexl不能升级到3.3，需要使用到静态方法的使用)    -->
         <commons-jexl3>3.2.1</commons-jexl3>
         <commons-lang3.version>3.3.2</commons-lang3.version>
-        <dinky.flink.version>1.14</dinky.flink.version>
+        <dinky.flink.version>1.18</dinky.flink.version>
         <docker.java.version>3.2.14</docker.java.version>
         <druid-starter>1.2.8</druid-starter>
         <easy.rules.version>4.1.0</easy.rules.version>
         <fastjson2.version>2.0.41</fastjson2.version>
-        <flink.version>1.14.6</flink.version>
+        <flink.version>1.18.0</flink.version>
         <guava.version>31.1-jre</guava.version>
         <h2database.version>1.4.200</h2database.version>
         <hamcrest.version>1.3</hamcrest.version>

diff --git a/script/bin/auto.sh b/script/bin/auto.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 
-FLINK_VERSION=${2:-1.14}
+FLINK_VERSION=${2:-1.18}
 
 JAR_NAME="dinky-admin"