Release 1.2.0 (LTS)

.github/workflows/e2e_test.yml

@@ -0,0 +1,83 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+name: E2E Test
+
+on:
+  workflow_dispatch:
+
+jobs:
+  init_env:
+    name: init env
+    runs-on: ubuntu-latest
+    services:
+      registry:
+        image: registry:2
+        ports:
+          - 5000:5000
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Init  Env Jar
+        run: |
+          wget -O e2e_test/docker-compose-env/dinky/mysql-connector-java-8.0.30.jar  https://repo1.maven.org/maven2/mysql/mysql-connector-java/8.0.30/mysql-connector-java-8.0.30.jar &&
+          wget -O e2e_test/docker-compose-env/flink/flink-shaded-hadoop-2-uber-2.8.3-10.0.jar  https://repo1.maven.org/maven2/org/apache/flink/flink-shaded-hadoop-2-uber/2.8.3-10.0/flink-shaded-hadoop-2-uber-2.8.3-10.0.jar
+      - name: Init  Docker Network
+        run: |
+          docker network create -d bridge dinky_net
+      - name: Init Run Docker MySQL
+        uses: hoverkraft-tech/[email protected]
+        with:
+          compose-file: ./e2e_test/docker-compose-env/mysql/docker-compose.yml
+      - name: Init Run Docker Hadoop
+        uses: hoverkraft-tech/[email protected]
+        with:
+          compose-file: ./e2e_test/docker-compose-env/hadoop/docker-compose.yml
+      # 设置 QEMU, 后面 docker buildx 依赖此.
+      - name: Init Run Docker Flink
+        uses: hoverkraft-tech/[email protected]
+        with:
+          compose-file: ./e2e_test/docker-compose-env/flink/docker-compose.yml
+      # 设置 QEMU, 后面 docker buildx 依赖此.
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+      # 设置 Docker buildx, 方便构建 Multi platform 镜像
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+        with:
+          driver-opts: network=host
+      - name: Build Dinky Image
+        uses: docker/build-push-action@v5
+        with:
+          file: ./deploy/docker/Dockerfile
+          # 是否 docker push
+          push: true
+          build-args: |
+            FLINK_VERSION=1.14
+            DINKY_VERSION=1
+          tags: |
+            localhost:5000/dinky/dinky-test:flink-1.14
+      - name: Init Run Docker Dinky
+        uses: hoverkraft-tech/[email protected]
+        with:
+          compose-file: ./e2e_test/docker-compose-env/dinky/docker-compose.yml
+      - name:  Run Docker Python Script
+#        uses: hoverkraft-tech/[email protected]
+#        with:
+#          compose-file: ./e2e_test/tools/docker-compose.yml
+        run: |
+          sleep 30 && cd ./e2e_test/tools && docker run  -v ./:/app -w /app   --net dinky_net --rm --entrypoint /bin/bash python:3.9 -c  'pip install  -r requirements.txt  &&  python main.py dinky14:8888'

.gitignore

@@ -30,7 +30,6 @@ dinky.db
 hs_err_pid*
 .idea/*
 !.idea/icon.svg
-!.idea/vcs.xml
 build
 target/*
 *.iml
@@ -63,4 +62,6 @@ tmp/*
 extends/*
 /.run/
 
-.idea
+.idea
+.idea/vcs.xml
+dinky-web/package-lock.json

README.md

@@ -62,7 +62,7 @@ Thank you to all the people who already contributed to Dinky!
 
 ## How to Deploy
 
-See [source code compilation](https://github.com/DataLinkDC/dinky/blob/dev/docs/docs/deploy_guide/compile.mdx) and [installation and deployment](https://github.com/DataLinkDC/dinky/blob/dev/docs/docs/deploy_guide/normal_deploy.mdx) for details.
+See [source code compilation](https://github.com/DataLinkDC/dinky/blob/dev/docs/docs/deploy_guide/compile_deploy.md) and [installation and deployment](https://github.com/DataLinkDC/dinky/blob/dev/docs/docs/deploy_guide/normal_deploy.mdx) for details.
 
 ## Thanks
 

README_zh_CN.md

@@ -66,7 +66,7 @@
 
 ## 如何部署
 
-详见 [源码编译](https://github.com/DataLinkDC/dinky/blob/dev/docs/docs/deploy_guide/compile.mdx) 和 [安装部署](https://github.com/DataLinkDC/dinky/blob/dev/docs/docs/deploy_guide/normal_deploy.mdx) 。
+详见 [源码编译](https://github.com/DataLinkDC/dinky/blob/dev/docs/docs/deploy_guide/compile_deploy.md) 和 [安装部署](https://github.com/DataLinkDC/dinky/blob/dev/docs/docs/deploy_guide/normal_deploy.mdx) 。
 
 ## 感谢
 

deploy/docker/Dockerfile

@@ -27,7 +27,7 @@ COPY --from=ui-build /build/dist/ /build/dinky-web/dist/
 
 RUN mvn package -Dmaven.test.skip=true -P prod,flink-single-version,flink-${FLINK_VERSION},fast
 RUN mkdir release &&  \
-    tar -C release -xvf build/dinky-release-${FLINK_VERSION}-${DINKY_VERSION}.tar.gz && \
+    tar -C release -xvf build/dinky-release-${FLINK_VERSION}-*.tar.gz && \
     mv release/dinky-release-* release/dinky
 
 
@@ -48,8 +48,8 @@ RUN rm -f  /opt/dinky/extends/flink${FLINK_VERSION}/flink/flink-table-planner-lo
 
 COPY --from=flink-base  /opt/flink/opt/flink-table-planner*.jar /opt/dinky/extends/flink${FLINK_VERSION}/flink/
 
-RUN mkdir /opt/dinky/customJar && chmod -R 777 /opt/dinky/ && sed -i 's/-Xms512M -Xmx2048M -XX:PermSize=512M/-XX:+UseContainerSupport -XX:InitialRAMPercentage=70.0 -XX:MaxRAMPercentage=70.0/g' auto.sh
+RUN mkdir /opt/dinky/customJar && chmod -R 777 /opt/dinky/ && sed -i 's/-Xms512M -Xmx2048M -XX:PermSize=512M/-XX:+UseContainerSupport -XX:InitialRAMPercentage=70.0 -XX:MaxRAMPercentage=70.0/g' ./bin/auto.sh
 
 EXPOSE 8888
 
-CMD  ./auto.sh startOnPending
+CMD  ./bin/auto.sh startOnPending

dinky-admin/pom.xml

@@ -359,10 +359,26 @@
             <groupId>org.dinky</groupId>
             <artifactId>dinky-alert-dingtalk</artifactId>
         </dependency>
+        <dependency>
+            <groupId>org.dinky</groupId>
+            <artifactId>dinky-alert-feishu</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.dinky</groupId>
+            <artifactId>dinky-alert-wechat</artifactId>
+        </dependency>
         <dependency>
             <groupId>org.dinky</groupId>
             <artifactId>dinky-alert-sms</artifactId>
         </dependency>
+        <dependency>
+            <groupId>org.dinky</groupId>
+            <artifactId>dinky-alert-email</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.dinky</groupId>
+            <artifactId>dinky-alert-http</artifactId>
+        </dependency>
     </dependencies>
 
     <build>

dinky-admin/src/main/assembly/package.xml

@@ -16,7 +16,7 @@
         <fileSet>
             <directory>${basedir}/src/main/bin</directory>
             <lineEnding>unix</lineEnding>
-            <outputDirectory/>
+            <outputDirectory>bin</outputDirectory>
             <fileMode>755</fileMode>
             <includes>
                 <include>*.sh</include>

dinky-admin/src/main/java/org/dinky/Dinky.java

@@ -19,6 +19,9 @@
 
 package org.dinky;
 
+import org.dinky.data.constant.DirConstant;
+import org.dinky.security.NoExitSecurityManager;
+
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
 import org.springframework.boot.autoconfigure.freemarker.FreeMarkerAutoConfiguration;
@@ -27,6 +30,7 @@
 
 import com.alibaba.druid.proxy.DruidDriver;
 
+import cn.hutool.core.io.FileUtil;
 import lombok.SneakyThrows;
 
 /**
@@ -45,11 +49,15 @@ public class Dinky {
 
     @SneakyThrows
     public static void main(String[] args) {
+        // Prevent System.exit calls
+        System.setSecurityManager(new NoExitSecurityManager());
         // Initialize the JDBC Driver, because the number of packages is very large, so it needs to be executed
         // asynchronously and loaded in advance
         // chinese: 初始化JDBC Driver，因为包的数量特别庞大，所以这里需要异步执行，并提前加载Driver
         new Thread(DruidDriver::getInstance).start();
 
+        FileUtil.mkdir(DirConstant.getTempRootDir());
+
         SpringApplication app = new SpringApplication(Dinky.class);
         app.run(args);
     }

dinky-admin/src/main/java/org/dinky/aop/exception/WebExceptionHandler.java

@@ -61,8 +61,8 @@ public class WebExceptionHandler {
     @ExceptionHandler
     @ResponseBody
     public Result<Void> busException(BusException e) {
-        log.error("BusException:", e);
-        return Result.failed(e.getMsg());
+        log.error(e.getMessage(), e);
+        return Result.failed(e.getMessage());
     }
 
     private static final Map<String, Status> ERR_CODE_MAPPING = MapUtil.<String, Status>builder()

dinky-admin/src/main/java/org/dinky/configure/AppConfig.java

@@ -86,11 +86,16 @@ public void addInterceptors(InterceptorRegistry registry) {
                 }))
                 .addPathPatterns("/api/**", "/openapi/**")
                 .excludePathPatterns(
-                        "/api/login", "/api/ldap/ldapEnableStatus", "/download/**", "/druid/**", "/api/version");
+                        "/api/login",
+                        "/api/sysConfig/getNeededCfg",
+                        "/api/sysConfig/setInitConfig",
+                        "/download/**",
+                        "/druid/**",
+                        "/api/version");
 
         registry.addInterceptor(new TenantInterceptor())
                 .addPathPatterns("/api/**")
-                .excludePathPatterns("/api/login", "/api/ldap/ldapEnableStatus")
+                .excludePathPatterns("/api/login", "/api/sysConfig/getNeededCfg", "/api/sysConfig/setInitConfig")
                 .addPathPatterns("/api/alertGroup/**")
                 .addPathPatterns("/api/alertHistory/**")
                 .addPathPatterns("/api/alertInstance/**")

dinky-admin/src/main/java/org/dinky/configure/MybatisPlusConfig.java

@@ -80,7 +80,7 @@ public class MybatisPlusConfig {
             "dinky_task_version");
 
     @Bean
-    @Profile("pgsql")
+    @Profile("postgresql")
     public PostgreSQLQueryInterceptor postgreSQLQueryInterceptor() {
         return new PostgreSQLQueryInterceptor();
     }
@@ -91,7 +91,7 @@ public PostgreSQLQueryInterceptor postgreSQLQueryInterceptor() {
      * @return {@linkplain PostgreSQLPrepareInterceptor}
      */
     @Bean
-    @Profile("pgsql")
+    @Profile("postgresql")
     public PostgreSQLPrepareInterceptor postgreSQLPrepareInterceptor() {
         return new PostgreSQLPrepareInterceptor();
     }

dinky-admin/src/main/java/org/dinky/context/ConsoleContextHolder.java

@@ -121,7 +121,7 @@ public boolean clearProcessLog(String processName) {
         if (FileUtil.exist(filePath)) {
             return FileUtil.del(filePath);
         }
-        return false;
+        return true;
     }
 
     /**

dinky-admin/src/main/java/org/dinky/controller/CatalogueController.java

@@ -128,6 +128,9 @@ public Result<String> upload(MultipartFile file, @PathVariable Integer id) {
             dataType = "Catalogue",
             dataTypeClass = Catalogue.class)
     public Result<Void> saveOrUpdateCatalogue(@RequestBody Catalogue catalogue) {
+        if (catalogueService.checkNameIsExistByParentId(catalogue)) {
+            return Result.failed(Status.NAME_IS_EXIST);
+        }
         if (catalogueService.saveOrUpdateOrRename(catalogue)) {
             return Result.succeed(Status.SAVE_SUCCESS);
         } else {

dinky-admin/src/main/java/org/dinky/controller/FlinkController.java

@@ -33,6 +33,7 @@
 import org.springframework.web.bind.annotation.RestController;
 
 import cn.dev33.satoken.annotation.SaCheckLogin;
+import cn.hutool.core.lang.Singleton;
 import io.swagger.annotations.Api;
 import io.swagger.annotations.ApiOperation;
 import lombok.RequiredArgsConstructor;
@@ -45,14 +46,13 @@
 @SaCheckLogin
 @RequiredArgsConstructor
 public class FlinkController {
-
-    protected static final CheckpointRead INSTANCE = new CheckpointRead();
     private final FlinkService flinkService;
 
     @GetMapping("/readCheckPoint")
     @ApiOperation("Read Checkpoint")
     public Result<Map<String, Map<String, CheckPointReadTable>>> readCheckPoint(String path, String operatorId) {
-        return Result.data(INSTANCE.readCheckpoint(path, operatorId));
+        CheckpointRead checkpointRead = Singleton.get(CheckpointRead.class);
+        return Result.data(checkpointRead.readCheckpoint(path, operatorId));
     }
 
     @GetMapping("/configOptions")

dinky-admin/src/main/java/org/dinky/controller/LdapController.java

@@ -25,7 +25,6 @@
 import org.dinky.data.enums.BusinessType;
 import org.dinky.data.enums.Status;
 import org.dinky.data.exception.AuthException;
-import org.dinky.data.model.SystemConfiguration;
 import org.dinky.data.model.rbac.User;
 import org.dinky.data.result.Result;
 import org.dinky.service.LdapService;
@@ -43,7 +42,6 @@
 import org.springframework.web.bind.annotation.RestController;
 
 import cn.dev33.satoken.annotation.SaCheckLogin;
-import cn.dev33.satoken.annotation.SaIgnore;
 import io.swagger.annotations.Api;
 import io.swagger.annotations.ApiImplicitParam;
 import io.swagger.annotations.ApiOperation;
@@ -63,13 +61,6 @@ public class LdapController {
     @Autowired
     UserService userService;
 
-    @GetMapping("/ldapEnableStatus")
-    @SaIgnore
-    @ApiOperation("Get LDAP enable status")
-    public Result<Boolean> ldapStatus() {
-        return Result.succeed(SystemConfiguration.getInstances().getLdapEnable().getValue());
-    }
-
     @GetMapping("/testConnection")
     @ApiOperation("Test connection to LDAP server")
     @SaCheckLogin

dinky-admin/src/main/java/org/dinky/controller/SchedulerController.java

@@ -66,9 +66,6 @@ public class SchedulerController {
             example = "1")
     public Result<TaskDefinition> getTaskDefinition(@ApiParam(value = "dinky任务id") @RequestParam Long dinkyTaskId) {
         TaskDefinition taskDefinitionInfo = schedulerService.getTaskDefinitionInfo(dinkyTaskId);
-        if (taskDefinitionInfo == null) {
-            return Result.failed(Status.DS_TASK_NOT_EXIST);
-        }
         return Result.succeed(taskDefinitionInfo);
     }
 
@@ -102,9 +99,9 @@ public Result<String> createOrUpdateTaskDefinition(@RequestBody DinkyTaskRequest
     }
 
     /**
-     * 获取任务组
+     * Get task group
      * @param projectCode
-     * @return
+     * @return Result object containing a list of task groups
      */
     @GetMapping(value = "/task/groups")
     public Result<List<TaskGroup>> getTaskGroups(@RequestParam("projectCode") Long projectCode) {