Merge pull request #102 from DataLinkDC/dev

修复bug
DataLinkDC · Jan 19, 2022 · 60d14f2 · 60d14f2
2 parents 1e3feff + 8161560
commit 60d14f2
Show file tree

Hide file tree

Showing 22 changed files with 334 additions and 127 deletions.
diff --git a/dlink-admin/pom.xml b/dlink-admin/pom.xml
@@ -66,7 +66,7 @@
                 </exclusion>
             </exclusions>
         </dependency>
-        <dependency>
+        <!--<dependency>
             <groupId>org.springframework.boot</groupId>
             <artifactId>spring-boot-starter-log4j2</artifactId>
             <exclusions>
@@ -75,12 +75,19 @@
                     <artifactId>slf4j-api</artifactId>
                 </exclusion>
             </exclusions>
+        </dependency>-->
+        <dependency>
+            <groupId>org.apache.logging.log4j</groupId>
+            <artifactId>log4j-api</artifactId>
         </dependency>
-        <!--<dependency>
+        <dependency>
+            <groupId>org.apache.logging.log4j</groupId>
+            <artifactId>log4j-core</artifactId>
+        </dependency>
+        <dependency>
             <groupId>org.slf4j</groupId>
-            <artifactId>slf4j-api</artifactId>
-            <version>1.7.30</version>
-        </dependency>-->
+            <artifactId>slf4j-log4j12</artifactId>
+        </dependency>
         <dependency>
             <groupId>mysql</groupId>
             <artifactId>mysql-connector-java</artifactId>

diff --git a/dlink-admin/src/main/java/com/dlink/job/Job2MysqlHandler.java b/dlink-admin/src/main/java/com/dlink/job/Job2MysqlHandler.java
@@ -3,6 +3,7 @@
 import cn.hutool.json.JSONUtil;
 import com.dlink.assertion.Asserts;
 import com.dlink.context.SpringContextUtils;
+import com.dlink.gateway.GatewayType;
 import com.dlink.model.Cluster;
 import com.dlink.model.History;
 import com.dlink.service.ClusterService;
@@ -66,7 +67,7 @@ public boolean success() {
         Job job = JobContextHolder.getJob();
         History history = new History();
         history.setId(job.getId());
-        if(Asserts.isNullString(job.getJobId())){
+        if(job.isUseGateway()&&Asserts.isNullString(job.getJobId())){
             job.setJobId("unknown-"+LocalDateTime.now().toString());
         }
         history.setJobId(job.getJobId());

diff --git a/dlink-admin/src/main/resources/log4j2.xml b/dlink-admin/src/main/resources/log4j2.xml
@@ -39,10 +39,4 @@
             <appender-ref ref="RollingFile"/>
         </root>
     </loggers>
-    <!--打印sql日志使用的，additivity是Logger是否继承root中的appender-ref定义的输出位置的标志，默认情况下Logger会继承，若是additivity设为false，则Logger只会在自己标签中的appender-ref里输出，不会在root中的指定的appender-ref里输出，若是additivity设为true，则还会把sql日志输出在root中指定的appender-ref中；level="debug"的原因是Mybatis中SQL语句的日志级别被设为DEBUG，SQL执行结果的日志级别为TRACE，只要将日志级别调整为DEBUG就可以输出SQL语句，而不会输出SQL执行结果-->
-    <logger name="com.dlink.mapper" level="debug" additivity="false">
-        <appender-ref ref="Console"/>
-        <appender-ref ref="RollingFile"/>
-    </logger>
-
 </configuration>
diff --git a/dlink-client/dlink-client-1.11/pom.xml b/dlink-client/dlink-client-1.11/pom.xml
@@ -52,7 +52,7 @@
                     <groupId>org.apache.hadoop</groupId>
                     <artifactId>hadoop-yarn-common</artifactId>
                 </exclusion>
-                <!--<exclusion>
+                <exclusion>
                     <groupId>org.apache.hadoop</groupId>
                     <artifactId>hadoop-common</artifactId>
                 </exclusion>
@@ -67,9 +67,13 @@
                 <exclusion>
                     <groupId>org.apache.hadoop</groupId>
                     <artifactId>hadoop-mapreduce-client-core</artifactId>
-                </exclusion>-->
+                </exclusion>
             </exclusions>
         </dependency>
+        <dependency>
+            <groupId>com.dlink</groupId>
+            <artifactId>dlink-client-hadoop</artifactId>
+        </dependency>
         <dependency>
             <groupId>org.apache.flink</groupId>
             <artifactId>flink-kubernetes_${scala.binary.version}</artifactId>

diff --git a/dlink-client/dlink-client-1.12/pom.xml b/dlink-client/dlink-client-1.12/pom.xml
@@ -53,7 +53,7 @@
                     <groupId>org.apache.hadoop</groupId>
                     <artifactId>hadoop-yarn-common</artifactId>
                 </exclusion>
-                <!--<exclusion>
+                <exclusion>
                     <groupId>org.apache.hadoop</groupId>
                     <artifactId>hadoop-common</artifactId>
                 </exclusion>
@@ -68,9 +68,13 @@
                 <exclusion>
                     <groupId>org.apache.hadoop</groupId>
                     <artifactId>hadoop-mapreduce-client-core</artifactId>
-                </exclusion>-->
+                </exclusion>
             </exclusions>
         </dependency>
+        <dependency>
+            <groupId>com.dlink</groupId>
+            <artifactId>dlink-client-hadoop</artifactId>
+        </dependency>
         <dependency>
             <groupId>org.apache.flink</groupId>
             <artifactId>flink-kubernetes_${scala.binary.version}</artifactId>

diff --git a/dlink-client/dlink-client-1.13/pom.xml b/dlink-client/dlink-client-1.13/pom.xml
@@ -53,7 +53,7 @@
                     <groupId>org.apache.hadoop</groupId>
                     <artifactId>hadoop-yarn-common</artifactId>
                 </exclusion>
-                <!--<exclusion>
+                <exclusion>
                     <groupId>org.apache.hadoop</groupId>
                     <artifactId>hadoop-common</artifactId>
                 </exclusion>
@@ -68,9 +68,13 @@
                 <exclusion>
                     <groupId>org.apache.hadoop</groupId>
                     <artifactId>hadoop-mapreduce-client-core</artifactId>
-                </exclusion>-->
+                </exclusion>
             </exclusions>
         </dependency>
+        <dependency>
+            <groupId>com.dlink</groupId>
+            <artifactId>dlink-client-hadoop</artifactId>
+        </dependency>
         <dependency>
             <groupId>org.apache.flink</groupId>
             <artifactId>flink-kubernetes_${scala.binary.version}</artifactId>

diff --git a/dlink-client/dlink-client-1.14/pom.xml b/dlink-client/dlink-client-1.14/pom.xml
@@ -52,24 +52,28 @@
                     <groupId>org.apache.hadoop</groupId>
                     <artifactId>hadoop-yarn-common</artifactId>
                 </exclusion>
-                <!--<exclusion>
+                <exclusion>
                     <groupId>org.apache.hadoop</groupId>
                     <artifactId>hadoop-common</artifactId>
                 </exclusion>
                 <exclusion>
                     <groupId>org.apache.hadoop</groupId>
                     <artifactId>hadoop-hdfs</artifactId>
-                </exclusion>-->
-                <!--<exclusion>
+                </exclusion>
+                <exclusion>
                     <groupId>org.apache.hadoop</groupId>
                     <artifactId>hadoop-yarn-client</artifactId>
-                </exclusion>-->
-                <!--<exclusion>
+                </exclusion>
+                <exclusion>
                     <groupId>org.apache.hadoop</groupId>
                     <artifactId>hadoop-mapreduce-client-core</artifactId>
-                </exclusion>-->
+                </exclusion>
             </exclusions>
         </dependency>
+        <dependency>
+            <groupId>com.dlink</groupId>
+            <artifactId>dlink-client-hadoop</artifactId>
+        </dependency>
         <dependency>
             <groupId>org.apache.flink</groupId>
             <artifactId>flink-kubernetes_${scala.binary.version}</artifactId>

diff --git a/dlink-client/dlink-client-hadoop/pom.xml b/dlink-client/dlink-client-hadoop/pom.xml
@@ -0,0 +1,151 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <artifactId>dlink-client</artifactId>
+        <groupId>com.dlink</groupId>
+        <version>0.5.0</version>
+    </parent>
+    <modelVersion>4.0.0</modelVersion>
+
+    <artifactId>dlink-client-hadoop</artifactId>
+    <properties>
+        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+        <!--<scope.type>provided</scope.type>-->
+        <hadoop.version>2.7.7</hadoop.version>
+        <scope.type>compile</scope.type>
+    </properties>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.apache.hadoop</groupId>
+            <artifactId>hadoop-common</artifactId>
+            <exclusions>
+                <exclusion>
+                    <groupId>com.google.guava</groupId>
+                    <artifactId>guava</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>javax.servlet</groupId>
+                    <artifactId>servlet-api</artifactId>
+                </exclusion>
+            </exclusions>
+            <version>${hadoop.version}</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.hadoop</groupId>
+            <artifactId>hadoop-hdfs</artifactId>
+            <version>${hadoop.version}</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>com.google.guava</groupId>
+                    <artifactId>guava</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>javax.servlet</groupId>
+                    <artifactId>servlet-api</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.hadoop</groupId>
+            <artifactId>hadoop-yarn-common</artifactId>
+            <version>${hadoop.version}</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>com.google.guava</groupId>
+                    <artifactId>guava</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>javax.servlet</groupId>
+                    <artifactId>servlet-api</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.hadoop</groupId>
+            <artifactId>hadoop-client</artifactId>
+            <version>${hadoop.version}</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.hadoop</groupId>
+            <artifactId>hadoop-yarn-client</artifactId>
+            <version>${hadoop.version}</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>com.google.guava</groupId>
+                    <artifactId>guava</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>javax.servlet</groupId>
+                    <artifactId>servlet-api</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.hadoop</groupId>
+            <artifactId>hadoop-mapreduce-client-core</artifactId>
+            <version>${hadoop.version}</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>com.google.guava</groupId>
+                    <artifactId>guava</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>javax.servlet</groupId>
+                    <artifactId>servlet-api</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-compiler-plugin</artifactId>
+                <configuration>
+                    <source>8</source>
+                    <target>8</target>
+                </configuration>
+            </plugin>
+
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-shade-plugin</artifactId>
+                <version>3.1.0</version>
+                <executions>
+                    <execution>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>shade</goal>
+                        </goals>
+                        <configuration>
+                            <createDependencyReducedPom>false</createDependencyReducedPom>
+                            <transformers>
+                            </transformers>
+                            <filters>
+                                <filter>
+                                    <artifact>*:*</artifact>
+                                    <excludes>
+                                        <exclude>META-INF/*.SF</exclude>
+                                        <exclude>META-INF/*.DSA</exclude>
+                                        <exclude>META-INF/*.RSA</exclude>
+                                    </excludes>
+                                </filter>
+                            </filters>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+
+        </plugins>
+    </build>
+
+</project>
diff --git a/dlink-client/pom.xml b/dlink-client/pom.xml
@@ -17,5 +17,6 @@
         <module>dlink-client-1.13</module>
         <module>dlink-client-1.11</module>
         <module>dlink-client-1.14</module>
+        <module>dlink-client-hadoop</module>
     </modules>
 </project>
diff --git a/dlink-core/pom.xml b/dlink-core/pom.xml
@@ -48,27 +48,27 @@
         <dependency>
             <groupId>com.dlink</groupId>
             <artifactId>dlink-client-1.13</artifactId>
-            <scope>provided</scope>
+            <scope>${scope.runtime}</scope>
         </dependency>
         <dependency>
             <groupId>com.dlink</groupId>
             <artifactId>dlink-executor</artifactId>
-            <scope>provided</scope>
+            <scope>${scope.runtime}</scope>
         </dependency>
         <dependency>
             <groupId>com.dlink</groupId>
             <artifactId>dlink-connector-jdbc-1.13</artifactId>
-            <scope>provided</scope>
+            <scope>${scope.runtime}</scope>
         </dependency>
         <dependency>
             <groupId>com.dlink</groupId>
             <artifactId>dlink-function</artifactId>
-            <scope>provided</scope>
+            <scope>${scope.runtime}</scope>
         </dependency>
         <dependency>
             <groupId>com.dlink</groupId>
             <artifactId>dlink-gateway</artifactId>
-            <scope>provided</scope>
+            <scope>${scope.runtime}</scope>
         </dependency>
     </dependencies>
 </project>
diff --git a/dlink-gateway/src/main/java/com/dlink/gateway/GatewayType.java b/dlink-gateway/src/main/java/com/dlink/gateway/GatewayType.java
@@ -46,4 +46,26 @@ public boolean equalsValue(String type){
         }
         return false;
     }
+
+    public static boolean isDeployCluster(String type){
+        switch (get(type)){
+            case YARN_APPLICATION:
+            case YARN_PER_JOB:
+            case KUBERNETES_APPLICATION:
+                return true;
+            default:
+                return false;
+        }
+    }
+
+    public boolean isDeployCluster(){
+        switch (value){
+            case "ya":
+            case "ypj":
+            case "ka":
+                return true;
+            default:
+                return false;
+        }
+    }
 }
diff --git a/dlink-web/config/defaultSettings.ts b/dlink-web/config/defaultSettings.ts
@@ -12,7 +12,7 @@ const Settings: LayoutSettings & {
   fixedHeader: false,
   fixSiderbar: true,
   colorWeak: false,
-  title: 'Dinky & Apache Flink',
+  title: 'Dinky 实时计算平台',
   pwa: false,
   logo: 'dinky.svg',
   iconfontUrl: '',