diff --git a/dinky-admin/src/main/java/org/dinky/init/EnvInit.java b/dinky-admin/src/main/java/org/dinky/init/EnvInit.java index 52919c57d1..d3d875708d 100644 --- a/dinky-admin/src/main/java/org/dinky/init/EnvInit.java +++ b/dinky-admin/src/main/java/org/dinky/init/EnvInit.java @@ -20,6 +20,11 @@ package org.dinky.init; import org.apache.hadoop.fs.FileSystem; + +import java.io.IOException; + +import javax.annotation.PreDestroy; + import org.springframework.boot.ApplicationArguments; import org.springframework.boot.ApplicationRunner; import org.springframework.context.ApplicationContext; @@ -31,9 +36,6 @@ import cn.hutool.system.SystemUtil; import lombok.extern.slf4j.Slf4j; -import javax.annotation.PreDestroy; -import java.io.IOException; - @Order(-1) @Component @Slf4j @@ -65,6 +67,7 @@ public void run(ApplicationArguments args) throws Exception { ipAddress, port); } + @PreDestroy private void destroy() throws IOException { FileSystem.closeAll(); diff --git a/dinky-client/dinky-client-1.14/src/main/java/org/apache/flink/yarn/YarnClusterDescriptor.java b/dinky-client/dinky-client-1.14/src/main/java/org/apache/flink/yarn/YarnClusterDescriptor.java index 26dcd2e436..b3ef234f9c 100644 --- a/dinky-client/dinky-client-1.14/src/main/java/org/apache/flink/yarn/YarnClusterDescriptor.java +++ b/dinky-client/dinky-client-1.14/src/main/java/org/apache/flink/yarn/YarnClusterDescriptor.java @@ -1004,8 +1004,9 @@ localizedKeytabPath, new Path(keytab), "", LocalResourceType.FILE, false, false) } amContainer.setLocalResources(fileUploader.getRegisteredLocalResources()); - // overwrite zh:这里必需需要剔除close,因为它会关闭filesystem,导致并发提交出现异常 en: Close must be culled here, as it closes the filesystem, causing an exception to the concurrent commit -// fileUploader.close(); + // overwrite zh:这里必需需要剔除close,因为它会关闭filesystem,导致并发提交出现异常 en: Close must be culled here, as it closes the + // filesystem, causing an exception to the concurrent commit + // fileUploader.close(); // Setup CLASSPATH and environment variables for ApplicationMaster final Map appMasterEnv = new HashMap<>(); @@ -1495,8 +1496,9 @@ public void run() { throw new IOException("Deleting files in " + yarnFilesDir + " was unsuccessful"); } - // overwrite zh:这里必需需要剔除close,因为它会关闭filesystem,导致并发提交出现异常 en: Close must be culled here, as it closes the filesystem, causing an exception to the concurrent commit -// fs.close(); + // overwrite zh:这里必需需要剔除close,因为它会关闭filesystem,导致并发提交出现异常 en: Close must be culled here, as it closes + // the filesystem, causing an exception to the concurrent commit + // fs.close(); } catch (IOException e) { LOG.error("Failed to delete Flink Jar and configuration files in HDFS", e); } diff --git a/dinky-client/dinky-client-1.15/src/main/java/org/apache/flink/yarn/YarnClusterDescriptor.java b/dinky-client/dinky-client-1.15/src/main/java/org/apache/flink/yarn/YarnClusterDescriptor.java index 6d7dddac0f..7d90b5adc6 100644 --- a/dinky-client/dinky-client-1.15/src/main/java/org/apache/flink/yarn/YarnClusterDescriptor.java +++ b/dinky-client/dinky-client-1.15/src/main/java/org/apache/flink/yarn/YarnClusterDescriptor.java @@ -1037,8 +1037,9 @@ localizedKeytabPath, new Path(keytab), "", LocalResourceType.FILE, false, false) } amContainer.setLocalResources(fileUploader.getRegisteredLocalResources()); - // overwrite zh:这里必需需要剔除close,因为它会关闭filesystem,导致并发提交出现异常 en: Close must be culled here, as it closes the filesystem, causing an exception to the concurrent commit -// fileUploader.close(); + // overwrite zh:这里必需需要剔除close,因为它会关闭filesystem,导致并发提交出现异常 en: Close must be culled here, as it closes the + // filesystem, causing an exception to the concurrent commit + // fileUploader.close(); // Setup CLASSPATH and environment variables for ApplicationMaster final Map appMasterEnv = new HashMap<>(); @@ -1546,8 +1547,9 @@ public void run() { throw new IOException("Deleting files in " + yarnFilesDir + " was unsuccessful"); } - // overwrite zh:这里必需需要剔除close,因为它会关闭filesystem,导致并发提交出现异常 en: Close must be culled here, as it closes the filesystem, causing an exception to the concurrent commit -// fs.close(); + // overwrite zh:这里必需需要剔除close,因为它会关闭filesystem,导致并发提交出现异常 en: Close must be culled here, as it closes + // the filesystem, causing an exception to the concurrent commit + // fs.close(); } catch (IOException e) { LOG.error("Failed to delete Flink Jar and configuration files in HDFS", e); } diff --git a/dinky-client/dinky-client-1.16/src/main/java/org/apache/flink/yarn/YarnClusterDescriptor.java b/dinky-client/dinky-client-1.16/src/main/java/org/apache/flink/yarn/YarnClusterDescriptor.java index 5476774554..0e44aa5a0f 100644 --- a/dinky-client/dinky-client-1.16/src/main/java/org/apache/flink/yarn/YarnClusterDescriptor.java +++ b/dinky-client/dinky-client-1.16/src/main/java/org/apache/flink/yarn/YarnClusterDescriptor.java @@ -1054,8 +1054,9 @@ localizedKeytabPath, new Path(keytab), "", LocalResourceType.FILE, false, false) } amContainer.setLocalResources(fileUploader.getRegisteredLocalResources()); - // overwrite zh:这里必需需要剔除close,因为它会关闭filesystem,导致并发提交出现异常 en: Close must be culled here, as it closes the filesystem, causing an exception to the concurrent commit -// fileUploader.close(); + // overwrite zh:这里必需需要剔除close,因为它会关闭filesystem,导致并发提交出现异常 en: Close must be culled here, as it closes the + // filesystem, causing an exception to the concurrent commit + // fileUploader.close(); // Setup CLASSPATH and environment variables for ApplicationMaster final Map appMasterEnv = generateApplicationMasterEnv( @@ -1551,8 +1552,9 @@ public void run() { throw new IOException("Deleting files in " + yarnFilesDir + " was unsuccessful"); } - // overwrite zh:这里必需需要剔除close,因为它会关闭filesystem,导致并发提交出现异常 en: Close must be culled here, as it closes the filesystem, causing an exception to the concurrent commit -// fs.close(); + // overwrite zh:这里必需需要剔除close,因为它会关闭filesystem,导致并发提交出现异常 en: Close must be culled here, as it closes + // the filesystem, causing an exception to the concurrent commit + // fs.close(); } catch (IOException e) { LOG.error("Failed to delete Flink Jar and configuration files in HDFS", e); } diff --git a/dinky-client/dinky-client-1.17/src/main/java/org/apache/flink/yarn/YarnClusterDescriptor.java b/dinky-client/dinky-client-1.17/src/main/java/org/apache/flink/yarn/YarnClusterDescriptor.java index f20d1563bd..7f912c0541 100644 --- a/dinky-client/dinky-client-1.17/src/main/java/org/apache/flink/yarn/YarnClusterDescriptor.java +++ b/dinky-client/dinky-client-1.17/src/main/java/org/apache/flink/yarn/YarnClusterDescriptor.java @@ -1047,8 +1047,9 @@ localizedKeytabPath, new Path(keytab), "", LocalResourceType.FILE, false, false) } amContainer.setLocalResources(fileUploader.getRegisteredLocalResources()); - // overwrite zh:这里必需需要剔除close,因为它会关闭filesystem,导致并发提交出现异常 en: Close must be culled here, as it closes the filesystem, causing an exception to the concurrent commit -// fileUploader.close(); + // overwrite zh:这里必需需要剔除close,因为它会关闭filesystem,导致并发提交出现异常 en: Close must be culled here, as it closes the + // filesystem, causing an exception to the concurrent commit + // fileUploader.close(); // Setup CLASSPATH and environment variables for ApplicationMaster final Map appMasterEnv = generateApplicationMasterEnv( @@ -1567,8 +1568,9 @@ public void run() { throw new IOException("Deleting files in " + yarnFilesDir + " was unsuccessful"); } - // overwrite zh:这里必需需要剔除close,因为它会关闭filesystem,导致并发提交出现异常 en: Close must be culled here, as it closes the filesystem, causing an exception to the concurrent commit -// fs.close(); + // overwrite zh:这里必需需要剔除close,因为它会关闭filesystem,导致并发提交出现异常 en: Close must be culled here, as it closes + // the filesystem, causing an exception to the concurrent commit + // fs.close(); } catch (IOException e) { LOG.error("Failed to delete Flink Jar and configuration files in HDFS", e); } diff --git a/dinky-client/dinky-client-1.18/src/main/java/org/apache/flink/yarn/YarnClusterDescriptor.java b/dinky-client/dinky-client-1.18/src/main/java/org/apache/flink/yarn/YarnClusterDescriptor.java index 1cb59827c8..3abeefacb3 100644 --- a/dinky-client/dinky-client-1.18/src/main/java/org/apache/flink/yarn/YarnClusterDescriptor.java +++ b/dinky-client/dinky-client-1.18/src/main/java/org/apache/flink/yarn/YarnClusterDescriptor.java @@ -1056,8 +1056,9 @@ localizedKeytabPath, new Path(keytab), "", LocalResourceType.FILE, false, false) amContainer.setLocalResources(fileUploader.getRegisteredLocalResources()); - // overwrite zh:这里必需需要剔除close,因为它会关闭filesystem,导致并发提交出现异常 en: Close must be culled here, as it closes the filesystem, causing an exception to the concurrent commit - //fileUploader.close(); + // overwrite zh:这里必需需要剔除close,因为它会关闭filesystem,导致并发提交出现异常 en: Close must be culled here, as it closes the + // filesystem, causing an exception to the concurrent commit + // fileUploader.close(); Utils.setAclsFor(amContainer, flinkConfiguration); @@ -1578,7 +1579,8 @@ public void run() { throw new IOException("Deleting files in " + yarnFilesDir + " was unsuccessful"); } - // overwrite zh:这里必需需要剔除close,因为它会关闭filesystem,导致并发提交出现异常 en: Close must be culled here, as it closes the filesystem, causing an exception to the concurrent commit + // overwrite zh:这里必需需要剔除close,因为它会关闭filesystem,导致并发提交出现异常 en: Close must be culled here, as it closes + // the filesystem, causing an exception to the concurrent commit // fs.close(); } catch (IOException e) { LOG.error("Failed to delete Flink Jar and configuration files in HDFS", e); diff --git a/dinky-client/dinky-client-1.19/src/main/java/org/apache/flink/yarn/YarnClusterDescriptor.java b/dinky-client/dinky-client-1.19/src/main/java/org/apache/flink/yarn/YarnClusterDescriptor.java index b2cb2beb60..ac6497fde8 100644 --- a/dinky-client/dinky-client-1.19/src/main/java/org/apache/flink/yarn/YarnClusterDescriptor.java +++ b/dinky-client/dinky-client-1.19/src/main/java/org/apache/flink/yarn/YarnClusterDescriptor.java @@ -1093,7 +1093,8 @@ localizedKeytabPath, new Path(keytab), "", LocalResourceType.FILE, false, false) } amContainer.setLocalResources(fileUploader.getRegisteredLocalResources()); - // overwrite zh:这里必需需要剔除close,因为它会关闭filesystem,导致并发提交出现异常 en: Close must be culled here, as it closes the filesystem, causing an exception to the concurrent commit + // overwrite zh:这里必需需要剔除close,因为它会关闭filesystem,导致并发提交出现异常 en: Close must be culled here, as it closes the + // filesystem, causing an exception to the concurrent commit // fileUploader.close(); Utils.setAclsFor(amContainer, flinkConfiguration); @@ -1614,7 +1615,8 @@ public void run() { throw new IOException("Deleting files in " + yarnFilesDir + " was unsuccessful"); } - // overwrite zh:这里必需需要剔除close,因为它会关闭filesystem,导致并发提交出现异常 en: Close must be culled here, as it closes the filesystem, causing an exception to the concurrent commit + // overwrite zh:这里必需需要剔除close,因为它会关闭filesystem,导致并发提交出现异常 en: Close must be culled here, as it closes + // the filesystem, causing an exception to the concurrent commit // fs.close(); } catch (IOException e) { LOG.error("Failed to delete Flink Jar and configuration files in HDFS", e); diff --git a/dinky-client/dinky-client-1.20/src/main/java/org/apache/flink/yarn/YarnClusterDescriptor.java b/dinky-client/dinky-client-1.20/src/main/java/org/apache/flink/yarn/YarnClusterDescriptor.java index ad6fa6a8f8..01ff1f40a2 100644 --- a/dinky-client/dinky-client-1.20/src/main/java/org/apache/flink/yarn/YarnClusterDescriptor.java +++ b/dinky-client/dinky-client-1.20/src/main/java/org/apache/flink/yarn/YarnClusterDescriptor.java @@ -1091,7 +1091,8 @@ localizedKeytabPath, new Path(keytab), "", LocalResourceType.FILE, false, false) } amContainer.setLocalResources(fileUploader.getRegisteredLocalResources()); - // overwrite zh:这里必需需要剔除close,因为它会关闭filesystem,导致并发提交出现异常 en: Close must be culled here, as it closes the filesystem, causing an exception to the concurrent commit + // overwrite zh:这里必需需要剔除close,因为它会关闭filesystem,导致并发提交出现异常 en: Close must be culled here, as it closes the + // filesystem, causing an exception to the concurrent commit // fileUploader.close(); Utils.setAclsFor(amContainer, flinkConfiguration); @@ -1613,9 +1614,11 @@ public void run() { throw new IOException("Deleting files in " + yarnFilesDir + " was unsuccessful"); } - // overwrite zh:这里必需需要剔除close,因为它会关闭filesystem,导致并发提交出现异常 en: Close must be culled here, as it closes the filesystem, causing an exception to the concurrent commit - // overwrite zh:这里必需需要剔除close,因为它会关闭filesystem,导致并发提交出现异常 en: Close must be culled here, as it closes the filesystem, causing an exception to the concurrent commit -// fs.close(); + // overwrite zh:这里必需需要剔除close,因为它会关闭filesystem,导致并发提交出现异常 en: Close must be culled here, as it closes + // the filesystem, causing an exception to the concurrent commit + // overwrite zh:这里必需需要剔除close,因为它会关闭filesystem,导致并发提交出现异常 en: Close must be culled here, as it closes + // the filesystem, causing an exception to the concurrent commit + // fs.close(); } catch (IOException e) { LOG.error("Failed to delete Flink Jar and configuration files in HDFS", e); }