diff --git a/dinky-admin/src/main/java/org/dinky/sse/StepSse.java b/dinky-admin/src/main/java/org/dinky/sse/StepSse.java index ed808d00d3..76c50bbbb9 100644 --- a/dinky-admin/src/main/java/org/dinky/sse/StepSse.java +++ b/dinky-admin/src/main/java/org/dinky/sse/StepSse.java @@ -25,7 +25,6 @@ import java.io.File; import java.io.IOException; -import java.nio.charset.StandardCharsets; import java.time.Instant; import java.util.ArrayList; import java.util.LinkedList; @@ -108,24 +107,15 @@ public synchronized void sendMsg(Object msg) { }); } - public synchronized void addFileMsg(Object msg) { - FileUtil.appendUtf8String(Convert.toStr(msg), getLogFile()); - } - public synchronized void addFileMsgCusLog(String msg) { - String content = "\n============= " + Convert.toStr(msg) + " =============\n"; + String content = "============= " + Convert.toStr(msg) + " ============="; addMsg(content); - FileUtil.appendUtf8String(content, getLogFile()); + FileUtil.appendUtf8String(content + "\n", getLogFile()); } public synchronized void addFileMsgLog(String msg) { addMsg(msg); - FileUtil.appendUtf8String(msg, getLogFile()); - } - - public synchronized void addFileLog(List data) { - sendMsg(getList(data)); - FileUtil.appendString(StrUtil.join("\n", data), getLogFile(), StandardCharsets.UTF_8); + FileUtil.appendUtf8String(msg + "\n", getLogFile()); } protected File getLogFile() { @@ -144,7 +134,7 @@ public void send() { cachedThreadPool.execute(this::sendSync); } - public void main() { + public void run() { this.status = 1; getStep(); FileUtil.del(getLogFile()); @@ -192,7 +182,7 @@ public void setFinish(boolean status) { gitProject.setBuildState(1); gitProject.setBuildStep(nexStepSse.getStep()); gitProject.updateById(); - nexStepSse.main(); + nexStepSse.run(); } } @@ -201,7 +191,7 @@ private void close() { try { emitter.complete(); } catch (Exception e) { - e.printStackTrace(); + log.error("", e); } }); // Manual GC is required here to release file IO(此处需要手动GC,释放文件IO) @@ -233,7 +223,7 @@ protected Dict getLogObj(Object data) { // // } - Object dataResult = (data instanceof List) ? StrUtil.join("\n", data) : JSONUtil.toJsonStr(data); + Object dataResult = (data instanceof List) ? StrUtil.join("\n", data) + "\n" : JSONUtil.toJsonStr(data); return Dict.create() .set("type", 1) .set("currentStep", getStep()) diff --git a/dinky-admin/src/main/java/org/dinky/sse/git/GetJarsStepSse.java b/dinky-admin/src/main/java/org/dinky/sse/git/GetJarsStepSse.java index a69ef21a8b..6d4ad7d3f6 100644 --- a/dinky-admin/src/main/java/org/dinky/sse/git/GetJarsStepSse.java +++ b/dinky-admin/src/main/java/org/dinky/sse/git/GetJarsStepSse.java @@ -57,7 +57,7 @@ public GetJarsStepSse( public void exec() { List jars = MavenUtil.getJars((File) params.get("pom")); List pathList = uploadResources(jars); - addFileMsg(pathList); + addMsg(String.join("\n", pathList)); params.put("jarPath", pathList); } diff --git a/dinky-admin/src/main/java/org/dinky/sse/git/HeadStepSse.java b/dinky-admin/src/main/java/org/dinky/sse/git/HeadStepSse.java index 0151db05c3..350e4ed26c 100644 --- a/dinky-admin/src/main/java/org/dinky/sse/git/HeadStepSse.java +++ b/dinky-admin/src/main/java/org/dinky/sse/git/HeadStepSse.java @@ -68,11 +68,11 @@ private void checkJava() { String mavenHome = MavenUtil.getMavenHome(); if (StrUtil.isBlank(mavenHome)) { - addFileMsg(Status.GIT_MAVEN_HOME_NOT_SET.getMessage()); + addFileMsgLog(Status.GIT_MAVEN_HOME_NOT_SET.getMessage()); setFinish(false); } String mavenVersionMsg = MavenUtil.getMavenVersion(); - mavenVersionMsg += "\n Your Maven Home is: " + mavenHome; + mavenVersionMsg += "Your Maven Home is: " + mavenHome; addFileMsgLog(mavenVersionMsg); } diff --git a/dinky-admin/src/main/java/org/dinky/sse/git/MavenStepSse.java b/dinky-admin/src/main/java/org/dinky/sse/git/MavenStepSse.java index 40ca54a22b..dc10f5fff1 100644 --- a/dinky-admin/src/main/java/org/dinky/sse/git/MavenStepSse.java +++ b/dinky-admin/src/main/java/org/dinky/sse/git/MavenStepSse.java @@ -79,7 +79,7 @@ public void exec() { null, getLogFile().getAbsolutePath(), CollUtil.newArrayList("clean", "package"), - StrUtil.split(gitProject.getBuildArgs(), " "), + gitProject.getBuildArgs(), this::addFileMsgLog); params.put("pom", pom); Assert.isTrue(state, "maven build failed"); diff --git a/dinky-admin/src/main/java/org/dinky/utils/GitProjectStepSseFactory.java b/dinky-admin/src/main/java/org/dinky/utils/GitProjectStepSseFactory.java index 0129f598e0..f1c6f3e980 100644 --- a/dinky-admin/src/main/java/org/dinky/utils/GitProjectStepSseFactory.java +++ b/dinky-admin/src/main/java/org/dinky/utils/GitProjectStepSseFactory.java @@ -63,7 +63,7 @@ public static void build(GitProject gitProject, Dict params) { List emitterList = new ArrayList<>(); StepSse headStepSse = getHeadStepPlan(gitProject.getCodeType(), sleep, emitterList, params); - cachedThreadPool.execute(headStepSse::main); + cachedThreadPool.execute(headStepSse::run); gitProject.setBuildStep(1); gitProject.setBuildState(1); gitProject.setLastBuild(DateUtil.date()); diff --git a/dinky-admin/src/main/java/org/dinky/utils/MavenUtil.java b/dinky-admin/src/main/java/org/dinky/utils/MavenUtil.java index 3302f0fbfb..52c02aff6f 100644 --- a/dinky-admin/src/main/java/org/dinky/utils/MavenUtil.java +++ b/dinky-admin/src/main/java/org/dinky/utils/MavenUtil.java @@ -33,7 +33,6 @@ import java.util.function.Consumer; import java.util.stream.Collectors; -import cn.hutool.core.collection.CollUtil; import cn.hutool.core.date.DateUtil; import cn.hutool.core.io.FileUtil; import cn.hutool.core.lang.Assert; @@ -68,10 +67,6 @@ public class MavenUtil { private static final TemplateEngine ENGINE = new FreemarkerEngine(new TemplateConfig("templates", TemplateConfig.ResourceMode.CLASSPATH)); - public static boolean build(String setting, String pom, String logFile, List args) { - return build(setting, pom, null, null, logFile, CollUtil.newArrayList("package"), args, null); - } - public static boolean build( String setting, String pom, @@ -79,7 +74,7 @@ public static boolean build( String repositoryDir, String logFile, List goals, - List args, + String args, Consumer consumer) { Assert.notBlank(pom, "the project pom file cannot be empty"); @@ -97,15 +92,17 @@ public static boolean build( } String mavenCommandLine = getMavenCommandLineByMvn(pom, mavenHome, localRepositoryDirectory, setting, goals, args); - Opt.ofNullable(consumer).ifPresent(c -> c.accept("Executing command: " + mavenCommandLine + "\n")); + Opt.ofNullable(consumer).ifPresent(c -> c.accept("Executing command: " + mavenCommandLine)); int waitValue = RuntimeUtils.run( mavenCommandLine, s -> { - s = DateUtil.date().toMsStr() + " - " + s + "\n"; - consumer.accept(s); + s = DateUtil.date().toMsStr() + " - " + s; + if (consumer != null) { + consumer.accept(s); + } }, - consumer::accept); + consumer); return waitValue == 0; } @@ -158,7 +155,7 @@ public static String getMavenCommandLineByMvn( String repositoryDir, String settingsPath, List goals, - List args) { + String args) { projectDir = StrUtil.wrap(projectDir, "\""); settingsPath = StrUtil.wrap(settingsPath, "\""); List commandLine = new LinkedList<>(); @@ -171,7 +168,7 @@ public static String getMavenCommandLineByMvn( commandLine.add("-Dclassworlds.conf=" + StrUtil.wrap(mavenHome + "/bin/m2.conf", "\"")); commandLine.add("-s " + settingsPath); commandLine.add("-f " + projectDir); - commandLine.add(StrUtil.join(" ", args)); + commandLine.add(StrUtil.wrap(StrUtil.replace(args, "\"", "\\*"), "\"")); commandLine.add(StrUtil.join(" ", goals)); return StrUtil.join(" ", commandLine); } diff --git a/dinky-admin/src/test/java/org/dinky/utils/MavenUtilTests.java b/dinky-admin/src/test/java/org/dinky/utils/MavenUtilTests.java deleted file mode 100644 index 83bc31957d..0000000000 --- a/dinky-admin/src/test/java/org/dinky/utils/MavenUtilTests.java +++ /dev/null @@ -1,93 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.dinky.utils; - -import org.dinky.function.util.UDFUtil; - -import java.io.File; -import java.util.Arrays; -import java.util.List; - -import org.junit.Ignore; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Test; - -/** - * @author ZackYoung - * @since 0.8.0 - */ -@Disabled("This is local test!") -public class MavenUtilTests { - - @Test - @Ignore - public void build() { - MavenUtil.build( - null, - "/Users/zackyoung/companyProjects/api-monitor/pom.xml", - "/usr/local/", - "/Users/zackyoung/.m2/repository", - "/Users/zackyoung/projects/ideaProjects/dlink/dinky-admin/tmp/git/build.log", - Arrays.asList("package"), - null, - null); - } - - @Test - @Ignore - public void cloneAndBuild() { - String branch = "master"; - - File gitFile = new File("/Users/zackyoung/Desktop/test/git/api-monitor/" + branch); - String logFile = "/Users/zackyoung/Desktop/test/git/api-monitor/" + branch + "_log/3.log"; - File pom = new File(gitFile, "pom.xml"); - - MavenUtil.build("/Volumes/文档及虚拟机/company/数宜信/OpenApi/settings2.xml", pom.getAbsolutePath(), logFile, null); - } - - @Test - @Ignore - public void getJars() { - String branch = "master"; - - File gitFile = new File("/Users/zackyoung/Desktop/test/git/api-monitor/" + branch); - String logFile = "/Users/zackyoung/Desktop/test/git/api-monitor/" + branch + "_log/3.log"; - File pom = new File(gitFile, "pom.xml"); - - List jars = MavenUtil.getJars(pom); - System.out.println(jars); - jars.parallelStream().forEach(jar -> { - List> udfClassByJar = UDFUtil.getUdfClassByJar(jar); - System.out.println(udfClassByJar); - }); - } - - @Test - @Ignore - public void getVersion() { - System.out.println(MavenUtil.getMavenVersion()); - } - - @Test - @Ignore - public void getSettings() { - System.out.println(MavenUtil.getMavenSettingsPath()); - } -} diff --git a/dinky-client/dinky-client-base/src/main/java/org/dinky/trans/dml/ExecuteJarOperation.java b/dinky-client/dinky-client-base/src/main/java/org/dinky/trans/dml/ExecuteJarOperation.java index fae74b15c8..280590661a 100644 --- a/dinky-client/dinky-client-base/src/main/java/org/dinky/trans/dml/ExecuteJarOperation.java +++ b/dinky-client/dinky-client-base/src/main/java/org/dinky/trans/dml/ExecuteJarOperation.java @@ -19,6 +19,8 @@ package org.dinky.trans.dml; +import static org.dinky.utils.RunTimeUtil.extractArgs; + import org.dinky.executor.CustomTableEnvironment; import org.dinky.trans.AbstractOperation; import org.dinky.trans.ExtendOperation; @@ -38,10 +40,7 @@ import java.io.File; import java.net.URL; -import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; -import java.util.Iterator; import java.util.List; import java.util.Optional; @@ -131,30 +130,6 @@ public static Pipeline getStreamGraph( } } - public static List extractArgs(String args) { - List programArgs = new ArrayList<>(); - if (StrUtil.isNotEmpty(args)) { - String[] array = args.split("\\s+"); - Iterator iter = Arrays.asList(array).iterator(); - while (iter.hasNext()) { - String v = iter.next(); - String p = v.substring(0, 1); - if (p.equals("'") || p.equals("\"")) { - String value = v; - if (!v.endsWith(p)) { - while (!value.endsWith(p) && iter.hasNext()) { - value += " " + iter.next(); - } - } - programArgs.add(value.substring(1, value.length() - 1)); - } else { - programArgs.add(v); - } - } - } - return programArgs; - } - @Override public String asSummaryString() { return statement; diff --git a/dinky-common/src/main/java/org/dinky/utils/RunTimeUtil.java b/dinky-common/src/main/java/org/dinky/utils/RunTimeUtil.java index a469306c6a..f9eab94948 100644 --- a/dinky-common/src/main/java/org/dinky/utils/RunTimeUtil.java +++ b/dinky-common/src/main/java/org/dinky/utils/RunTimeUtil.java @@ -20,6 +20,8 @@ package org.dinky.utils; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Iterator; import java.util.List; import java.util.Stack; @@ -35,6 +37,36 @@ */ public class RunTimeUtil { + /** + * Extract parameters + * + * @param args parameters + * @return List of parameters + */ + public static List extractArgs(String args) { + List programArgs = new ArrayList<>(); + if (StrUtil.isNotEmpty(args)) { + String[] array = args.split("\\s+"); + Iterator iter = Arrays.asList(array).iterator(); + while (iter.hasNext()) { + String v = iter.next(); + String p = v.substring(0, 1); + if (p.equals("'") || p.equals("\"")) { + String value = v; + if (!v.endsWith(p)) { + while (!value.endsWith(p) && iter.hasNext()) { + value += " " + iter.next(); + } + } + programArgs.add(value.substring(1, value.length() - 1)); + } else { + programArgs.add(v); + } + } + } + return programArgs; + } + public static void recovery(Object obj) { obj = null; System.gc(); diff --git a/dinky-client/dinky-client-base/src/test/java/org/dinky/trans/dml/ExecuteJarOperationTest.java b/dinky-common/src/test/java/org/dinky/util/RunTimeUtilTest.java similarity index 92% rename from dinky-client/dinky-client-base/src/test/java/org/dinky/trans/dml/ExecuteJarOperationTest.java rename to dinky-common/src/test/java/org/dinky/util/RunTimeUtilTest.java index 4a6e175316..4a36452ce8 100644 --- a/dinky-client/dinky-client-base/src/test/java/org/dinky/trans/dml/ExecuteJarOperationTest.java +++ b/dinky-common/src/test/java/org/dinky/util/RunTimeUtilTest.java @@ -17,9 +17,9 @@ * */ -package org.dinky.trans.dml; +package org.dinky.util; -import static org.junit.jupiter.api.Assertions.*; +import org.dinky.utils.RunTimeUtil; import java.util.List; @@ -28,8 +28,7 @@ import com.google.common.collect.Lists; -class ExecuteJarOperationTest { - +public class RunTimeUtilTest { public static final List RESULT1 = Lists.newArrayList( "merge_into", "--warehouse", @@ -53,7 +52,7 @@ class ExecuteJarOperationTest { @Test void extractArgs() { - List args1 = ExecuteJarOperation.extractArgs( + List args1 = RunTimeUtil.extractArgs( "merge_into --warehouse hdfs:///tmp/paimon --database default --table T --source_table S --on \"T.id = S.order_id\" --merge_actions matched-upsert,matched-delete --matched_upsert_condition \"T.price > 100\" --matched_upsert_set \"mark = 'important'\" --matched_delete_condition \"T.price < 10\""); Assert.assertArrayEquals(args1.toArray(new String[0]), RESULT1.toArray(new String[0])); } diff --git a/dinky-web/src/locales/en-US/pages.ts b/dinky-web/src/locales/en-US/pages.ts index 1690ae7c41..933aadcbfa 100644 --- a/dinky-web/src/locales/en-US/pages.ts +++ b/dinky-web/src/locales/en-US/pages.ts @@ -1003,7 +1003,8 @@ export default { 'rc.gp.build.step.4': 'Get Artifact', 'rc.gp.build.step.5': 'Analysis UDF', 'rc.gp.build.step.6': 'Finish', - 'rc.gp.buildArgs': 'Build Arguments', + 'rc.gp.buildArgs': + 'Build parameters: Instructions need to be included in characters, please use single quotes \'\', double quotes will be escaped as \\"\\"', 'rc.gp.buildConfirm': 'Are you sure to start building this project? ', 'rc.gp.buildFail': 'Build failed', 'rc.gp.buildState': 'Build State', diff --git a/dinky-web/src/locales/zh-CN/pages.ts b/dinky-web/src/locales/zh-CN/pages.ts index e813efff9a..58f3a45a19 100644 --- a/dinky-web/src/locales/zh-CN/pages.ts +++ b/dinky-web/src/locales/zh-CN/pages.ts @@ -929,7 +929,7 @@ export default { 'rc.gp.build.step.4': '获取产物', 'rc.gp.build.step.5': '分析 UDF', 'rc.gp.build.step.6': '完成', - 'rc.gp.buildArgs': '构建参数', + 'rc.gp.buildArgs': '构建参数:指令需要字符包含,请使用单引号\'\',双引号则""会被转义为\\"\\"', 'rc.gp.buildConfirm': '确定开始构建该项目吗?', 'rc.gp.buildFail': '构建失败', 'rc.gp.buildState': '构建状态', diff --git a/dinky-web/src/pages/RegCenter/GitProject/components/BuildSteps/index.tsx b/dinky-web/src/pages/RegCenter/GitProject/components/BuildSteps/index.tsx index ca6ed12ea9..776608eb3a 100644 --- a/dinky-web/src/pages/RegCenter/GitProject/components/BuildSteps/index.tsx +++ b/dinky-web/src/pages/RegCenter/GitProject/components/BuildSteps/index.tsx @@ -138,7 +138,7 @@ export const BuildSteps: React.FC = (props) => { } if (data !== undefined) { logList[currentStep].push(data); - setLog(logList[currentStep]?.join('\n')); + setLog(logList[currentStep]?.join('')); } } else if (type === 2) { showDataStep = currentStep;