From 873410dc48aa9e18a501ac9d33be19bc125709cf Mon Sep 17 00:00:00 2001 From: peacewong Date: Sat, 7 Sep 2024 22:47:48 +0800 Subject: [PATCH] add new utils method --- .../linkis/common/utils/LinkisUtils.java | 165 ++++++++++++++++++ .../apache/linkis/common/utils/MD5Utils.java | 45 +++++ .../variable/VariableOperationTest.java | 10 +- .../storage/fs/impl/HDFSFileSystem.java | 2 +- .../linkis/storage/fs/impl/OSSFileSystem.java | 3 +- .../impl/DefaultECMRegisterService.scala | 2 +- .../parser/CommonEntranceParser.scala | 2 +- 7 files changed, 216 insertions(+), 13 deletions(-) create mode 100644 linkis-commons/linkis-common/src/main/java/org/apache/linkis/common/utils/LinkisUtils.java create mode 100644 linkis-commons/linkis-common/src/main/java/org/apache/linkis/common/utils/MD5Utils.java diff --git a/linkis-commons/linkis-common/src/main/java/org/apache/linkis/common/utils/LinkisUtils.java b/linkis-commons/linkis-common/src/main/java/org/apache/linkis/common/utils/LinkisUtils.java new file mode 100644 index 0000000000..353f80f1da --- /dev/null +++ b/linkis-commons/linkis-common/src/main/java/org/apache/linkis/common/utils/LinkisUtils.java @@ -0,0 +1,165 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.common.utils; + +import org.apache.linkis.common.exception.ErrorException; +import org.apache.linkis.common.exception.FatalException; +import org.apache.linkis.common.exception.WarnException; + +import java.util.concurrent.Callable; +import java.util.function.Function; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class LinkisUtils { + private static final Logger logger = LoggerFactory.getLogger(LinkisUtils.class); + + public static T tryCatch(Callable tryOp, Function catchOp) { + T result = null; + try { + result = tryOp.call(); + } catch (Throwable t) { + if (t instanceof FatalException) { + logger.error("Fatal error, system exit...", t); + System.exit(((FatalException) t).getErrCode()); + } else if (t instanceof VirtualMachineError) { + logger.error("Fatal error, system exit...", t); + System.exit(-1); + } else if (null != t.getCause() + && (t.getCause() instanceof FatalException + || t.getCause() instanceof VirtualMachineError)) { + logger.error("Caused by fatal error, system exit...", t); + System.exit(-1); + } else if (t instanceof Error) { + logger.error("Throw error", t); + throw (Error) t; + } else { + result = catchOp.apply(t); + } + } + return result; + } + + public static void tryFinally(Runnable tryOp, Runnable finallyOp) { + try { + tryOp.run(); + } finally { + finallyOp.run(); + } + } + + public static T tryAndWarn(Callable tryOp, Logger log) { + return tryCatch( + tryOp, + t -> { + if (t instanceof ErrorException) { + ErrorException error = (ErrorException) t; + log.error( + "Warning code（警告码）: {}, Warning message（警告信息）: {}.", + error.getErrCode(), + error.getDesc(), + error); + + } else if (t instanceof WarnException) { + WarnException warn = (WarnException) t; + log.warn( + "Warning code（警告码）: {}, Warning message（警告信息）: {}.", + warn.getErrCode(), + warn.getDesc(), + warn); + + } else { + log.warn("", t); + } + return null; + }); + } + + public static void tryAndErrorMsg(Runnable tryOp, String message, Logger log) { + try { + tryOp.run(); + } catch (WarnException t) { + WarnException warn = (WarnException) t; + log.warn( + "Warning code（警告码）: {}, Warning message（警告信息）: {}.", warn.getErrCode(), warn.getDesc()); + log.warn(message, warn); + } catch (Exception t) { + log.warn(message, t); + } + } + + public static void tryAndWarn(Runnable tryOp, Logger log) { + try { + tryOp.run(); + } catch (Throwable error) { + if (error instanceof WarnException) { + WarnException warn = (WarnException) error; + log.warn( + "Warning code（警告码）: {}, Warning message（警告信息）: {}.", + warn.getErrCode(), + warn.getDesc(), + error); + } else { + log.warn("", error); + } + } + } + + public static void tryAndWarnMsg(Runnable tryOp, String message, Logger log) { + try { + tryOp.run(); + } catch (WarnException t) { + WarnException warn = (WarnException) t; + log.warn( + "Warning code（警告码）: {}, Warning message（警告信息）: {}.", warn.getErrCode(), warn.getDesc()); + log.warn(message, warn); + } catch (Exception t) { + log.warn(message, t); + } + } + + public static T tryAndWarnMsg(Callable tryOp, String message, Logger log) { + return tryCatch( + tryOp, + t -> { + if (t instanceof ErrorException) { + ErrorException error = (ErrorException) t; + log.warn( + "Warning code（警告码）: {}, Warning message（警告信息）: {}.", + error.getErrCode(), + error.getDesc()); + log.warn(message, error); + } else if (t instanceof WarnException) { + WarnException warn = (WarnException) t; + log.warn( + "Warning code（警告码）: {}, Warning message（警告信息）: {}.", + warn.getErrCode(), + warn.getDesc()); + log.warn(message, warn); + } else { + log.warn(message, t); + } + return null; + }); + } + + public static String getJvmUser() { + return System.getProperty("user.name"); + } +} diff --git a/linkis-commons/linkis-common/src/main/java/org/apache/linkis/common/utils/MD5Utils.java b/linkis-commons/linkis-common/src/main/java/org/apache/linkis/common/utils/MD5Utils.java new file mode 100644 index 0000000000..1291b8bb68 --- /dev/null +++ b/linkis-commons/linkis-common/src/main/java/org/apache/linkis/common/utils/MD5Utils.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.common.utils; + +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; + +public class MD5Utils { + + /** + * @param plaintext + * @return + * @throws NoSuchAlgorithmException + */ + public static String encrypt(String plaintext) throws NoSuchAlgorithmException { + // 使用 MD5 算法创建 MessageDigest 对象 + MessageDigest md = MessageDigest.getInstance("MD5"); + // 更新 MessageDigest 对象中的字节数据 + md.update(plaintext.getBytes()); + // 对更新后的数据计算哈希值，存储在 byte 数组中 + byte[] digest = md.digest(); + // 将 byte 数组转换为十六进制字符串 + StringBuilder sb = new StringBuilder(); + for (byte b : digest) { + sb.append(String.format("%02x", b & 0xff)); + } + // 返回十六进制字符串 + return sb.toString(); + } +} diff --git a/linkis-commons/linkis-common/src/test/java/org/apache/linkis/common/variable/VariableOperationTest.java b/linkis-commons/linkis-common/src/test/java/org/apache/linkis/common/variable/VariableOperationTest.java index b24bad2467..ffc07a689e 100644 --- a/linkis-commons/linkis-common/src/test/java/org/apache/linkis/common/variable/VariableOperationTest.java +++ b/linkis-commons/linkis-common/src/test/java/org/apache/linkis/common/variable/VariableOperationTest.java @@ -40,19 +40,13 @@ public class VariableOperationTest { public void testSqlFormat() throws VariableOperationFailedException { String jsonOld = "select \n" - + "\"&{yyyy-MM}\",\n" - + "\"&{yyyy-MM-dd HHmmss}\",\n" - + "\"&yyyyMMddHH\",\n" - + "\"&{yyyy-MM-dd-HH}\""; + + "\"&{yyyy-MM}\""; String jsonNew = VariableOperationUtils.replaces(zonedDateTime, jsonOld); System.out.println(jsonNew); assertEquals( jsonNew, "select \n" - + "\"2022-04\",\n" - + "\"2022-04-02 173507\",\n" - + "\"&yyyyMMddHH\",\n" - + "\"2022-04-02-17\""); + + "\"2022-04\""); } @Test diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/fs/impl/HDFSFileSystem.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/fs/impl/HDFSFileSystem.java index 56d846b31d..f40d75c040 100644 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/fs/impl/HDFSFileSystem.java +++ b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/fs/impl/HDFSFileSystem.java @@ -182,7 +182,7 @@ public void init(Map properties) throws IOException { throw new IOException("User cannot be empty(用户不能为空)"); } - if (label == null && (boolean) Configuration.IS_MULTIPLE_YARN_CLUSTER().getValue()) { + if (label == null && (boolean) Configuration.IS_MULTIPLE_YARN_CLUSTER()) { label = StorageConfiguration.LINKIS_STORAGE_FS_LABEL.getValue(); } conf = HDFSUtils.getConfigurationByLabel(user, label); diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/fs/impl/OSSFileSystem.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/fs/impl/OSSFileSystem.java index 16e5beb539..2d52b83049 100644 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/fs/impl/OSSFileSystem.java +++ b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/fs/impl/OSSFileSystem.java @@ -152,8 +152,7 @@ public FsPathListWithError listPathWithError(FsPath path) throws IOException { public void init(Map properties) throws IOException { // read origin configs from hadoop conf if (label == null - && (boolean) - org.apache.linkis.common.conf.Configuration.IS_MULTIPLE_YARN_CLUSTER().getValue()) { + && (boolean) org.apache.linkis.common.conf.Configuration.IS_MULTIPLE_YARN_CLUSTER()) { label = StorageConfiguration.LINKIS_STORAGE_FS_LABEL.getValue(); } conf = HDFSUtils.getConfigurationByLabel(user, label); diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/DefaultECMRegisterService.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/DefaultECMRegisterService.scala index d88f270862..51f223e476 100644 --- a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/DefaultECMRegisterService.scala +++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/DefaultECMRegisterService.scala @@ -62,7 +62,7 @@ class DefaultECMRegisterService extends ECMRegisterService with ECMEventListener ENGINE_CONN_MANAGER_SPRING_NAME ) - if (Configuration.IS_MULTIPLE_YARN_CLUSTER.getValue.asInstanceOf[Boolean]) { + if (Configuration.IS_MULTIPLE_YARN_CLUSTER) { labels.asScala += LabelKeyConstant.YARN_CLUSTER_KEY -> (ECM_YARN_CLUSTER_TYPE + "_" + ECM_YARN_CLUSTER_NAME) } diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/parser/CommonEntranceParser.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/parser/CommonEntranceParser.scala index afc18bdc19..58fc1f45c3 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/parser/CommonEntranceParser.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/parser/CommonEntranceParser.scala @@ -195,7 +195,7 @@ class CommonEntranceParser(val persistenceManager: PersistenceManager) } private def generateAndVerifyClusterLabel(labels: util.Map[String, Label[_]]): Unit = { - if (!Configuration.IS_MULTIPLE_YARN_CLUSTER.getValue.asInstanceOf[Boolean]) { + if (!Configuration.IS_MULTIPLE_YARN_CLUSTER) { return } var clusterLabel = labels