From 6101f8aadee2a2c3f51f4c094f4e793d6cbf4deb Mon Sep 17 00:00:00 2001 From: peacewong Date: Mon, 19 Aug 2024 10:50:15 +0800 Subject: [PATCH 01/90] bak --- linkis-commons/linkis-httpclient/pom.xml | 24 ---- .../linkis/server/PerformanceInterceptor.java | 10 +- .../transform/JavaCollectionSerializer.scala | 50 -------- .../linkis/rpc/transform/RPCFormats.scala | 26 ---- .../apache/linkis/rpc/RPCFormatsTest.scala | 93 -------------- .../linkis/storage/domain/MethodEntity.scala | 8 +- .../client/response/JobProgressResult.scala | 17 +-- .../linkis-engineconn-manager-server/pom.xml | 7 -- .../execute/ComputationExecutor.scala | 2 +- .../linkis-application-manager/pom.xml | 7 -- .../manager/rm/external/yarn/YarnAppInfo.java | 78 ++++++++++++ .../manager/am/rpc/ManagerRPCFormats.scala | 35 ------ .../linkis/manager/am/utils/AMUtils.scala | 36 +++--- .../rm/external/yarn/YarnAppInfo.scala | 29 ----- .../impl/DefaultReqResourceService.scala | 6 +- .../service/impl/DefaultResourceManager.scala | 2 +- .../DriverAndYarnReqResourceService.scala | 4 - .../linkis/manager/rm/utils/RMUtils.scala | 44 ++++--- .../linkis-manager-common/pom.xml | 14 --- .../common/entity/resource/Resource.scala | 116 +++--------------- .../serializer/NodeResourceSerializer.scala | 62 ---------- .../RegisterEMRequestSerializer.scala | 58 --------- .../manager/common/utils/ResourceUtils.scala | 84 +++++++++---- .../linkis/manager/rm/ResultResource.scala | 22 ---- .../spark/executor/SparkSqlExecutor.scala | 10 +- 25 files changed, 231 insertions(+), 613 deletions(-) delete mode 100644 linkis-commons/linkis-rpc/src/main/scala/org/apache/linkis/rpc/transform/JavaCollectionSerializer.scala delete mode 100644 linkis-commons/linkis-rpc/src/main/scala/org/apache/linkis/rpc/transform/RPCFormats.scala delete mode 100644 linkis-commons/linkis-rpc/src/test/scala/org/apache/linkis/rpc/RPCFormatsTest.scala create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/rm/external/yarn/YarnAppInfo.java delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/rpc/ManagerRPCFormats.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/external/yarn/YarnAppInfo.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/scala/org/apache/linkis/manager/common/serializer/NodeResourceSerializer.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/scala/org/apache/linkis/manager/common/serializer/RegisterEMRequestSerializer.scala diff --git a/linkis-commons/linkis-httpclient/pom.xml b/linkis-commons/linkis-httpclient/pom.xml index 473b591a08..1951e3cd4f 100644 --- a/linkis-commons/linkis-httpclient/pom.xml +++ b/linkis-commons/linkis-httpclient/pom.xml @@ -43,30 +43,6 @@ ${httpmime.version} - - org.json4s - json4s-jackson_${scala.binary.version} - ${json4s.version} - - - org.scala-lang - scala-library - - - com.fasterxml.jackson.core - jackson-databind - - - com.fasterxml.jackson.core - jackson-annotations - - - com.fasterxml.jackson.core - jackson-core - - - - diff --git a/linkis-commons/linkis-module/src/main/java/org/apache/linkis/server/PerformanceInterceptor.java b/linkis-commons/linkis-module/src/main/java/org/apache/linkis/server/PerformanceInterceptor.java index de5d367c53..2a9cb2dd02 100644 --- a/linkis-commons/linkis-module/src/main/java/org/apache/linkis/server/PerformanceInterceptor.java +++ b/linkis-commons/linkis-module/src/main/java/org/apache/linkis/server/PerformanceInterceptor.java @@ -47,11 +47,11 @@ public void afterCompletion( long endTime = System.currentTimeMillis(); long executeTime = endTime - startTime; logger.info( - "Request client address：{} request URL: {} Method: {} taken: {} ms", - LinkisSpringUtils.getClientIP(request), - request.getRequestURI(), - request.getMethod(), - executeTime); + "Request client address：{} request URL: {} Method: {} taken: {} ms", + LinkisSpringUtils.getClientIP(request), + request.getRequestURI(), + request.getMethod(), + executeTime); } } } diff --git a/linkis-commons/linkis-rpc/src/main/scala/org/apache/linkis/rpc/transform/JavaCollectionSerializer.scala b/linkis-commons/linkis-rpc/src/main/scala/org/apache/linkis/rpc/transform/JavaCollectionSerializer.scala deleted file mode 100644 index 5c9e163689..0000000000 --- a/linkis-commons/linkis-rpc/src/main/scala/org/apache/linkis/rpc/transform/JavaCollectionSerializer.scala +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.rpc.transform - -import org.apache.linkis.server.BDPJettyServerHelper - -import org.json4s.{CustomSerializer, JArray, JObject} -import org.json4s.jackson.JsonMethods.parse -import org.json4s.jackson.Serialization.write - -// TODO is now only the simplest implementation, and there is a need to optimize it later.(TODO 现在只做最简单的实现，后续有需要再优化) - -object JavaCollectionSerializer - extends CustomSerializer[java.util.List[_]](implicit formats => - ( - { case j: JArray => - BDPJettyServerHelper.gson.fromJson(write(j), classOf[java.util.List[_]]) - }, - { case list: java.util.List[_] => - parse(BDPJettyServerHelper.gson.toJson(list)) - } - ) - ) - -object JavaMapSerializer - extends CustomSerializer[java.util.Map[_, _]](implicit formats => - ( - { case j: JObject => - BDPJettyServerHelper.gson.fromJson(write(j), classOf[java.util.Map[_, _]]) - }, - { case map: java.util.Map[_, _] => - parse(BDPJettyServerHelper.gson.toJson(map)) - } - ) - ) diff --git a/linkis-commons/linkis-rpc/src/main/scala/org/apache/linkis/rpc/transform/RPCFormats.scala b/linkis-commons/linkis-rpc/src/main/scala/org/apache/linkis/rpc/transform/RPCFormats.scala deleted file mode 100644 index 4e94584a37..0000000000 --- a/linkis-commons/linkis-rpc/src/main/scala/org/apache/linkis/rpc/transform/RPCFormats.scala +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.rpc.transform - -import org.json4s.Serializer - -trait RPCFormats { - - def getSerializers: Array[Serializer[_]] - -} diff --git a/linkis-commons/linkis-rpc/src/test/scala/org/apache/linkis/rpc/RPCFormatsTest.scala b/linkis-commons/linkis-rpc/src/test/scala/org/apache/linkis/rpc/RPCFormatsTest.scala deleted file mode 100644 index fbaf1b9b22..0000000000 --- a/linkis-commons/linkis-rpc/src/test/scala/org/apache/linkis/rpc/RPCFormatsTest.scala +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.rpc - -import org.apache.linkis.rpc.transform.{JavaCollectionSerializer, JavaMapSerializer} - -import org.apache.commons.lang3.ClassUtils - -import java.lang.reflect.ParameterizedType -import java.util - -import org.json4s.{CustomSerializer, DefaultFormats, Extraction} -import org.json4s.JsonAST.JObject -import org.json4s.JsonDSL._ -import org.json4s.jackson.Serialization -import org.json4s.reflect.ManifestFactory - -object RPCFormatsTest { - - trait ResultResource - class AvailableResource(val ticketId: String) extends ResultResource - - object ResultResourceSerializer - extends CustomSerializer[ResultResource](implicit formats => - ( - { case JObject(List(("AvailableResource", JObject(List(("ticketId", ticketId)))))) => - new AvailableResource(ticketId.extract[String]) - }, - { case r: AvailableResource => - ("AvailableResource", ("ticketId", Extraction.decompose(r.ticketId))) - } - ) - ) - - def testRPC1(args: Array[String]): Unit = { - implicit val formats = DefaultFormats + ResultResourceSerializer - val serializerClasses = formats.customSerializers - .map(_.getClass.getGenericSuperclass match { - case p: ParameterizedType => - val params = p.getActualTypeArguments - if (params == null || params.isEmpty) null - else params(0).asInstanceOf[Class[_]] - }) - .filter(_ != null) - val a = new AvailableResource("aaa") - val str = Serialization.write(a) - println(str) - val clazz = classOf[AvailableResource] - println(serializerClasses) - val realClass1 = serializerClasses.find(ClassUtils.isAssignable(clazz, _)) - println(realClass1) - val realClass = realClass1.getOrElse(clazz) - val obj = Serialization.read(str)(formats, ManifestFactory.manifestOf(realClass)) - println(obj) - println(classOf[Array[_]].getClass.getName) - } - - case class TestCollection1(a: String, list: java.util.List[String]) - case class TestCollection2(a: String, list: java.util.Map[String, Integer]) - - def testRPC2(args: Array[String]): Unit = { - implicit val formats = DefaultFormats + JavaCollectionSerializer + JavaMapSerializer - // val a = TestCollection1("1", new util.ArrayList[String]()) - val a = TestCollection2("1", new util.HashMap[String, Integer]()) - // a.list.add("1111") - a.list.put("1111", 2) - val str = Serialization.write(a) - println(str) - val realClass = classOf[TestCollection2] - val obj = Serialization.read(str)(formats, ManifestFactory.manifestOf(realClass)) - println(obj) - } - - def main(args: Array[String]): Unit = { - testRPC2(args) - } - -} diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/domain/MethodEntity.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/domain/MethodEntity.scala index ec9cf4f15f..fac0a2d01b 100644 --- a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/domain/MethodEntity.scala +++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/domain/MethodEntity.scala @@ -20,8 +20,6 @@ package org.apache.linkis.storage.domain import java.lang.reflect.Type import com.google.gson.GsonBuilder -import org.json4s.DefaultFormats -import org.json4s.jackson.Serialization.write /** * @param id @@ -58,8 +56,6 @@ case class MethodEntity( object MethodEntitySerializer { - implicit val formats = DefaultFormats - import org.json4s.jackson.JsonMethods._ val gson = new GsonBuilder().setDateFormat("yyyy-MM-dd'T'HH:mm:ssZ").create /** @@ -67,14 +63,14 @@ object MethodEntitySerializer { * @param code * @return */ - def deserializer(code: String): MethodEntity = parse(code).extract[MethodEntity] + def deserializer(code: String): MethodEntity = gson.fromJson(code, classOf[MethodEntity]) /** * Serialize MethodEntity to code 序列化MethodEntity为code * @param methodEntity * @return */ - def serializer(methodEntity: MethodEntity): String = write(methodEntity) + def serializer(methodEntity: MethodEntity): String = gson.toJson(methodEntity) /** * Serialize a java object as a string 序列化java对象为字符串 diff --git a/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/ujes/client/response/JobProgressResult.scala b/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/ujes/client/response/JobProgressResult.scala index 0cf163c286..e3330d00a8 100644 --- a/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/ujes/client/response/JobProgressResult.scala +++ b/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/ujes/client/response/JobProgressResult.scala @@ -17,6 +17,7 @@ package org.apache.linkis.ujes.client.response +import org.apache.linkis.common.utils.JsonUtils import org.apache.linkis.httpclient.dws.annotation.DWSHttpMessageResult import org.apache.linkis.protocol.engine.JobProgressInfo @@ -24,9 +25,6 @@ import java.util import scala.collection.JavaConverters._ -import org.json4s._ -import org.json4s.jackson.Serialization._ - @DWSHttpMessageResult("/api/rest_j/v\\d+/entrance/(\\S+)/progress") class JobProgressResult extends UJESJobResult { @@ -34,15 +32,20 @@ class JobProgressResult extends UJESJobResult { private var progressInfo: util.List[util.Map[String, AnyRef]] = _ private var progressInfos: Array[JobProgressInfo] = _ - private implicit val formats = DefaultFormats - def setProgress(progress: Float): Unit = this.progress = progress def getProgress: Float = progress def setProgressInfo(progressInfo: util.List[util.Map[String, AnyRef]]): Unit = { this.progressInfo = progressInfo - progressInfos = - progressInfo.asScala.map(map => read[JobProgressInfo](write(map.asScala.toMap))).toArray + progressInfos = progressInfo.asScala + .map(map => + JsonUtils.jackson + .readValue( + JsonUtils.jackson.writeValueAsString(map.asScala.toMap), + classOf[JobProgressInfo] + ) + ) + .toArray } def getProgressInfo: Array[JobProgressInfo] = progressInfos diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/pom.xml b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/pom.xml index 7b35922a2f..41022d30da 100644 --- a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/pom.xml +++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/pom.xml @@ -73,13 +73,6 @@ provided - - org.json4s - json4s-jackson_${scala.binary.version} - ${json4s.version} - provided - - diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/execute/ComputationExecutor.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/execute/ComputationExecutor.scala index b156ea9aa0..5a06ca007f 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/execute/ComputationExecutor.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/execute/ComputationExecutor.scala @@ -95,7 +95,7 @@ abstract class ComputationExecutor(val outputPrintLimit: Int = 1000) protected val failedTasks: Count = new Count - private var lastTask: EngineConnTask = _ + protected var lastTask: EngineConnTask = _ private val MAX_TASK_EXECUTE_NUM = ComputationExecutorConf.ENGINE_MAX_TASK_EXECUTE_NUM.getValue( EngineConnObject.getEngineCreationContext.getOptions diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/pom.xml b/linkis-computation-governance/linkis-manager/linkis-application-manager/pom.xml index a50c2be120..86b661a963 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/pom.xml +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/pom.xml @@ -101,13 +101,6 @@ ${project.version} - - org.json4s - json4s-jackson_${scala.binary.version} - ${json4s.version} - provided - - com.google.code.gson gson diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/rm/external/yarn/YarnAppInfo.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/rm/external/yarn/YarnAppInfo.java new file mode 100644 index 0000000000..00d24d8ac3 --- /dev/null +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/rm/external/yarn/YarnAppInfo.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.manager.rm.external.yarn; + +import org.apache.linkis.manager.common.entity.resource.YarnResource; +import org.apache.linkis.manager.rm.external.domain.ExternalAppInfo; + +public class YarnAppInfo implements ExternalAppInfo { + private String id; + private String user; + private String status; + private String applicationType; + private YarnResource usedResource; + + public YarnAppInfo( + String id, String user, String status, String applicationType, YarnResource usedResource) { + this.id = id; + this.user = user; + this.status = status; + this.applicationType = applicationType; + this.usedResource = usedResource; + } + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public String getUser() { + return user; + } + + public void setUser(String user) { + this.user = user; + } + + public String getStatus() { + return status; + } + + public void setStatus(String status) { + this.status = status; + } + + public String getApplicationType() { + return applicationType; + } + + public void setApplicationType(String applicationType) { + this.applicationType = applicationType; + } + + public YarnResource getUsedResource() { + return usedResource; + } + + public void setUsedResource(YarnResource usedResource) { + this.usedResource = usedResource; + } +} diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/rpc/ManagerRPCFormats.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/rpc/ManagerRPCFormats.scala deleted file mode 100644 index c710cdd303..0000000000 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/rpc/ManagerRPCFormats.scala +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.manager.am.rpc - -import org.apache.linkis.manager.common.entity.resource.ResourceSerializer -import org.apache.linkis.manager.common.serializer.NodeResourceSerializer -import org.apache.linkis.manager.rm.ResultResourceSerializer -import org.apache.linkis.rpc.transform.RPCFormats - -import org.springframework.stereotype.Component - -import org.json4s.Serializer - -@Component -class ManagerRPCFormats extends RPCFormats { - - override def getSerializers: Array[Serializer[_]] = - Array(ResultResourceSerializer, ResourceSerializer, NodeResourceSerializer) - -} diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/utils/AMUtils.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/utils/AMUtils.scala index 5f4b91a245..454c8211a5 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/utils/AMUtils.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/utils/AMUtils.scala @@ -37,14 +37,11 @@ import java.util import scala.collection.JavaConverters._ import com.google.gson.JsonObject -import org.json4s.DefaultFormats -import org.json4s.jackson.Serialization.write object AMUtils { lazy val GSON = BDPJettyServerHelper.gson - implicit val formats = DefaultFormats + ResourceSerializer + NodeResourceSerializer val mapper = BDPJettyServerHelper.jacksonJson def copyToEMVo(EMNodes: Array[EMNode]): util.ArrayList[EMNodeVo] = { @@ -62,26 +59,32 @@ object AMUtils { if (node.getNodeResource.getMaxResource != null) { EMNodeVo.setMaxResource( mapper - .readValue(write(node.getNodeResource.getMaxResource), classOf[util.Map[String, Any]]) + .readValue( + mapper.writeValueAsString(node.getNodeResource.getMaxResource), + classOf[util.Map[String, Any]] + ) ) } if (node.getNodeResource.getMinResource != null) { EMNodeVo.setMinResource( mapper - .readValue(write(node.getNodeResource.getMinResource), classOf[util.Map[String, Any]]) + .readValue( + mapper.writeValueAsString(node.getNodeResource.getMinResource), + classOf[util.Map[String, Any]] + ) ) } if (node.getNodeResource.getUsedResource != null) { EMNodeVo.setUsedResource( mapper.readValue( - write(node.getNodeResource.getUsedResource), + mapper.writeValueAsString(node.getNodeResource.getUsedResource), classOf[util.Map[String, Any]] ) ) } else { EMNodeVo.setUsedResource( mapper.readValue( - write(Resource.initResource(ResourceType.Default)), + mapper.writeValueAsString(Resource.initResource(ResourceType.Default)), classOf[util.Map[String, Any]] ) ) @@ -89,7 +92,7 @@ object AMUtils { if (node.getNodeResource.getLockedResource != null) { EMNodeVo.setLockedResource( mapper.readValue( - write(node.getNodeResource.getLockedResource), + mapper.writeValueAsString(node.getNodeResource.getLockedResource), classOf[util.Map[String, Any]] ) ) @@ -97,7 +100,7 @@ object AMUtils { if (node.getNodeResource.getExpectedResource != null) { EMNodeVo.setExpectedResource( mapper.readValue( - write(node.getNodeResource.getExpectedResource), + mapper.writeValueAsString(node.getNodeResource.getExpectedResource), classOf[util.Map[String, Any]] ) ) @@ -105,7 +108,7 @@ object AMUtils { if (node.getNodeResource.getLeftResource != null) { EMNodeVo.setLeftResource( mapper.readValue( - write(node.getNodeResource.getLeftResource), + mapper.writeValueAsString(node.getNodeResource.getLeftResource), classOf[util.Map[String, Any]] ) ) @@ -172,7 +175,7 @@ object AMUtils { if (!node.getLabels.isEmpty) { val engineTypeLabel = - node.getLabels.asScala.find(_.isInstanceOf[EngineTypeLabel]).getOrElse(null) + node.getLabels.asScala.find(_.isInstanceOf[EngineTypeLabel]).orNull if (engineTypeLabel != null) { AMEngineNodeVo.setEngineType( engineTypeLabel.asInstanceOf[EngineTypeLabel] getEngineType @@ -202,12 +205,13 @@ object AMUtils { case _ => node.getNodeResource.getUsedResource } AMEngineNodeVo.setUsedResource( - mapper.readValue(write(realResource), classOf[util.Map[String, Any]]) + mapper + .readValue(mapper.writeValueAsString(realResource), classOf[util.Map[String, Any]]) ) } else { AMEngineNodeVo.setUsedResource( mapper.readValue( - write(Resource.initResource(ResourceType.Default)), + mapper.writeValueAsString(Resource.initResource(ResourceType.Default)), classOf[util.Map[String, Any]] ) ) @@ -215,7 +219,7 @@ object AMUtils { if (node.getNodeResource.getLockedResource != null) { AMEngineNodeVo.setLockedResource( mapper.readValue( - write(node.getNodeResource.getLockedResource), + mapper.writeValueAsString(node.getNodeResource.getLockedResource), classOf[util.Map[String, Any]] ) ) @@ -223,7 +227,7 @@ object AMUtils { if (node.getNodeResource.getExpectedResource != null) { AMEngineNodeVo.setExpectedResource( mapper.readValue( - write(node.getNodeResource.getExpectedResource), + mapper.writeValueAsString(node.getNodeResource.getExpectedResource), classOf[util.Map[String, Any]] ) ) @@ -231,7 +235,7 @@ object AMUtils { if (node.getNodeResource.getLeftResource != null) { AMEngineNodeVo.setLeftResource( mapper.readValue( - write(node.getNodeResource.getLeftResource), + mapper.writeValueAsString(node.getNodeResource.getLeftResource), classOf[util.Map[String, Any]] ) ) diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/external/yarn/YarnAppInfo.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/external/yarn/YarnAppInfo.scala deleted file mode 100644 index 570e669d72..0000000000 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/external/yarn/YarnAppInfo.scala +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.manager.rm.external.yarn - -import org.apache.linkis.manager.common.entity.resource.YarnResource -import org.apache.linkis.manager.rm.external.domain.ExternalAppInfo - -case class YarnAppInfo( - id: String, - user: String, - status: String, - applicationType: String, - usedResource: YarnResource -) extends ExternalAppInfo diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/service/impl/DefaultReqResourceService.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/service/impl/DefaultReqResourceService.scala index 45670e2df4..5211277002 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/service/impl/DefaultReqResourceService.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/service/impl/DefaultReqResourceService.scala @@ -17,16 +17,12 @@ package org.apache.linkis.manager.rm.service.impl -import org.apache.linkis.manager.common.entity.resource.{ResourceSerializer, ResourceType} +import org.apache.linkis.manager.common.entity.resource.ResourceType import org.apache.linkis.manager.rm.service.{LabelResourceService, RequestResourceService} -import org.json4s.DefaultFormats - class DefaultReqResourceService(labelResourceService: LabelResourceService) extends RequestResourceService(labelResourceService) { - implicit val formats = DefaultFormats + ResourceSerializer - override val resourceType: ResourceType = ResourceType.Default } diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/service/impl/DefaultResourceManager.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/service/impl/DefaultResourceManager.scala index 8bdadf3c1d..457bae9bed 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/service/impl/DefaultResourceManager.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/service/impl/DefaultResourceManager.scala @@ -828,7 +828,7 @@ class DefaultResourceManager extends ResourceManager with Logging with Initializ */ override def getResourceInfo(serviceInstances: Array[ServiceInstance]): ResourceInfo = { - val resourceInfo = new ResourceInfo(Lists.newArrayList()) + val resourceInfo = ResourceInfo(Lists.newArrayList()) serviceInstances.foreach({ serviceInstance => val rmNode = new InfoRMNode var aggregatedResource: NodeResource = null diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/service/impl/DriverAndYarnReqResourceService.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/service/impl/DriverAndYarnReqResourceService.scala index 9249136f25..1039fec795 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/service/impl/DriverAndYarnReqResourceService.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/service/impl/DriverAndYarnReqResourceService.scala @@ -37,15 +37,11 @@ import org.apache.linkis.manager.rm.utils.AcrossClusterRulesJudgeUtils.{ import org.apache.commons.lang3.StringUtils -import org.json4s.DefaultFormats - class DriverAndYarnReqResourceService( labelResourceService: LabelResourceService, externalResourceService: ExternalResourceService ) extends RequestResourceService(labelResourceService) { - implicit val formats = DefaultFormats + ResourceSerializer - override val resourceType: ResourceType = DriverAndYarn override def canRequestResource( diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/utils/RMUtils.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/utils/RMUtils.scala index 32244ea8e8..41618649fd 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/utils/RMUtils.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/utils/RMUtils.scala @@ -17,8 +17,8 @@ package org.apache.linkis.manager.rm.utils -import org.apache.linkis.common.conf.{CommonVars, Configuration, TimeType} -import org.apache.linkis.common.utils.{ByteTimeUtils, Logging, Utils} +import org.apache.linkis.common.conf.{CommonVars, TimeType} +import org.apache.linkis.common.utils.{ByteTimeUtils, JsonUtils, Logging, Utils} import org.apache.linkis.manager.am.vo.EMNodeVo import org.apache.linkis.manager.common.constant.RMConstant import org.apache.linkis.manager.common.entity.persistence.{ @@ -26,7 +26,6 @@ import org.apache.linkis.manager.common.entity.persistence.{ PersistenceResource } import org.apache.linkis.manager.common.entity.resource.{Resource, _} -import org.apache.linkis.manager.common.serializer.NodeResourceSerializer import org.apache.linkis.manager.common.utils.ResourceUtils import org.apache.linkis.manager.label.LabelManagerUtils.labelFactory import org.apache.linkis.manager.label.builder.CombinedLabelBuilder @@ -34,7 +33,6 @@ import org.apache.linkis.manager.label.builder.factory.LabelBuilderFactoryContex import org.apache.linkis.manager.label.entity.engine.{EngineType, EngineTypeLabel, UserCreatorLabel} import org.apache.linkis.manager.rm.conf.ResourceStatus import org.apache.linkis.manager.rm.restful.vo.{UserCreatorEngineType, UserResourceVo} -import org.apache.linkis.server.BDPJettyServerHelper import org.apache.commons.lang3.StringUtils @@ -44,13 +42,10 @@ import java.util.{List, UUID} import scala.collection.JavaConverters.asScalaBufferConverter import com.google.common.collect.Lists -import org.json4s.DefaultFormats -import org.json4s.jackson.Serialization.{read, write} object RMUtils extends Logging { - implicit val formats = DefaultFormats + ResourceSerializer + NodeResourceSerializer - val mapper = BDPJettyServerHelper.jacksonJson + val jacksonUtil = JsonUtils.jackson val MANAGER_KILL_ENGINE_EAIT = CommonVars("wds.linkis.manager.rm.kill.engine.wait", new TimeType("30s")) @@ -77,11 +72,11 @@ object RMUtils extends Logging { val RM_RESOURCE_ACTION_RECORD = CommonVars("wds.linkis.manager.rm.resource.action.record", true) def deserializeResource(plainResource: String): Resource = { - read[Resource](plainResource) + jacksonUtil.readValue(plainResource, classOf[Resource]) } def serializeResource(resource: Resource): String = { - write(resource) + jacksonUtil.writeValueAsString(resource) } def toUserResourceVo(userResource: UserResource): UserResourceVo = { @@ -102,27 +97,42 @@ object RMUtils extends Logging { if (userResource.getId != null) userResourceVo.setId(userResource.getId) if (userResource.getUsedResource != null) { userResourceVo.setUsedResource( - mapper.readValue(write(userResource.getUsedResource), classOf[util.Map[String, Any]]) + jacksonUtil.readValue( + jacksonUtil.writeValueAsString(userResource.getUsedResource), + classOf[util.Map[String, Any]] + ) ) } if (userResource.getLeftResource != null) { userResourceVo.setLeftResource( - mapper.readValue(write(userResource.getLeftResource), classOf[util.Map[String, Any]]) + jacksonUtil.readValue( + jacksonUtil.writeValueAsString(userResource.getLeftResource), + classOf[util.Map[String, Any]] + ) ) } if (userResource.getLockedResource != null) { userResourceVo.setLockedResource( - mapper.readValue(write(userResource.getLockedResource), classOf[util.Map[String, Any]]) + jacksonUtil.readValue( + jacksonUtil.writeValueAsString(userResource.getLockedResource), + classOf[util.Map[String, Any]] + ) ) } if (userResource.getMaxResource != null) { userResourceVo.setMaxResource( - mapper.readValue(write(userResource.getMaxResource), classOf[util.Map[String, Any]]) + jacksonUtil.readValue( + jacksonUtil.writeValueAsString(userResource.getMaxResource), + classOf[util.Map[String, Any]] + ) ) } if (userResource.getMinResource != null) { userResourceVo.setMinResource( - mapper.readValue(write(userResource.getMinResource), classOf[util.Map[String, Any]]) + jacksonUtil.readValue( + jacksonUtil.writeValueAsString(userResource.getMinResource), + classOf[util.Map[String, Any]] + ) ) } if (userResource.getResourceType != null) { @@ -202,9 +212,9 @@ object RMUtils extends Logging { return null } if (firstNodeResource == null) { - return secondNodeResource.asInstanceOf[CommonNodeResource] + secondNodeResource.asInstanceOf[CommonNodeResource] } else { - return firstNodeResource.asInstanceOf[CommonNodeResource] + firstNodeResource.asInstanceOf[CommonNodeResource] } } diff --git a/linkis-computation-governance/linkis-manager/linkis-manager-common/pom.xml b/linkis-computation-governance/linkis-manager/linkis-manager-common/pom.xml index 08822a5c45..a3884163fc 100644 --- a/linkis-computation-governance/linkis-manager/linkis-manager-common/pom.xml +++ b/linkis-computation-governance/linkis-manager/linkis-manager-common/pom.xml @@ -52,20 +52,6 @@ ${project.version} provided - - - org.json4s - json4s-jackson_${scala.binary.version} - ${json4s.version} - provided - - - org.scala-lang - scala-library - - - - diff --git a/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/scala/org/apache/linkis/manager/common/entity/resource/Resource.scala b/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/scala/org/apache/linkis/manager/common/entity/resource/Resource.scala index 0cfb4ae055..6a8c27ea92 100644 --- a/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/scala/org/apache/linkis/manager/common/entity/resource/Resource.scala +++ b/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/scala/org/apache/linkis/manager/common/entity/resource/Resource.scala @@ -28,10 +28,7 @@ import java.text.MessageFormat import scala.collection.JavaConverters._ -import org.json4s.{CustomSerializer, DefaultFormats, Extraction} -import org.json4s.JsonAST.JObject -import org.json4s.JsonDSL._ -import org.json4s.jackson.Serialization +import com.fasterxml.jackson.annotation.JsonIgnoreProperties abstract class Resource { def add(r: Resource): Resource @@ -131,7 +128,9 @@ object Resource extends Logging { case class UserAvailableResource(moduleName: String, resource: Resource) +@JsonIgnoreProperties(ignoreUnknown = true) class MemoryResource(val memory: Long) extends Resource { + def this() = this(Long.MaxValue) private implicit def toMemoryResource(r: Resource): MemoryResource = r match { case t: MemoryResource => t @@ -170,7 +169,9 @@ class MemoryResource(val memory: Long) extends Resource { } +@JsonIgnoreProperties(ignoreUnknown = true) class CPUResource(val cores: Int) extends Resource { + def this() = this(Integer.MAX_VALUE) private implicit def toCPUResource(r: Resource): CPUResource = r match { case t: CPUResource => t @@ -210,7 +211,9 @@ class CPUResource(val cores: Int) extends Resource { override def toString: String = toJson } +@JsonIgnoreProperties(ignoreUnknown = true) class LoadResource(val memory: Long, val cores: Int) extends Resource { + def this() = this(Long.MaxValue, Integer.MAX_VALUE) private implicit def toLoadResource(r: Resource): LoadResource = r match { case t: LoadResource => t @@ -255,7 +258,9 @@ class LoadResource(val memory: Long, val cores: Int) extends Resource { override def toString: String = toJson } +@JsonIgnoreProperties(ignoreUnknown = true) class LoadInstanceResource(val memory: Long, val cores: Int, val instances: Int) extends Resource { + def this() = this(Long.MaxValue, Integer.MAX_VALUE, Integer.MAX_VALUE) implicit def toLoadInstanceResource(r: Resource): LoadInstanceResource = r match { case t: LoadInstanceResource => t @@ -308,6 +313,7 @@ class LoadInstanceResource(val memory: Long, val cores: Int, val instances: Int) } +@JsonIgnoreProperties(ignoreUnknown = true) class InstanceResource(val instances: Int) extends CPUResource(instances) { override protected def toResource(cores: Int): Resource = new InstanceResource(cores) @@ -325,6 +331,7 @@ class InstanceResource(val instances: Int) extends CPUResource(instances) { * @param queueCores * @param queueInstances */ +@JsonIgnoreProperties(ignoreUnknown = true) class YarnResource( val queueMemory: Long, val queueCores: Int, @@ -333,6 +340,8 @@ class YarnResource( val applicationId: String = "" ) extends Resource { + def this() = this(Long.MaxValue, Integer.MAX_VALUE, Integer.MAX_VALUE, "default") + implicit def toYarnResource(r: Resource): YarnResource = r match { case t: YarnResource => t case _ => new YarnResource(Long.MaxValue, Integer.MAX_VALUE, Integer.MAX_VALUE, "default") @@ -409,12 +418,18 @@ class YarnResource( } +@JsonIgnoreProperties(ignoreUnknown = true) class DriverAndYarnResource( val loadInstanceResource: LoadInstanceResource, val yarnResource: YarnResource ) extends Resource with Logging { + def this() = this( + new LoadInstanceResource(Long.MaxValue, Integer.MAX_VALUE, Integer.MAX_VALUE), + new YarnResource(Long.MaxValue, Integer.MAX_VALUE, Integer.MAX_VALUE) + ) + private implicit def DriverAndYarnResource(r: Resource): DriverAndYarnResource = r match { case t: DriverAndYarnResource => t case y: YarnResource => new DriverAndYarnResource(new LoadInstanceResource(0, 0, 0), y) @@ -556,6 +571,7 @@ class DriverAndYarnResource( } +@JsonIgnoreProperties(ignoreUnknown = true) class SpecialResource(val resources: java.util.Map[String, AnyVal]) extends Resource { def this(resources: Map[String, AnyVal]) = this(resources.asJava) @@ -754,95 +770,3 @@ class SpecialResource(val resources: java.util.Map[String, AnyVal]) extends Reso override def toJson: String = s"Special:$resources" } - -object ResourceSerializer - extends CustomSerializer[Resource](implicit formats => - ( - { - case JObject(List(("memory", memory))) => new MemoryResource(memory.extract[Long]) - case JObject(List(("cores", cores))) => new CPUResource(cores.extract[Int]) - case JObject(List(("instance", instances))) => - new InstanceResource(instances.extract[Int]) - case JObject(List(("memory", memory), ("cores", cores))) => - new LoadResource(memory.extract[Long], cores.extract[Int]) - case JObject(List(("memory", memory), ("cores", cores), ("instance", instances))) => - new LoadInstanceResource( - memory.extract[Long], - cores.extract[Int], - instances.extract[Int] - ) - case JObject( - List( - ("applicationId", applicationId), - ("queueName", queueName), - ("queueMemory", queueMemory), - ("queueCores", queueCores), - ("queueInstances", queueInstances) - ) - ) => - new YarnResource( - queueMemory.extract[Long], - queueCores.extract[Int], - queueInstances.extract[Int], - queueName.extract[String], - applicationId.extract[String] - ) - case JObject( - List( - ( - "DriverAndYarnResource", - JObject( - List( - ("loadInstanceResource", loadInstanceResource), - ("yarnResource", yarnResource) - ) - ) - ) - ) - ) => - implicit val formats = DefaultFormats - new DriverAndYarnResource( - loadInstanceResource.extract[LoadInstanceResource], - yarnResource.extract[YarnResource] - ) - case JObject(List(("resources", resources))) => - new SpecialResource(resources.extract[Map[String, AnyVal]]) - case JObject(list) => - throw new ResourceWarnException( - NOT_RESOURCE_STRING.getErrorCode, - NOT_RESOURCE_STRING.getErrorDesc + list - ) - }, - { - case m: MemoryResource => ("memory", m.memory) - case c: CPUResource => ("cores", c.cores) - case i: InstanceResource => ("instance", i.instances) - case l: LoadResource => ("memory", l.memory) ~ ("cores", l.cores) - case li: LoadInstanceResource => - ("memory", li.memory) ~ ("cores", li.cores) ~ ("instance", li.instances) - case yarn: YarnResource => - ( - "applicationId", - yarn.applicationId - ) ~ ("queueName", yarn.queueName) ~ ("queueMemory", yarn.queueMemory) ~ ("queueCores", yarn.queueCores) ~ ("queueInstances", yarn.queueInstances) - case dy: DriverAndYarnResource => - implicit val formats = DefaultFormats - ( - "DriverAndYarnResource", - new JObject( - List( - ("loadInstanceResource", Extraction.decompose(dy.loadInstanceResource)), - ("yarnResource", Extraction.decompose(dy.yarnResource)) - ) - ) - ) - case s: SpecialResource => - ("resources", Serialization.write(s.resources.asScala.toMap)) - case r: Resource => - throw new ResourceWarnException( - NOT_RESOURCE_TYPE.getErrorCode, - MessageFormat.format(NOT_RESOURCE_TYPE.getErrorDesc, r.getClass) - ) - } - ) - ) diff --git a/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/scala/org/apache/linkis/manager/common/serializer/NodeResourceSerializer.scala b/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/scala/org/apache/linkis/manager/common/serializer/NodeResourceSerializer.scala deleted file mode 100644 index e8a54ea73a..0000000000 --- a/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/scala/org/apache/linkis/manager/common/serializer/NodeResourceSerializer.scala +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.manager.common.serializer - -import org.apache.linkis.manager.common.entity.resource._ - -import org.json4s.{CustomSerializer, DefaultFormats, Extraction} -import org.json4s.JsonAST.JObject -import org.json4s.JsonDSL._ - -object NodeResourceSerializer - extends CustomSerializer[NodeResource](implicit formats => - ( - { - case JObject( - List( - ("resourceType", resourceType), - ("maxResource", maxResource), - ("minResource", minResource), - ("usedResource", usedResource), - ("lockedResource", lockedResource), - ("expectedResource", expectedResource), - ("leftResource", leftResource) - ) - ) => - val resource = new CommonNodeResource - resource.setResourceType(ResourceType.valueOf(resourceType.extract[String])) - resource.setMaxResource(maxResource.extract[Resource]) - resource.setMinResource(minResource.extract[Resource]) - resource.setUsedResource(usedResource.extract[Resource]) - resource.setLockedResource(lockedResource.extract[Resource]) - resource.setExpectedResource(expectedResource.extract[Resource]) - resource.setLeftResource(leftResource.extract[Resource]) - resource - }, - { case c: CommonNodeResource => - implicit val formats = DefaultFormats + ResourceSerializer - ("resourceType", c.getResourceType.toString) ~ - ("maxResource", Extraction.decompose(c.getMaxResource)) ~ - ("minResource", Extraction.decompose(c.getMinResource)) ~ - ("usedResource", Extraction.decompose(c.getUsedResource)) ~ - ("lockedResource", Extraction.decompose(c.getLockedResource)) ~ - ("expectedResource", Extraction.decompose(c.getExpectedResource)) ~ - ("leftResource", Extraction.decompose(c.getLeftResource)) - } - ) - ) diff --git a/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/scala/org/apache/linkis/manager/common/serializer/RegisterEMRequestSerializer.scala b/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/scala/org/apache/linkis/manager/common/serializer/RegisterEMRequestSerializer.scala deleted file mode 100644 index 223e75b767..0000000000 --- a/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/scala/org/apache/linkis/manager/common/serializer/RegisterEMRequestSerializer.scala +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.manager.common.serializer - -import org.apache.linkis.common.ServiceInstance -import org.apache.linkis.manager.common.entity.resource.{CommonNodeResource, ResourceSerializer} -import org.apache.linkis.manager.common.protocol.em.RegisterEMRequest - -import org.json4s.{CustomSerializer, DefaultFormats, Extraction} -import org.json4s.JsonAST.JObject -import org.json4s.JsonDSL._ - -object RegisterEMRequestSerializer - extends CustomSerializer[RegisterEMRequest](implicit formats => - ( - { - case JObject( - List( - ("serviceInstance", serviceInstance), - ("labels", labels), - ("nodeResource", nodeResource), - ("user", user), - ("alias", alias) - ) - ) => - val registerEMRequest = new RegisterEMRequest - registerEMRequest.setServiceInstance(serviceInstance.extract[ServiceInstance]) - registerEMRequest.setAlias(alias.extract[String]) - // registerEMRequest.setLabels(labels.extract[java.util.HashMap[String, Object]]) - registerEMRequest.setUser(user.extract[String]) - registerEMRequest.setNodeResource(nodeResource.extract[CommonNodeResource]) - registerEMRequest - }, - { case c: RegisterEMRequest => - implicit val formats = DefaultFormats + ResourceSerializer + NodeResourceSerializer - ("serviceInstance", Extraction.decompose(c.getServiceInstance)) ~ - ("labels", Extraction.decompose(c.getLabels)) ~ - ("nodeResource", Extraction.decompose(c.getNodeResource)) ~ - ("user", c.getUser) ~ - ("alias", c.getAlias) - } - ) - ) diff --git a/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/scala/org/apache/linkis/manager/common/utils/ResourceUtils.scala b/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/scala/org/apache/linkis/manager/common/utils/ResourceUtils.scala index 959c437524..99a61cf490 100644 --- a/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/scala/org/apache/linkis/manager/common/utils/ResourceUtils.scala +++ b/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/scala/org/apache/linkis/manager/common/utils/ResourceUtils.scala @@ -17,22 +17,36 @@ package org.apache.linkis.manager.common.utils +import org.apache.linkis.common.utils.JsonUtils import org.apache.linkis.manager.common.entity.persistence.PersistenceResource import org.apache.linkis.manager.common.entity.resource._ -import org.json4s.DefaultFormats -import org.json4s.jackson.Serialization.{read, write} - object ResourceUtils { - implicit val formats = DefaultFormats + ResourceSerializer - - def deserializeResource(plainResource: String): Resource = { - read[Resource](plainResource) + def deserializeResource(plainResource: String, resourceType: ResourceType): Resource = { + if (resourceType.equals(ResourceType.CPU)) { + JsonUtils.jackson.readValue(plainResource, classOf[CPUResource]) + } else if (resourceType.equals(ResourceType.DriverAndYarn)) { + JsonUtils.jackson.readValue(plainResource, classOf[DriverAndYarnResource]) + } else if (resourceType.equals(ResourceType.Instance)) { + JsonUtils.jackson.readValue(plainResource, classOf[InstanceResource]) + } else if (resourceType.equals(ResourceType.LoadInstance)) { + JsonUtils.jackson.readValue(plainResource, classOf[LoadInstanceResource]) + } else if (resourceType.equals(ResourceType.Load)) { + JsonUtils.jackson.readValue(plainResource, classOf[LoadResource]) + } else if (resourceType.equals(ResourceType.Memory)) { + JsonUtils.jackson.readValue(plainResource, classOf[MemoryResource]) + } else if (resourceType.equals(ResourceType.Special)) { + JsonUtils.jackson.readValue(plainResource, classOf[SpecialResource]) + } else if (resourceType.equals(ResourceType.Yarn)) { + JsonUtils.jackson.readValue(plainResource, classOf[YarnResource]) + } else { + JsonUtils.jackson.readValue(plainResource, classOf[LoadResource]) + } } def serializeResource(resource: Resource): String = { - write(resource) + JsonUtils.jackson.writeValueAsString(resource) } def toPersistenceResource(nodeResource: NodeResource): PersistenceResource = { @@ -65,24 +79,37 @@ object ResourceUtils { def fromPersistenceResource(persistenceResource: PersistenceResource): CommonNodeResource = { if (persistenceResource == null) return null val nodeResource = new CommonNodeResource + val resourceType = ResourceType.valueOf(persistenceResource.getResourceType) nodeResource.setId(persistenceResource.getId) if (persistenceResource.getMaxResource != null) { - nodeResource.setMaxResource(deserializeResource(persistenceResource.getMaxResource)) + nodeResource.setMaxResource( + deserializeResource(persistenceResource.getMaxResource, resourceType) + ) } if (persistenceResource.getMinResource != null) { - nodeResource.setMinResource(deserializeResource(persistenceResource.getMinResource)) + nodeResource.setMinResource( + deserializeResource(persistenceResource.getMinResource, resourceType) + ) } if (persistenceResource.getLockedResource != null) { - nodeResource.setLockedResource(deserializeResource(persistenceResource.getLockedResource)) + nodeResource.setLockedResource( + deserializeResource(persistenceResource.getLockedResource, resourceType) + ) } if (persistenceResource.getExpectedResource != null) { - nodeResource.setExpectedResource(deserializeResource(persistenceResource.getExpectedResource)) + nodeResource.setExpectedResource( + deserializeResource(persistenceResource.getExpectedResource, resourceType) + ) } if (persistenceResource.getLeftResource != null) { - nodeResource.setLeftResource(deserializeResource(persistenceResource.getLeftResource)) + nodeResource.setLeftResource( + deserializeResource(persistenceResource.getLeftResource, resourceType) + ) } if (persistenceResource.getUsedResource != null) { - nodeResource.setUsedResource(deserializeResource(persistenceResource.getUsedResource)) + nodeResource.setUsedResource( + deserializeResource(persistenceResource.getUsedResource, resourceType) + ) } if (persistenceResource.getCreateTime != null) { nodeResource.setCreateTime(persistenceResource.getCreateTime) @@ -90,31 +117,44 @@ object ResourceUtils { if (persistenceResource.getUpdateTime != null) { nodeResource.setUpdateTime(persistenceResource.getUpdateTime) } - nodeResource.setResourceType(ResourceType.valueOf(persistenceResource.getResourceType)) + nodeResource.setResourceType(resourceType) nodeResource } def fromPersistenceResourceAndUser(persistenceResource: PersistenceResource): UserResource = { if (persistenceResource == null) return null val nodeResource = new UserResource + val resourceType = ResourceType.valueOf(persistenceResource.getResourceType) nodeResource.setId(persistenceResource.getId) if (persistenceResource.getMaxResource != null) { - nodeResource.setMaxResource(deserializeResource(persistenceResource.getMaxResource)) + nodeResource.setMaxResource( + deserializeResource(persistenceResource.getMaxResource, resourceType) + ) } if (persistenceResource.getMinResource != null) { - nodeResource.setMinResource(deserializeResource(persistenceResource.getMinResource)) + nodeResource.setMinResource( + deserializeResource(persistenceResource.getMinResource, resourceType) + ) } if (persistenceResource.getLockedResource != null) { - nodeResource.setLockedResource(deserializeResource(persistenceResource.getLockedResource)) + nodeResource.setLockedResource( + deserializeResource(persistenceResource.getLockedResource, resourceType) + ) } if (persistenceResource.getExpectedResource != null) { - nodeResource.setExpectedResource(deserializeResource(persistenceResource.getExpectedResource)) + nodeResource.setExpectedResource( + deserializeResource(persistenceResource.getExpectedResource, resourceType) + ) } if (persistenceResource.getLeftResource != null) { - nodeResource.setLeftResource(deserializeResource(persistenceResource.getLeftResource)) + nodeResource.setLeftResource( + deserializeResource(persistenceResource.getLeftResource, resourceType) + ) } if (persistenceResource.getUsedResource != null) { - nodeResource.setUsedResource(deserializeResource(persistenceResource.getUsedResource)) + nodeResource.setUsedResource( + deserializeResource(persistenceResource.getUsedResource, resourceType) + ) } if (persistenceResource.getCreateTime != null) { nodeResource.setCreateTime(persistenceResource.getCreateTime) @@ -122,7 +162,7 @@ object ResourceUtils { if (persistenceResource.getUpdateTime != null) { nodeResource.setUpdateTime(persistenceResource.getUpdateTime) } - nodeResource.setResourceType(ResourceType.valueOf(persistenceResource.getResourceType)) + nodeResource.setResourceType(resourceType) nodeResource } diff --git a/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/scala/org/apache/linkis/manager/rm/ResultResource.scala b/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/scala/org/apache/linkis/manager/rm/ResultResource.scala index 017cb189ac..619e7bd221 100644 --- a/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/scala/org/apache/linkis/manager/rm/ResultResource.scala +++ b/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/scala/org/apache/linkis/manager/rm/ResultResource.scala @@ -17,30 +17,8 @@ package org.apache.linkis.manager.rm -import org.json4s.{CustomSerializer, Extraction} -import org.json4s.JsonAST.JObject -import org.json4s.JsonDSL._ - trait ResultResource case class NotEnoughResource(val reason: String = null) extends ResultResource case class AvailableResource(val ticketId: String) extends ResultResource - -object ResultResourceSerializer - extends CustomSerializer[ResultResource](implicit formats => - ( - { - case JObject(List(("NotEnoughResource", JObject(List(("reason", reason)))))) => - NotEnoughResource(reason.extract[String]) - case JObject(List(("AvailableResource", JObject(List(("ticketId", ticketId)))))) => - AvailableResource(ticketId.extract[String]) - }, - { - case r: NotEnoughResource => - ("NotEnoughResource", ("reason", Extraction.decompose(r.reason))) - case r: AvailableResource => - ("AvailableResource", ("ticketId", Extraction.decompose(r.ticketId))) - } - ) - ) diff --git a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/executor/SparkSqlExecutor.scala b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/executor/SparkSqlExecutor.scala index 33b3dadb38..bbaaa700dd 100644 --- a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/executor/SparkSqlExecutor.scala +++ b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/executor/SparkSqlExecutor.scala @@ -25,13 +25,10 @@ import org.apache.linkis.engineplugin.spark.entity.SparkEngineSession import org.apache.linkis.engineplugin.spark.utils.EngineUtils import org.apache.linkis.governance.common.constant.job.JobRequestConstants import org.apache.linkis.governance.common.paser.SQLCodeParser -import org.apache.linkis.scheduler.executer.{ - ErrorExecuteResponse, - ExecuteResponse, - SuccessExecuteResponse -} - +import org.apache.linkis.scheduler.executer.{ErrorExecuteResponse, ExecuteResponse, SuccessExecuteResponse} import org.apache.commons.lang3.exception.ExceptionUtils +import org.apache.linkis.engineconn.computation.executor.entity.EngineConnTask +import org.apache.linkis.governance.common.entity.ExecutionNodeStatus import java.lang.reflect.InvocationTargetException @@ -89,6 +86,7 @@ class SparkSqlExecutor(sparkEngineSession: SparkEngineSession, id: Long) ) ) ) + transformTaskStatus(lastTask, ExecutionNodeStatus.Succeed) SQLSession.showDF( sparkEngineSession.sparkContext, jobGroup, From adf9b5dab9c383edc44f2b13ceea1146e5a3282a Mon Sep 17 00:00:00 2001 From: peacewong Date: Tue, 20 Aug 2024 21:20:23 +0800 Subject: [PATCH 02/90] =?UTF-8?q?=E4=BF=AE=E5=A4=8Djson4s=E9=97=AE?= =?UTF-8?q?=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../rm/external/yarn/YarnQueueInfo.java | 86 ++++ .../external/yarn/YarnResourceRequester.java | 472 ++++++++++++++++++ .../external/yarn/YarnResourceRequester.scala | 434 ---------------- .../spark/executor/SparkSqlExecutor.scala | 11 +- 4 files changed, 566 insertions(+), 437 deletions(-) create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/rm/external/yarn/YarnQueueInfo.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/rm/external/yarn/YarnResourceRequester.java delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/external/yarn/YarnResourceRequester.scala diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/rm/external/yarn/YarnQueueInfo.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/rm/external/yarn/YarnQueueInfo.java new file mode 100644 index 0000000000..92ca997869 --- /dev/null +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/rm/external/yarn/YarnQueueInfo.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.manager.rm.external.yarn; + +import org.apache.linkis.manager.common.entity.resource.YarnResource; + +public class YarnQueueInfo { + + public YarnQueueInfo( + YarnResource maxResource, + YarnResource usedResource, + int maxApps, + int numPendingApps, + int numActiveApps) { + this.maxResource = maxResource; + this.usedResource = usedResource; + this.maxApps = maxApps; + this.numPendingApps = numPendingApps; + this.numActiveApps = numActiveApps; + } + + private YarnResource maxResource; + + private YarnResource usedResource; + + private int maxApps; + + private int numPendingApps; + + private int numActiveApps; + + public YarnResource getMaxResource() { + return maxResource; + } + + public void setMaxResource(YarnResource maxResource) { + this.maxResource = maxResource; + } + + public YarnResource getUsedResource() { + return usedResource; + } + + public void setUsedResource(YarnResource usedResource) { + this.usedResource = usedResource; + } + + public int getMaxApps() { + return maxApps; + } + + public void setMaxApps(int maxApps) { + this.maxApps = maxApps; + } + + public int getNumPendingApps() { + return numPendingApps; + } + + public void setNumPendingApps(int numPendingApps) { + this.numPendingApps = numPendingApps; + } + + public int getNumActiveApps() { + return numActiveApps; + } + + public void setNumActiveApps(int numActiveApps) { + this.numActiveApps = numActiveApps; + } +} diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/rm/external/yarn/YarnResourceRequester.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/rm/external/yarn/YarnResourceRequester.java new file mode 100644 index 0000000000..ffc61c52bc --- /dev/null +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/rm/external/yarn/YarnResourceRequester.java @@ -0,0 +1,472 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.manager.rm.external.yarn; + +import org.apache.linkis.manager.am.util.LinkisUtils; +import org.apache.linkis.manager.common.conf.RMConfiguration; +import org.apache.linkis.manager.common.entity.resource.CommonNodeResource; +import org.apache.linkis.manager.common.entity.resource.NodeResource; +import org.apache.linkis.manager.common.entity.resource.ResourceType; +import org.apache.linkis.manager.common.entity.resource.YarnResource; +import org.apache.linkis.manager.common.exception.RMErrorException; +import org.apache.linkis.manager.common.exception.RMWarnException; +import org.apache.linkis.manager.rm.external.domain.ExternalAppInfo; +import org.apache.linkis.manager.rm.external.domain.ExternalResourceIdentifier; +import org.apache.linkis.manager.rm.external.domain.ExternalResourceProvider; +import org.apache.linkis.manager.rm.external.request.ExternalResourceRequester; +import org.apache.linkis.manager.rm.utils.RequestKerberosUrlUtils; + +import org.apache.commons.lang3.StringUtils; +import org.apache.http.HttpHeaders; +import org.apache.http.HttpResponse; +import org.apache.http.client.HttpClient; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.util.EntityUtils; + +import java.io.IOException; +import java.text.MessageFormat; +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static org.apache.linkis.manager.common.errorcode.ManagerCommonErrorCodeSummary.*; + +public class YarnResourceRequester implements ExternalResourceRequester { + private static final Logger logger = LoggerFactory.getLogger(YarnResourceRequester.class); + + private final String HASTATE_ACTIVE = "active"; + private static final ObjectMapper objectMapper = new ObjectMapper(); + private final Map rmAddressMap = new ConcurrentHashMap<>(); + + private static final HttpClient httpClient = HttpClients.createDefault(); + + private String getAuthorizationStr(ExternalResourceProvider provider) { + String user = (String) provider.getConfigMap().getOrDefault("user", ""); + String pwd = (String) provider.getConfigMap().getOrDefault("pwd", ""); + String authKey = user + ":" + pwd; + return Base64.getMimeEncoder().encodeToString(authKey.getBytes()); + } + + @Override + public NodeResource requestResourceInfo( + ExternalResourceIdentifier identifier, ExternalResourceProvider provider) { + String rmWebAddress = getAndUpdateActiveRmWebAddress(provider); + logger.info("rmWebAddress: " + rmWebAddress); + + String queueName = ((YarnResourceIdentifier) identifier).getQueueName(); + String realQueueName = "root." + queueName; + + return LinkisUtils.tryCatch( + () -> { + YarnQueueInfo resources = getResources(rmWebAddress, realQueueName, queueName, provider); + + CommonNodeResource nodeResource = new CommonNodeResource(); + nodeResource.setMaxResource(resources.getMaxResource()); + nodeResource.setUsedResource(resources.getUsedResource()); + nodeResource.setMaxApps(resources.getMaxApps()); + nodeResource.setNumPendingApps(resources.getNumPendingApps()); + nodeResource.setNumActiveApps(resources.getNumActiveApps()); + return nodeResource; + }, + t -> { + throw new RMErrorException( + YARN_QUEUE_EXCEPTION.getErrorCode(), YARN_QUEUE_EXCEPTION.getErrorDesc(), t); + }); + } + + public Optional maxEffectiveHandle( + Optional queueValue, + String rmWebAddress, + String queueName, + ExternalResourceProvider provider) { + try { + JsonNode metrics = getResponseByUrl("metrics", rmWebAddress, provider); + JsonNode clusterMetrics = metrics.path("clusterMetrics"); + long totalMemory = clusterMetrics.path("totalMB").asLong(); + long totalCores = clusterMetrics.path("totalVirtualCores").asLong(); + if (queueValue.isPresent()) { + JsonNode jsonNode = queueValue.get(); + double absoluteCapacity = jsonNode.path("absoluteCapacity").asDouble(); + + YarnResource yarnResource = + new YarnResource( + (long) Math.floor(absoluteCapacity * totalMemory * 1024L * 1024L / 100), + (int) Math.floor(absoluteCapacity * totalCores / 100), + 0, + queueName, + ""); + return Optional.of(yarnResource); + } + return Optional.empty(); + } catch (Exception e) { + logger.warn("maxEffectiveHandle parse failed", e); + return Optional.empty(); + } + } + + public static Optional getQueue(JsonNode queues, String realQueueName) { + if (queues instanceof ArrayNode) { + for (JsonNode q : (ArrayNode) queues) { + String yarnQueueName = q.get("queueName").asText(); + if (yarnQueueName.equals(realQueueName)) { + return Optional.of(q); + } else if (realQueueName.startsWith(yarnQueueName + ".")) { + Optional childQueue = getQueue(getChildQueues(q), realQueueName); + if (childQueue.isPresent()) { + return childQueue; + } + } + } + return Optional.empty(); + } else if (queues instanceof ObjectNode) { + ObjectNode queueObj = (ObjectNode) queues; + JsonNode queueName = queueObj.get("queueName"); + if (queueName != null && queueName.asText().equals(realQueueName)) { + return Optional.of(queueObj); + } else { + JsonNode childQueues = queueObj.get("childQueues"); + if (childQueues == null) { + return Optional.empty(); + } else { + return getQueue(childQueues, realQueueName); + } + } + } else { + return Optional.empty(); + } + } + + public static JsonNode getChildQueues(JsonNode resp) { + JsonNode queues = resp.get("childQueues").get("queue"); + if (queues != null + && !queues.isNull() + && !queues.isMissingNode() + && queues.isArray() + && ((ArrayNode) queues).size() > 0) { + return queues; + } else { + return resp.get("childQueues"); + } + } + + public Optional getQueueOfCapacity(JsonNode queues, String realQueueName) { + if (queues.isArray()) { + for (JsonNode q : queues) { + String yarnQueueName = q.get("queueName").asText(); + if (yarnQueueName.equals(realQueueName)) { + return Optional.of(q); + } else if (q.has("queues")) { + Optional matchQueue = getQueueOfCapacity(q.get("queues"), realQueueName); + if (matchQueue.isPresent()) { + return matchQueue; + } + } + } + return Optional.empty(); + } else if (queues.isObject()) { + if (queues.has("queueName") && queues.get("queueName").asText().equals(realQueueName)) { + return Optional.of(queues); + } else if (queues.has("queues")) { + Optional matchQueue = getQueueOfCapacity(queues.get("queues"), realQueueName); + if (matchQueue.isPresent()) { + return matchQueue; + } + } + return Optional.empty(); + } else { + return Optional.empty(); + } + } + + static JsonNode getChildQueuesOfCapacity(JsonNode resp) { + // = resp \ "queues" \ "queue" + return resp.path("queues").path("queue"); + } + + public YarnQueueInfo getResources( + String rmWebAddress, + String realQueueName, + String queueName, + ExternalResourceProvider provider) { + JsonNode resp = getResponseByUrl("scheduler", rmWebAddress, provider); + JsonNode schedulerInfo = resp.path("scheduler").path("schedulerInfo"); + String schedulerType = schedulerInfo.path("type").asText(); + if ("capacityScheduler".equals(schedulerType)) { + realQueueName = queueName; + JsonNode childQueues = getChildQueuesOfCapacity(schedulerInfo); + Optional queue = getQueueOfCapacity(childQueues, realQueueName); + if (queue == null || !queue.isPresent()) { + logger.debug( + "cannot find any information about queue " + queueName + ", response: " + resp); + throw new RMWarnException( + YARN_NOT_EXISTS_QUEUE.getErrorCode(), + MessageFormat.format(YARN_NOT_EXISTS_QUEUE.getErrorDesc(), queueName)); + } + JsonNode queueInfo = queue.get(); + return new YarnQueueInfo( + maxEffectiveHandle(queue, rmWebAddress, queueName, provider).get(), + getYarnResource(queue.map(node -> node.path("resourcesUsed")), queueName).get(), + queueInfo.path("maxApps").asInt(), + queueInfo.path("numPendingApps").asInt(), + queueInfo.path("numActiveApps").asInt()); + } else if ("fairScheduler".equals(schedulerType)) { + Optional queue; + if ("root".equals(queueName)) { + queue = Optional.of(schedulerInfo.path("rootQueue")); + } else { + JsonNode childQueues = getChildQueues(schedulerInfo.path("rootQueue")); + queue = getQueue(childQueues, realQueueName); + } + if (queue != null || !queue.isPresent()) { + logger.debug( + "cannot find any information about queue " + queueName + ", response: " + resp); + throw new RMWarnException( + YARN_NOT_EXISTS_QUEUE.getErrorCode(), + MessageFormat.format(YARN_NOT_EXISTS_QUEUE.getErrorDesc(), queueName)); + } + Optional maxResources = queue.map(node -> node.path("maxResources")); + Optional usedResources = queue.map(node -> node.path("usedResources")); + JsonNode queueInfo = queue.get(); + int numPendingApps = 0; + int numActiveApps = 0; + if (!"root".equals(queueName)) { + numPendingApps = queueInfo.path("numPendingApps").asInt(); + numActiveApps = queueInfo.path("numActiveApps").asInt(); + } + return new YarnQueueInfo( + getYarnResource(maxResources, queueName).get(), + getYarnResource(usedResources, queueName).get(), + queueInfo.path("maxApps").asInt(), + numPendingApps, + numActiveApps); + } else { + logger.debug( + "only support fairScheduler or capacityScheduler, schedulerType: " + + schedulerType + + ", response: " + + resp); + throw new RMWarnException( + ONLY_SUPPORT_FAIRORCAPA.getErrorCode(), + MessageFormat.format(ONLY_SUPPORT_FAIRORCAPA.getErrorDesc(), schedulerType)); + } + } + + public static Optional getYarnResource( + Optional jsonNode, String queueName) { + if (jsonNode.isPresent()) { + JsonNode r = jsonNode.get(); + return Optional.of( + new YarnResource( + r.get("memory").asLong() * 1024L * 1024L, r.get("vCores").asInt(), 0, queueName, "")); + } + return Optional.empty(); + } + + public static Optional getAllocatedYarnResource( + Optional jsonNode, String queueName) { + if (jsonNode.isPresent()) { + JsonNode r = jsonNode.get(); + return Optional.of( + new YarnResource( + r.get("allocatedMB").asLong() * 1024L * 1024L, + r.get("allocatedVCores").asInt(), + 0, + queueName, + "")); + } + return Optional.empty(); + } + + @Override + public List requestAppInfo( + ExternalResourceIdentifier identifier, ExternalResourceProvider provider) { + + String rmWebAddress = getAndUpdateActiveRmWebAddress(provider); + + String queueName = ((YarnResourceIdentifier) identifier).getQueueName(); + String realQueueName = "root." + queueName; + + JsonNode resp = getResponseByUrl("apps", rmWebAddress, provider).path("apps").path("app"); + if (resp.isMissingNode()) { + return new ArrayList<>(); + } + ArrayNode appArray = (ArrayNode) resp; + + List appInfoList = new ArrayList<>(); + Iterator iterator = appArray.elements(); + while (iterator.hasNext()) { + JsonNode app = iterator.next(); + String yarnQueueName = app.get("queue").asText(); + String state = app.get("state").asText(); + if (yarnQueueName.equals(realQueueName) + && (state.equals("RUNNING") || state.equals("ACCEPTED"))) { + String id = app.get("id").asText(); + String user = app.get("user").asText(); + String applicationType = app.get("applicationType").asText(); + Optional yarnResource = + getAllocatedYarnResource(Optional.ofNullable(app), queueName); + if (yarnResource.isPresent()) { + YarnAppInfo appInfo = + new YarnAppInfo(id, user, state, applicationType, yarnResource.get()); + appInfoList.add(appInfo); + } + } + } + return appInfoList; + } + + @Override + public ResourceType getResourceType() { + return ResourceType.Yarn; + } + + private JsonNode getResponseByUrl( + String url, String rmWebAddress, ExternalResourceProvider provider) { + + HttpGet httpGet = new HttpGet(rmWebAddress + "/ws/v1/cluster/" + url); + httpGet.addHeader("Accept", "application/json"); + Object authorEnable = provider.getConfigMap().get("authorEnable"); + HttpResponse httpResponse = null; + if (authorEnable instanceof Boolean) { + if ((Boolean) authorEnable) { + httpGet.addHeader(HttpHeaders.AUTHORIZATION, "Basic " + getAuthorizationStr(provider)); + } + } + Object kerberosEnable = provider.getConfigMap().get("kerberosEnable"); + if (kerberosEnable instanceof Boolean) { + if ((Boolean) kerberosEnable) { + String principalName = (String) provider.getConfigMap().get("principalName"); + String keytabPath = (String) provider.getConfigMap().get("keytabPath"); + String krb5Path = (String) provider.getConfigMap().get("krb5Path"); + if (StringUtils.isNotBlank(krb5Path)) { + logger.warn( + "krb5Path: {} has been specified, but not allow to be set to avoid conflict", + krb5Path); + } + RequestKerberosUrlUtils requestKuu = + new RequestKerberosUrlUtils(principalName, keytabPath, false); + HttpResponse response = + requestKuu.callRestUrl(rmWebAddress + "/ws/v1/cluster/" + url, principalName); + httpResponse = response; + } else { + try { + httpResponse = httpClient.execute(httpGet); + } catch (IOException e) { + logger.warn("getResponseByUrl failed", e); + throw new RMErrorException( + YARN_QUEUE_EXCEPTION.getErrorCode(), YARN_QUEUE_EXCEPTION.getErrorDesc(), e); + } + } + } else { + try { + httpResponse = httpClient.execute(httpGet); + } catch (IOException e) { + logger.warn("getResponseByUrl failed", e); + throw new RMErrorException( + YARN_QUEUE_EXCEPTION.getErrorCode(), YARN_QUEUE_EXCEPTION.getErrorDesc(), e); + } + } + + String entityString = ""; + try { + entityString = EntityUtils.toString(httpResponse.getEntity()); + } catch (IOException e) { + logger.warn("getResponseByUrl failed", e); + throw new RMErrorException( + YARN_QUEUE_EXCEPTION.getErrorCode(), YARN_QUEUE_EXCEPTION.getErrorDesc(), e); + } + JsonNode jsonNode = null; + try { + jsonNode = objectMapper.readTree(entityString); + } catch (Exception e) { + logger.warn("getResponseByUrl failed", e); + throw new RMErrorException( + YARN_QUEUE_EXCEPTION.getErrorCode(), YARN_QUEUE_EXCEPTION.getErrorDesc(), e); + } + return jsonNode; + } + + public String getAndUpdateActiveRmWebAddress(ExternalResourceProvider provider) { + String haAddress = (String) provider.getConfigMap().get("rmWebAddress"); + String activeAddress = rmAddressMap.get(haAddress); + if (StringUtils.isBlank(activeAddress)) { + synchronized (haAddress.intern()) { + if (StringUtils.isBlank(activeAddress)) { + if (logger.isDebugEnabled()) { + logger.debug( + "Cannot find value of haAddress : " + + haAddress + + " in cacheMap with size " + + rmAddressMap.size()); + } + if (StringUtils.isNotBlank(haAddress)) { + String[] addresses = + haAddress.split(RMConfiguration.DEFAULT_YARN_RM_WEB_ADDRESS_DELIMITER.getValue()); + for (String address : addresses) { + try { + JsonNode response = getResponseByUrl("info", address, provider); + JsonNode haStateValue = response.path("clusterInfo").path("haState"); + if (!haStateValue.isMissingNode() && haStateValue.isTextual()) { + String haState = haStateValue.asText(); + if (HASTATE_ACTIVE.equalsIgnoreCase(haState)) { + activeAddress = address; + } else { + logger.warn("Resourcemanager : " + address + " haState : " + haState); + } + } + } catch (Exception e) { + logger.error("Get Yarn resourcemanager info error, " + e.getMessage(), e); + } + } + } + if (StringUtils.isNotBlank(activeAddress)) { + if (logger.isDebugEnabled()) { + logger.debug("Put (" + haAddress + ", " + activeAddress + ") to cacheMap."); + } + rmAddressMap.put(haAddress, activeAddress); + } else { + throw new RMErrorException( + GET_YARN_EXCEPTION.getErrorCode(), + MessageFormat.format(GET_YARN_EXCEPTION.getErrorDesc(), haAddress)); + } + } + } + } + if (logger.isDebugEnabled()) { + logger.debug("Get active rm address : " + activeAddress + " from haAddress : " + haAddress); + } + return activeAddress; + } + + @Override + public Boolean reloadExternalResourceAddress(ExternalResourceProvider provider) { + if (null != provider) { + String rmWebHaAddress = (String) provider.getConfigMap().get("rmWebAddress"); + rmAddressMap.remove(rmWebHaAddress); + getAndUpdateActiveRmWebAddress(provider); + } + return true; + } +} diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/external/yarn/YarnResourceRequester.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/external/yarn/YarnResourceRequester.scala deleted file mode 100644 index 209a4a4141..0000000000 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/external/yarn/YarnResourceRequester.scala +++ /dev/null @@ -1,434 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.manager.rm.external.yarn - -import org.apache.linkis.common.utils.{Logging, Utils} -import org.apache.linkis.manager.common.conf.RMConfiguration -import org.apache.linkis.manager.common.entity.resource.{ - CommonNodeResource, - NodeResource, - ResourceType, - YarnResource -} -import org.apache.linkis.manager.common.errorcode.ManagerCommonErrorCodeSummary._ -import org.apache.linkis.manager.common.exception.{RMErrorException, RMWarnException} -import org.apache.linkis.manager.rm.external.domain.{ - ExternalAppInfo, - ExternalResourceIdentifier, - ExternalResourceProvider -} -import org.apache.linkis.manager.rm.external.request.ExternalResourceRequester -import org.apache.linkis.manager.rm.utils.RequestKerberosUrlUtils - -import org.apache.commons.lang3.StringUtils -import org.apache.http.{HttpHeaders, HttpResponse} -import org.apache.http.client.methods.HttpGet -import org.apache.http.impl.client.HttpClients -import org.apache.http.util.EntityUtils - -import java.text.MessageFormat -import java.util -import java.util.Base64 -import java.util.concurrent.ConcurrentHashMap - -import scala.collection.JavaConverters._ -import scala.collection.mutable.ArrayBuffer - -import org.json4s.JsonAST._ -import org.json4s.JValue -import org.json4s.jackson.JsonMethods.parse - -class YarnResourceRequester extends ExternalResourceRequester with Logging { - - private val HASTATE_ACTIVE = "active" - - private val rmAddressMap: util.Map[String, String] = new ConcurrentHashMap[String, String]() - - private def getAuthorizationStr(provider: ExternalResourceProvider) = { - val user = provider.getConfigMap.getOrDefault("user", "").asInstanceOf[String] - val pwd = provider.getConfigMap.getOrDefault("pwd", "").asInstanceOf[String] - val authKey = user + ":" + pwd - Base64.getMimeEncoder.encodeToString(authKey.getBytes) - } - - override def requestResourceInfo( - identifier: ExternalResourceIdentifier, - provider: ExternalResourceProvider - ): NodeResource = { - val rmWebAddress = getAndUpdateActiveRmWebAddress(provider) - logger.info(s"rmWebAddress: $rmWebAddress") - val queueName = identifier.asInstanceOf[YarnResourceIdentifier].getQueueName - - def getYarnResource(jValue: Option[JValue]) = jValue.map(r => - new YarnResource( - (r \ "memory").asInstanceOf[JInt].values.toLong * 1024L * 1024L, - (r \ "vCores").asInstanceOf[JInt].values.toInt, - 0, - queueName - ) - ) - - def maxEffectiveHandle(queueValue: Option[JValue]): Option[YarnResource] = { - val metrics = getResponseByUrl("metrics", rmWebAddress, provider) - val totalResouceInfoResponse = ( - (metrics \ "clusterMetrics" \ "totalMB").asInstanceOf[JInt].values.toLong, - (metrics \ "clusterMetrics" \ "totalVirtualCores").asInstanceOf[JInt].values.toLong - ) - queueValue.map(r => { - val absoluteCapacity = r \ "absoluteCapacity" match { - case jDecimal: JDecimal => - jDecimal.values.toDouble - case jDouble: JDouble => - jDouble.values - case _ => - 0d - } - val effectiveResource = absoluteCapacity - new YarnResource( - math - .floor(effectiveResource * totalResouceInfoResponse._1 * 1024L * 1024L / 100) - .toLong, - math.floor(effectiveResource * totalResouceInfoResponse._2 / 100).toInt, - 0, - queueName - ) - }) - } - - var realQueueName = "root." + queueName - - def getQueue(queues: JValue): Option[JValue] = queues match { - case JArray(queue) => - queue.foreach { q => - val yarnQueueName = (q \ "queueName").asInstanceOf[JString].values - if (yarnQueueName == realQueueName) return Some(q) - else if (realQueueName.startsWith(yarnQueueName + ".")) { - return getQueue(getChildQueues(q)) - } - } - None - case JObject(queue) => - if ( - queue - .find(_._1 == "queueName") - .exists(_._2.asInstanceOf[JString].values == realQueueName) - ) { - Some(queues) - } else { - val childQueues = queue.find(_._1 == "childQueues") - if (childQueues.isEmpty) None - else getQueue(childQueues.map(_._2).get) - } - case _ => None - } - - def getChildQueues(resp: JValue): JValue = { - val queues = resp \ "childQueues" \ "queue" - - if ( - queues != null && queues != JNull && queues != JNothing && null != queues.children && queues.children.nonEmpty - ) { - logger.info(s"queues:$queues") - queues - } else resp \ "childQueues" - } - - def getQueueOfCapacity(queues: JValue): Option[JValue] = queues match { - case JArray(queue) => - queue.foreach { q => - val yarnQueueName = (q \ "queueName").asInstanceOf[JString].values - if (yarnQueueName == realQueueName) return Some(q) - else if ((q \ "queues").toOption.nonEmpty) { - val matchQueue = getQueueOfCapacity(getChildQueuesOfCapacity(q)) - if (matchQueue.nonEmpty) return matchQueue - } - } - None - case JObject(queue) => - if ( - queue - .find(_._1 == "queueName") - .exists(_._2.asInstanceOf[JString].values == realQueueName) - ) { - return Some(queues) - } else if ((queues \ "queues").toOption.nonEmpty) { - val matchQueue = getQueueOfCapacity(getChildQueuesOfCapacity(queues)) - if (matchQueue.nonEmpty) return matchQueue - } - None - case _ => None - } - - def getChildQueuesOfCapacity(resp: JValue) = resp \ "queues" \ "queue" - - def getResources() = { - val resp = getResponseByUrl("scheduler", rmWebAddress, provider) - val schedulerType = - (resp \ "scheduler" \ "schedulerInfo" \ "type").asInstanceOf[JString].values - if ("capacityScheduler".equals(schedulerType)) { - realQueueName = queueName - val childQueues = getChildQueuesOfCapacity(resp \ "scheduler" \ "schedulerInfo") - val queue = getQueueOfCapacity(childQueues) - val queueOption = Option(queue) match { - case Some(queue) => queue - case None => - logger.debug(s"cannot find any information about queue $queueName, response: " + resp) - throw new RMWarnException( - YARN_NOT_EXISTS_QUEUE.getErrorCode, - MessageFormat.format(YARN_NOT_EXISTS_QUEUE.getErrorDesc, queueName) - ) - } - val queueInfo = queueOption.get.asInstanceOf[JObject] - ( - maxEffectiveHandle(queue).get, - getYarnResource(queue.map(_ \ "resourcesUsed")).get, - (queueInfo \ "maxApps").asInstanceOf[JInt].values.toInt, - (queueInfo \ "numPendingApps").asInstanceOf[JInt].values.toInt, - (queueInfo \ "numActiveApps").asInstanceOf[JInt].values.toInt - ) - } else if ("fairScheduler".equals(schedulerType)) { - if ("root".equals(queueName)) { - // get cluster total resource - val queue = (resp \ "scheduler" \ "schedulerInfo" \ "rootQueue") - val rootQueue: Option[JValue] = Some(queue) - val rootQueueInfo = rootQueue.get.asInstanceOf[JObject] - ( - getYarnResource(rootQueue.map(_ \ "maxResources")).get, - getYarnResource(rootQueue.map(_ \ "usedResources")).get, - (rootQueueInfo \ "maxApps").asInstanceOf[JInt].values.toInt, - 0, - 0 - ) - } else { - val childQueues = getChildQueues(resp \ "scheduler" \ "schedulerInfo" \ "rootQueue") - val queue = getQueue(childQueues) - if (queue.isEmpty || queue.get == null) { - logger.debug(s"cannot find any information about queue $queueName, response: " + resp) - throw new RMWarnException( - YARN_NOT_EXISTS_QUEUE.getErrorCode, - MessageFormat.format(YARN_NOT_EXISTS_QUEUE.getErrorDesc, queueName) - ) - } - val queueInfo = queue.get.asInstanceOf[JObject] - ( - getYarnResource(queue.map(_ \ "maxResources")).get, - getYarnResource(queue.map(_ \ "usedResources")).get, - (queueInfo \ "maxApps").asInstanceOf[JInt].values.toInt, - (queueInfo \ "numPendingApps").asInstanceOf[JInt].values.toInt, - (queueInfo \ "numActiveApps").asInstanceOf[JInt].values.toInt - ) - } - } else { - logger.debug( - s"only support fairScheduler or capacityScheduler, schedulerType: $schedulerType , response: " + resp - ) - throw new RMWarnException( - ONLY_SUPPORT_FAIRORCAPA.getErrorCode, - MessageFormat.format(ONLY_SUPPORT_FAIRORCAPA.getErrorDesc(), schedulerType) - ) - } - } - - Utils.tryCatch { - val yarnResource = getResources() - val nodeResource = new CommonNodeResource - nodeResource.setMaxResource(yarnResource._1) - nodeResource.setUsedResource(yarnResource._2) - nodeResource.setMaxApps(yarnResource._3) - nodeResource.setNumPendingApps(yarnResource._4) - nodeResource.setNumActiveApps(yarnResource._5) - nodeResource - }(t => { - throw new RMErrorException( - YARN_QUEUE_EXCEPTION.getErrorCode, - YARN_QUEUE_EXCEPTION.getErrorDesc, - t - ) - }) - } - - override def requestAppInfo( - identifier: ExternalResourceIdentifier, - provider: ExternalResourceProvider - ): java.util.List[ExternalAppInfo] = { - - val rmWebAddress = getAndUpdateActiveRmWebAddress(provider) - - val queueName = identifier.asInstanceOf[YarnResourceIdentifier].getQueueName - - def getYarnResource(jValue: Option[JValue]) = jValue.map(r => - new YarnResource( - (r \ "allocatedMB").asInstanceOf[JInt].values.toLong * 1024L * 1024L, - (r \ "allocatedVCores").asInstanceOf[JInt].values.toInt, - 0, - queueName - ) - ) - - val realQueueName = "root." + queueName - - def getAppInfos(): Array[ExternalAppInfo] = { - val resp = getResponseByUrl("apps", rmWebAddress, provider) - resp \ "apps" \ "app" match { - case JArray(apps) => - val appInfoBuffer = new ArrayBuffer[YarnAppInfo]() - apps.foreach { app => - val yarnQueueName = (app \ "queue").asInstanceOf[JString].values - val state = (app \ "state").asInstanceOf[JString].values - if (yarnQueueName == realQueueName && (state == "RUNNING" || state == "ACCEPTED")) { - val appInfo = new YarnAppInfo( - (app \ "id").asInstanceOf[JString].values, - (app \ "user").asInstanceOf[JString].values, - state, - (app \ "applicationType").asInstanceOf[JString].values, - getYarnResource(Some(app)).get - ) - appInfoBuffer.append(appInfo) - } - } - appInfoBuffer.toArray - case _ => new ArrayBuffer[YarnAppInfo](0).toArray - } - } - - Utils.tryCatch(getAppInfos().toList.asJava)(t => { - throw new RMErrorException( - YARN_APPLICATION_EXCEPTION.getErrorCode, - YARN_APPLICATION_EXCEPTION.getErrorDesc, - t - ) - }) - } - - override def getResourceType: ResourceType = ResourceType.Yarn - - private def getResponseByUrl( - url: String, - rmWebAddress: String, - provider: ExternalResourceProvider - ) = { - val httpGet = new HttpGet(rmWebAddress + "/ws/v1/cluster/" + url) - httpGet.addHeader("Accept", "application/json") - val authorEnable: Any = provider.getConfigMap.get("authorEnable"); - var httpResponse: HttpResponse = null - authorEnable match { - case flag: Boolean => - if (flag) { - httpGet.addHeader(HttpHeaders.AUTHORIZATION, "Basic " + getAuthorizationStr(provider)) - } - case _ => - } - val kerberosEnable: Any = provider.getConfigMap.get("kerberosEnable"); - kerberosEnable match { - case flag: Boolean => - if (flag) { - val principalName = provider.getConfigMap.get("principalName").asInstanceOf[String] - val keytabPath = provider.getConfigMap.get("keytabPath").asInstanceOf[String] - val krb5Path = provider.getConfigMap.get("krb5Path").asInstanceOf[String] - val requestKuu = new RequestKerberosUrlUtils(principalName, keytabPath, krb5Path, false) - val response = - requestKuu.callRestUrl(rmWebAddress + "/ws/v1/cluster/" + url, principalName) - httpResponse = response; - } else { - val response = YarnResourceRequester.httpClient.execute(httpGet) - httpResponse = response - } - case _ => - val response = YarnResourceRequester.httpClient.execute(httpGet) - httpResponse = response - } - parse(EntityUtils.toString(httpResponse.getEntity())) - } - - def getAndUpdateActiveRmWebAddress(provider: ExternalResourceProvider): String = { - // todo check if it will stuck for many requests - val haAddress = provider.getConfigMap.get("rmWebAddress").asInstanceOf[String] - var activeAddress = rmAddressMap.get(haAddress) - if (StringUtils.isBlank(activeAddress)) haAddress.intern().synchronized { - if (StringUtils.isBlank(activeAddress)) { - if (logger.isDebugEnabled()) { - logger.debug( - s"Cannot find value of haAddress : ${haAddress} in cacheMap with size ${rmAddressMap.size()}" - ) - } - if (StringUtils.isNotBlank(haAddress)) { - haAddress - .split(RMConfiguration.DEFAULT_YARN_RM_WEB_ADDRESS_DELIMITER.getValue) - .foreach(address => { - Utils.tryCatch { - val response = getResponseByUrl("info", address, provider) - response \ "clusterInfo" \ "haState" match { - case state: JString => - if (HASTATE_ACTIVE.equalsIgnoreCase(state.s)) { - activeAddress = address - } else { - logger.warn(s"Resourcemanager : ${address} haState : ${state.s}") - } - case _ => - } - } { case e: Exception => - logger.error("Get Yarn resourcemanager info error, " + e.getMessage, e) - } - }) - } - if (StringUtils.isNotBlank(activeAddress)) { - if (logger.isDebugEnabled()) { - logger.debug(s"Put (${haAddress}, ${activeAddress}) to cacheMap.") - } - rmAddressMap.put(haAddress, activeAddress) - } else { - throw new RMErrorException( - GET_YARN_EXCEPTION.getErrorCode, - MessageFormat.format(GET_YARN_EXCEPTION.getErrorDesc(), haAddress) - ) - } - } - } - if (logger.isDebugEnabled()) { - logger.debug(s"Get active rm address : ${activeAddress} from haAddress : ${haAddress}") - } - activeAddress - } - - override def reloadExternalResourceAddress( - provider: ExternalResourceProvider - ): java.lang.Boolean = { - if (null != provider) { - val rmWebHaAddress = provider.getConfigMap.get("rmWebAddress").asInstanceOf[String] - rmAddressMap.remove(rmWebHaAddress) - getAndUpdateActiveRmWebAddress(provider) - } - true - } - -} - -object YarnResourceRequester extends Logging { - - private val httpClient = HttpClients.createDefault() - - def init(): Unit = { - addShutdownHook() - } - - def addShutdownHook(): Unit = { - logger.info("Register shutdown hook to release httpclient connection") - Utils.addShutdownHook(httpClient.close()) - } - -} diff --git a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/executor/SparkSqlExecutor.scala b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/executor/SparkSqlExecutor.scala index bbaaa700dd..ada91499c3 100644 --- a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/executor/SparkSqlExecutor.scala +++ b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/executor/SparkSqlExecutor.scala @@ -18,17 +18,22 @@ package org.apache.linkis.engineplugin.spark.executor import org.apache.linkis.common.utils.Utils +import org.apache.linkis.engineconn.computation.executor.entity.EngineConnTask import org.apache.linkis.engineconn.computation.executor.execute.EngineExecutionContext import org.apache.linkis.engineplugin.spark.common.{Kind, SparkSQL} import org.apache.linkis.engineplugin.spark.config.SparkConfiguration import org.apache.linkis.engineplugin.spark.entity.SparkEngineSession import org.apache.linkis.engineplugin.spark.utils.EngineUtils import org.apache.linkis.governance.common.constant.job.JobRequestConstants +import org.apache.linkis.governance.common.entity.ExecutionNodeStatus import org.apache.linkis.governance.common.paser.SQLCodeParser -import org.apache.linkis.scheduler.executer.{ErrorExecuteResponse, ExecuteResponse, SuccessExecuteResponse} +import org.apache.linkis.scheduler.executer.{ + ErrorExecuteResponse, + ExecuteResponse, + SuccessExecuteResponse +} + import org.apache.commons.lang3.exception.ExceptionUtils -import org.apache.linkis.engineconn.computation.executor.entity.EngineConnTask -import org.apache.linkis.governance.common.entity.ExecutionNodeStatus import java.lang.reflect.InvocationTargetException From 8b4535069f0f484e5ee3baa8704b310031d1074c Mon Sep 17 00:00:00 2001 From: peacewong Date: Wed, 21 Aug 2024 12:08:32 +0800 Subject: [PATCH 03/90] =?UTF-8?q?=E8=A7=A3=E5=86=B3=E7=BC=96=E8=AF=91?= =?UTF-8?q?=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../external/yarn/YarnResourceRequester.java | 34 ++++++++----------- .../manager/rm/restful/RMMonitorRest.scala | 12 +++---- .../manager/common/conf/RMConfiguration.scala | 2 -- 3 files changed, 20 insertions(+), 28 deletions(-) diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/rm/external/yarn/YarnResourceRequester.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/rm/external/yarn/YarnResourceRequester.java index ffc61c52bc..cd78c561ef 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/rm/external/yarn/YarnResourceRequester.java +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/rm/external/yarn/YarnResourceRequester.java @@ -17,8 +17,6 @@ package org.apache.linkis.manager.rm.external.yarn; -import org.apache.linkis.manager.am.util.LinkisUtils; -import org.apache.linkis.manager.common.conf.RMConfiguration; import org.apache.linkis.manager.common.entity.resource.CommonNodeResource; import org.apache.linkis.manager.common.entity.resource.NodeResource; import org.apache.linkis.manager.common.entity.resource.ResourceType; @@ -78,22 +76,19 @@ public NodeResource requestResourceInfo( String queueName = ((YarnResourceIdentifier) identifier).getQueueName(); String realQueueName = "root." + queueName; - return LinkisUtils.tryCatch( - () -> { - YarnQueueInfo resources = getResources(rmWebAddress, realQueueName, queueName, provider); - - CommonNodeResource nodeResource = new CommonNodeResource(); - nodeResource.setMaxResource(resources.getMaxResource()); - nodeResource.setUsedResource(resources.getUsedResource()); - nodeResource.setMaxApps(resources.getMaxApps()); - nodeResource.setNumPendingApps(resources.getNumPendingApps()); - nodeResource.setNumActiveApps(resources.getNumActiveApps()); - return nodeResource; - }, - t -> { - throw new RMErrorException( - YARN_QUEUE_EXCEPTION.getErrorCode(), YARN_QUEUE_EXCEPTION.getErrorDesc(), t); - }); + try { + YarnQueueInfo resources = getResources(rmWebAddress, realQueueName, queueName, provider); + CommonNodeResource nodeResource = new CommonNodeResource(); + nodeResource.setMaxResource(resources.getMaxResource()); + nodeResource.setUsedResource(resources.getUsedResource()); + nodeResource.setMaxApps(resources.getMaxApps()); + nodeResource.setNumPendingApps(resources.getNumPendingApps()); + nodeResource.setNumActiveApps(resources.getNumActiveApps()); + return nodeResource; + } catch (Exception e) { + throw new RMErrorException( + YARN_QUEUE_EXCEPTION.getErrorCode(), YARN_QUEUE_EXCEPTION.getErrorDesc(), e); + } } public Optional maxEffectiveHandle( @@ -422,8 +417,7 @@ public String getAndUpdateActiveRmWebAddress(ExternalResourceProvider provider) + rmAddressMap.size()); } if (StringUtils.isNotBlank(haAddress)) { - String[] addresses = - haAddress.split(RMConfiguration.DEFAULT_YARN_RM_WEB_ADDRESS_DELIMITER.getValue()); + String[] addresses = haAddress.split(";"); for (String address : addresses) { try { JsonNode response = getResponseByUrl("info", address, provider); diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/restful/RMMonitorRest.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/restful/RMMonitorRest.scala index ca86b8fc0e..24c03ab323 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/restful/RMMonitorRest.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/restful/RMMonitorRest.scala @@ -418,10 +418,10 @@ class RMMonitorRest extends Logging { val yarnAppsInfo = externalResourceService.getAppInfo(ResourceType.Yarn, labelContainer, yarnIdentifier) val userList = - yarnAppsInfo.asScala.groupBy(_.asInstanceOf[YarnAppInfo].user).keys.toList.asJava + yarnAppsInfo.asScala.groupBy(_.asInstanceOf[YarnAppInfo].getUser).keys.toList.asJava Utils.tryCatch { val nodesList = getEngineNodesByUserList(userList, true) - yarnAppsInfo.asScala.groupBy(_.asInstanceOf[YarnAppInfo].user).foreach { userAppInfo => + yarnAppsInfo.asScala.groupBy(_.asInstanceOf[YarnAppInfo].getUser).foreach { userAppInfo => var busyResource = Resource.initResource(ResourceType.Yarn).asInstanceOf[YarnResource] var idleResource = Resource.initResource(ResourceType.Yarn).asInstanceOf[YarnResource] val appIdToEngineNode = new mutable.HashMap[String, EngineNode]() @@ -441,15 +441,15 @@ class RMMonitorRest extends Logging { }) } userAppInfo._2.foreach { appInfo => - appIdToEngineNode.get(appInfo.asInstanceOf[YarnAppInfo].id) match { + appIdToEngineNode.get(appInfo.asInstanceOf[YarnAppInfo].getId) match { case Some(node) => if (NodeStatus.Busy == node.getNodeStatus) { - busyResource = busyResource.add(appInfo.asInstanceOf[YarnAppInfo].usedResource) + busyResource = busyResource.add(appInfo.asInstanceOf[YarnAppInfo].getUsedResource) } else { - idleResource = idleResource.add(appInfo.asInstanceOf[YarnAppInfo].usedResource) + idleResource = idleResource.add(appInfo.asInstanceOf[YarnAppInfo].getUsedResource) } case None => - busyResource = busyResource.add(appInfo.asInstanceOf[YarnAppInfo].usedResource) + busyResource = busyResource.add(appInfo.asInstanceOf[YarnAppInfo].getUsedResource) } } diff --git a/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/scala/org/apache/linkis/manager/common/conf/RMConfiguration.scala b/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/scala/org/apache/linkis/manager/common/conf/RMConfiguration.scala index e8bd40d9ac..1557ccac48 100644 --- a/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/scala/org/apache/linkis/manager/common/conf/RMConfiguration.scala +++ b/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/scala/org/apache/linkis/manager/common/conf/RMConfiguration.scala @@ -85,7 +85,5 @@ object RMConfiguration { val DEFAULT_YARN_TYPE = CommonVars("wds.linkis.rm.default.yarn.cluster.type", "Yarn") val EXTERNAL_RETRY_NUM = CommonVars("wds.linkis.rm.external.retry.num", 3) - val DEFAULT_YARN_RM_WEB_ADDRESS_DELIMITER = - CommonVars("wds.linkis.rm.default.yarn.webaddress.delimiter", ";") } From 934b205467a35ac972b41356c425a15bc9a081d0 Mon Sep 17 00:00:00 2001 From: aiceflower Date: Sat, 24 Aug 2024 17:31:24 +0800 Subject: [PATCH 04/90] Dev 1.7.1 add python module load hook (#584) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * ai generate code init * 人工修改代码 * 人工修改提交 --- .../linkis/common/conf/Configuration.scala | 2 + .../executor/hook/PythonModuleLoad.scala | 152 +++++++++++++ .../hook/PythonModuleLoadEngineConnHook.scala | 64 ++++++ .../executor/hook/PythonSparkEngineHook.scala | 45 ++++ .../resources/linkis-engineconn.properties | 2 +- .../linkis/jobhistory/util/QueryUtils.scala | 11 +- .../linkis/udf/entity/PythonModuleInfoVO.java | 209 ++++++++++++++++++ .../api/rpc/RequestPythonModuleProtocol.scala | 27 +++ .../rpc/ResponsePythonModuleProtocol.scala | 33 +++ 9 files changed, 537 insertions(+), 8 deletions(-) create mode 100644 linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonModuleLoad.scala create mode 100644 linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonModuleLoadEngineConnHook.scala create mode 100644 linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonSparkEngineHook.scala create mode 100644 linkis-public-enhancements/linkis-pes-common/src/main/java/org/apache/linkis/udf/entity/PythonModuleInfoVO.java create mode 100644 linkis-public-enhancements/linkis-pes-common/src/main/scala/org/apache/linkis/udf/api/rpc/RequestPythonModuleProtocol.scala create mode 100644 linkis-public-enhancements/linkis-pes-common/src/main/scala/org/apache/linkis/udf/api/rpc/ResponsePythonModuleProtocol.scala diff --git a/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/conf/Configuration.scala b/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/conf/Configuration.scala index 417c377038..163d7aa4db 100644 --- a/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/conf/Configuration.scala +++ b/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/conf/Configuration.scala @@ -70,6 +70,8 @@ object Configuration extends Logging { val VARIABLE_OPERATION: Boolean = CommonVars("wds.linkis.variable.operation", false).getValue + val IS_VIEW_FS_ENV = CommonVars("wds.linkis.env.is.viewfs", true) + val ERROR_MSG_TIP = CommonVars( "linkis.jobhistory.error.msg.tip", diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonModuleLoad.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonModuleLoad.scala new file mode 100644 index 0000000000..c1f6bff1ed --- /dev/null +++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonModuleLoad.scala @@ -0,0 +1,152 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineconn.computation.executor.hook + +import org.apache.linkis.common.utils.{Logging, Utils} +import org.apache.linkis.engineconn.computation.executor.conf.ComputationExecutorConf +import org.apache.linkis.engineconn.computation.executor.execute.{ComputationExecutor, EngineExecutionContext} +import org.apache.linkis.engineconn.core.engineconn.EngineConnManager +import org.apache.linkis.engineconn.core.executor.ExecutorManager +import org.apache.linkis.manager.label.entity.Label +import org.apache.linkis.manager.label.entity.engine.RunType.RunType +import org.apache.linkis.rpc.Sender +import org.apache.linkis.udf.UDFClientConfiguration +import org.apache.linkis.udf.api.rpc.{RequestPythonModuleProtocol, ResponsePythonModuleProtocol} +import org.apache.linkis.udf.entity.PythonModuleInfoVO +import org.apache.commons.lang3.StringUtils +import org.apache.linkis.common.conf.Configuration.IS_VIEW_FS_ENV + +import java.util +import scala.collection.JavaConverters._ +import scala.collection.mutable + +/** + * The PythonModuleLoad class is designed to load Python modules into the execution environment + * dynamically. This class is not an extension of UDFLoad, but shares a similar philosophy of + * handling dynamic module loading based on user preferences and system configurations. + */ +abstract class PythonModuleLoad extends Logging { + + /** Abstract properties to be defined by the subclass */ + protected val engineType: String + protected val runType: RunType + + protected def getEngineType(): String = engineType + + protected def constructCode(pythonModuleInfo: PythonModuleInfoVO): String + + private def queryPythonModuleRpc( + userName: String, + engineType: String + ): java.util.List[PythonModuleInfoVO] = { + val infoList = Sender + .getSender(UDFClientConfiguration.UDF_SERVICE_NAME.getValue) + .ask(RequestPythonModuleProtocol(userName, engineType)) + .asInstanceOf[ResponsePythonModuleProtocol] + .getModulesInfo() + infoList + } + + protected def getLoadPythonModuleCode: Array[String] = { + val engineCreationContext = + EngineConnManager.getEngineConnManager.getEngineConn.getEngineCreationContext + val user = engineCreationContext.getUser + + var infoList: util.List[PythonModuleInfoVO] = + Utils.tryAndWarn(queryPythonModuleRpc(user, getEngineType())) + if (infoList == null) { + logger.info("rpc get info is empty.") + infoList = new util.ArrayList[PythonModuleInfoVO]() + } + + if (infoList.isEmpty) { + val pmi = new PythonModuleInfoVO() + pmi.setPath("viewfs:///apps-data/hadoop/hello_world.py") + infoList.add(pmi) + + val pmi1 = new PythonModuleInfoVO() + pmi1.setPath("viewfs:///apps-data/hadoop/redis2.zip") + infoList.add(pmi1) + } + + // 替换Viewfs + if (IS_VIEW_FS_ENV.getValue) { + infoList.asScala.foreach { info => + val path = info.getPath + info.setPath(path.replace("hdfs://", "viewfs://")) + } + } + + logger.info(s"${user} load python modules: ") + infoList.asScala.foreach(l => logger.info(s"module name:${l.getName}, path:${l.getPath}\n")) + + // 创建加载code + val codes: mutable.Buffer[String] = infoList.asScala + .filter { info => StringUtils.isNotEmpty(info.getPath) } + .map(constructCode) + // 打印codes + val str: String = codes.mkString("\n") + logger.info(s"python codes: $str") + codes.toArray + } + + private def executeFunctionCode(codes: Array[String], executor: ComputationExecutor): Unit = { + if (null == codes || null == executor) { + return + } + codes.foreach { code => + logger.info("Submit function registration to engine, code: " + code) + Utils.tryCatch(executor.executeLine(new EngineExecutionContext(executor), code)) { + t: Throwable => + logger.error("Failed to load python module", t) + null + } + } + } + + /** + * Generate and execute the code necessary for loading Python modules. + * + * @param executor + * An object capable of executing code in the current engine context. + */ + protected def loadPythonModules(labels: Array[Label[_]]): Unit = { + + val codes = getLoadPythonModuleCode + logger.info(s"codes length: ${codes.length}") + if (null != codes && codes.nonEmpty) { + val executor = ExecutorManager.getInstance.getExecutorByLabels(labels) + if (executor != null) { + val className = executor.getClass.getName + logger.info(s"executor class: ${className}") + } else { + logger.error(s"Failed to load python, executor is null") + } + + executor match { + case computationExecutor: ComputationExecutor => + executeFunctionCode(codes, computationExecutor) + case _ => + } + } + logger.info(s"Successful to load python, engineType : ${engineType}") + } + +} + +// Note: The actual implementation of methods like `executeFunctionCode` and `construct diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonModuleLoadEngineConnHook.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonModuleLoadEngineConnHook.scala new file mode 100644 index 0000000000..9eb48c40cc --- /dev/null +++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonModuleLoadEngineConnHook.scala @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineconn.computation.executor.hook + +import org.apache.linkis.common.utils.{Logging, Utils} +import org.apache.linkis.engineconn.common.creation.EngineCreationContext +import org.apache.linkis.engineconn.common.engineconn.EngineConn +import org.apache.linkis.engineconn.common.hook.EngineConnHook +import org.apache.linkis.manager.label.entity.Label +import org.apache.linkis.manager.label.entity.engine.CodeLanguageLabel + +abstract class PythonModuleLoadEngineConnHook + extends PythonModuleLoad + with EngineConnHook + with Logging { + + override def afterExecutionExecute( + engineCreationContext: EngineCreationContext, + engineConn: EngineConn + ): Unit = { + Utils.tryAndWarnMsg{ + val codeLanguageLabel = new CodeLanguageLabel + codeLanguageLabel.setCodeType(runType.toString) + logger.info(s"engineType: ${engineType}") + val labels = Array[Label[_]](codeLanguageLabel) + loadPythonModules(labels) + }(s"Failed to load Python Modules: ${engineType}") + + } + + override def afterEngineServerStartFailed( + engineCreationContext: EngineCreationContext, + throwable: Throwable + ): Unit = { + logger.error(s"Failed to start Engine Server: ${throwable.getMessage}", throwable) + } + + override def beforeCreateEngineConn(engineCreationContext: EngineCreationContext): Unit = { + logger.info("Preparing to load Python Module...") + } + + override def beforeExecutionExecute( + engineCreationContext: EngineCreationContext, + engineConn: EngineConn + ): Unit = { + logger.info(s"Before executing command on load Python Module.") + } + +} diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonSparkEngineHook.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonSparkEngineHook.scala new file mode 100644 index 0000000000..0fe554f93d --- /dev/null +++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonSparkEngineHook.scala @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineconn.computation.executor.hook + +import org.apache.linkis.manager.label.entity.engine.RunType +import org.apache.linkis.manager.label.entity.engine.RunType.RunType +import org.apache.linkis.udf.entity.PythonModuleInfoVO + +/** + * 定义一个用于Spark引擎的Python模块加载与执行挂钩的类 + */ +class PythonSparkEngineHook extends PythonModuleLoadEngineConnHook { + + // 设置engineType属性为"spark"，表示此挂钩适用于Spark数据处理引擎 + override val engineType: String = "spark" + + // 设置runType属性为RunType.PYSPARK，表示此挂钩将执行PySpark类型的代码 + override protected val runType: RunType = RunType.PYSPARK + + // 重写constructCode方法，用于根据Python模块信息构造加载模块的代码 + override protected def constructCode(pythonModuleInfo: PythonModuleInfoVO): String = { + // 使用pythonModuleInfo的path属性，构造SparkContext.addPyFile的命令字符串 + // 这个命令在PySpark环境中将模块文件添加到所有worker上，以便在代码中可以使用 + val path: String = pythonModuleInfo.getPath + val loadCode = s"sc.addPyFile('${path}')" + logger.info(s"pythonLoadCode: ${loadCode}") + loadCode + } + +} diff --git a/linkis-engineconn-plugins/spark/src/main/resources/linkis-engineconn.properties b/linkis-engineconn-plugins/spark/src/main/resources/linkis-engineconn.properties index 3de8a6512b..a535e31ea0 100644 --- a/linkis-engineconn-plugins/spark/src/main/resources/linkis-engineconn.properties +++ b/linkis-engineconn-plugins/spark/src/main/resources/linkis-engineconn.properties @@ -24,7 +24,7 @@ wds.linkis.engineconn.debug.enable=true wds.linkis.engineconn.plugin.default.class=org.apache.linkis.engineplugin.spark.SparkEngineConnPlugin -wds.linkis.engine.connector.hooks=org.apache.linkis.engineconn.computation.executor.hook.ComputationEngineConnHook,org.apache.linkis.engineconn.computation.executor.hook.PyUdfEngineHook,org.apache.linkis.engineconn.computation.executor.hook.ScalaUdfEngineHook,org.apache.linkis.engineconn.computation.executor.hook.JarUdfEngineHook,org.apache.linkis.engineconn.computation.executor.hook.SparkInitSQLHook +wds.linkis.engine.connector.hooks=org.apache.linkis.engineconn.computation.executor.hook.ComputationEngineConnHook,org.apache.linkis.engineconn.computation.executor.hook.PyUdfEngineHook,org.apache.linkis.engineconn.computation.executor.hook.ScalaUdfEngineHook,org.apache.linkis.engineconn.computation.executor.hook.JarUdfEngineHook,org.apache.linkis.engineconn.computation.executor.hook.SparkInitSQLHook,org.apache.linkis.engineconn.computation.executor.hook.PythonSparkEngineHook linkis.spark.once.yarn.restful.url=http://127.0.0.1:8088 diff --git a/linkis-public-enhancements/linkis-jobhistory/src/main/scala/org/apache/linkis/jobhistory/util/QueryUtils.scala b/linkis-public-enhancements/linkis-jobhistory/src/main/scala/org/apache/linkis/jobhistory/util/QueryUtils.scala index 762f008abc..e82517eeeb 100644 --- a/linkis-public-enhancements/linkis-jobhistory/src/main/scala/org/apache/linkis/jobhistory/util/QueryUtils.scala +++ b/linkis-public-enhancements/linkis-jobhistory/src/main/scala/org/apache/linkis/jobhistory/util/QueryUtils.scala @@ -17,23 +17,21 @@ package org.apache.linkis.jobhistory.util +import org.apache.commons.io.IOUtils +import org.apache.commons.lang3.time.DateFormatUtils import org.apache.linkis.common.conf.CommonVars +import org.apache.linkis.common.conf.Configuration.IS_VIEW_FS_ENV import org.apache.linkis.common.io.FsPath import org.apache.linkis.common.utils.{Logging, Utils} import org.apache.linkis.governance.common.entity.job.{JobRequest, SubJobDetail} -import org.apache.linkis.jobhistory.conf.JobhistoryConfiguration import org.apache.linkis.jobhistory.entity.JobHistory import org.apache.linkis.storage.FSFactory import org.apache.linkis.storage.fs.FileSystem import org.apache.linkis.storage.utils.{FileSystemUtils, StorageUtils} -import org.apache.commons.io.IOUtils -import org.apache.commons.lang3.time.DateFormatUtils - import java.io.{BufferedReader, InputStream, InputStreamReader, OutputStream} import java.text.SimpleDateFormat -import java.util -import java.util.{Arrays, Date} +import java.util.Date import java.util.regex.Pattern object QueryUtils extends Logging { @@ -44,7 +42,6 @@ object QueryUtils extends Logging { private val CODE_STORE_PREFIX_VIEW_FS = CommonVars("wds.linkis.query.store.prefix.viewfs", "hdfs:///apps-data/") - private val IS_VIEW_FS_ENV = CommonVars("wds.linkis.env.is.viewfs", true) private val CODE_STORE_SUFFIX = CommonVars("wds.linkis.query.store.suffix", "") private val CODE_STORE_LENGTH = CommonVars("wds.linkis.query.code.store.length", 50000) private val CHARSET = "utf-8" diff --git a/linkis-public-enhancements/linkis-pes-common/src/main/java/org/apache/linkis/udf/entity/PythonModuleInfoVO.java b/linkis-public-enhancements/linkis-pes-common/src/main/java/org/apache/linkis/udf/entity/PythonModuleInfoVO.java new file mode 100644 index 0000000000..1c6a2af99a --- /dev/null +++ b/linkis-public-enhancements/linkis-pes-common/src/main/java/org/apache/linkis/udf/entity/PythonModuleInfoVO.java @@ -0,0 +1,209 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.udf.entity; + +import java.sql.Timestamp; + +/** PythonModuleInfo实体类，用于表示Python模块包信息。 这个类包含了模块的详细信息，如名称、描述、路径、引擎类型、加载状态、过期状态等。 */ +public class PythonModuleInfoVO { + // 自增id，用于唯一标识每一个模块 + private Long id; + + // Python模块名称 + private String name; + + // Python模块描述 + private String description; + + // HDFS路径，存储模块的物理位置 + private String path; + + // 引擎类型，例如：python, spark 或 all + private String engineType; + + // 创建用户，记录创建模块的用户信息 + private String createUser; + + // 修改用户，记录最后修改模块的用户信息 + private String updateUser; + + // 是否加载，0-未加载，1-已加载 + private boolean isLoad; + + // 是否过期，0-未过期，1-已过期 + private Boolean isExpire; + + // 创建时间，记录模块创建的时间 + private Timestamp createTime; + + // 修改时间，记录模块最后修改的时间 + private Timestamp updateTime; + + // 默认构造函数 + public PythonModuleInfoVO() {} + + // 具有所有参数的构造函数 + public PythonModuleInfoVO( + Long id, + String name, + String description, + String path, + String engineType, + String createUser, + String updateUser, + boolean isLoad, + Boolean isExpire, + Timestamp createTime, + Timestamp updateTime) { + this.id = id; + this.name = name; + this.description = description; + this.path = path; + this.engineType = engineType; + this.createUser = createUser; + this.updateUser = updateUser; + this.isLoad = isLoad; + this.isExpire = isExpire; + this.createTime = createTime; + this.updateTime = updateTime; + } + + // Getter和Setter方法 + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public String getPath() { + return path; + } + + public void setPath(String path) { + this.path = path; + } + + public String getEngineType() { + return engineType; + } + + public void setEngineType(String engineType) { + this.engineType = engineType; + } + + public String getCreateUser() { + return createUser; + } + + public void setCreateUser(String createUser) { + this.createUser = createUser; + } + + public String getUpdateUser() { + return updateUser; + } + + public void setUpdateUser(String updateUser) { + this.updateUser = updateUser; + } + + public boolean isLoad() { + return isLoad; + } + + public void setLoad(boolean isLoad) { + this.isLoad = isLoad; + } + + public Boolean isExpire() { + return isExpire; + } + + public void setExpire(Boolean isExpire) { + this.isExpire = isExpire; + } + + public Timestamp getCreateTime() { + return createTime; + } + + public void setCreateTime(Timestamp createTime) { + this.createTime = createTime; + } + + public Timestamp getUpdateTime() { + return updateTime; + } + + public void setUpdateTime(Timestamp updateTime) { + this.updateTime = updateTime; + } + + // 重写toString方法，用于调试和日志记录 + @Override + public String toString() { + return "PythonModuleInfo{" + + "id=" + + id + + ", name='" + + name + + '\'' + + ", description='" + + description + + '\'' + + ", path='" + + path + + '\'' + + ", engineType='" + + engineType + + '\'' + + ", createUser='" + + createUser + + '\'' + + ", updateUser='" + + updateUser + + '\'' + + ", isLoad=" + + isLoad + + ", isExpire=" + + isExpire + + ", createTime=" + + createTime + + ", updateTime=" + + updateTime + + '}'; + } +} diff --git a/linkis-public-enhancements/linkis-pes-common/src/main/scala/org/apache/linkis/udf/api/rpc/RequestPythonModuleProtocol.scala b/linkis-public-enhancements/linkis-pes-common/src/main/scala/org/apache/linkis/udf/api/rpc/RequestPythonModuleProtocol.scala new file mode 100644 index 0000000000..b581d86de5 --- /dev/null +++ b/linkis-public-enhancements/linkis-pes-common/src/main/scala/org/apache/linkis/udf/api/rpc/RequestPythonModuleProtocol.scala @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.udf.api.rpc + +import org.apache.linkis.protocol.{CacheableProtocol, RetryableProtocol} + +trait PythonModuleProtocol + +case class RequestPythonModuleProtocol(userName: String, engineType: String) + extends RetryableProtocol + with CacheableProtocol + with PythonModuleProtocol diff --git a/linkis-public-enhancements/linkis-pes-common/src/main/scala/org/apache/linkis/udf/api/rpc/ResponsePythonModuleProtocol.scala b/linkis-public-enhancements/linkis-pes-common/src/main/scala/org/apache/linkis/udf/api/rpc/ResponsePythonModuleProtocol.scala new file mode 100644 index 0000000000..4ff5c0f8db --- /dev/null +++ b/linkis-public-enhancements/linkis-pes-common/src/main/scala/org/apache/linkis/udf/api/rpc/ResponsePythonModuleProtocol.scala @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.udf.api.rpc + +import org.apache.linkis.udf.entity.PythonModuleInfoVO + +import scala.collection.JavaConverters._ + +class ResponsePythonModuleProtocol(val pythonModules: java.util.List[PythonModuleInfoVO]) + extends PythonModuleProtocol { + + // 如果PythonModuleProtocol需要实现某些方法，你可以在这里实现或覆盖它们 + // 例如，下面是一个假设的示例，展示如何可能实现或覆盖一个方法 + def getModulesInfo(): java.util.List[PythonModuleInfoVO] = { + pythonModules + } + +} From 227511f7e28edda76547443a657ea432a80637f1 Mon Sep 17 00:00:00 2001 From: v-kkhuang <62878639+v-kkhuang@users.noreply.github.com> Date: Sat, 24 Aug 2024 17:44:04 +0800 Subject: [PATCH 05/90] =?UTF-8?q?=E3=80=901.7.1=E3=80=91python=E7=89=A9?= =?UTF-8?q?=E6=96=99=E7=AE=A1=E7=90=86=20(#583)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 代码由AI自动生成接口和AI插件生成 * 代码由AI自动生成接口和AI插件生成 * 代码由人工修改 --------- Co-authored-by: “v_kkhuang” <“420895376@qq.com”> --- .../filesystem/restful/api/FsRestfulApi.java | 66 +++++ .../apache/linkis/udf/api/UDFRestfulApi.java | 268 ++++++++++++++++++ .../udf/dao/PythonModuleInfoMapper.java | 44 +++ .../linkis/udf/entity/PythonModuleInfo.java | 158 +++++++++++ .../udf/service/PythonModuleInfoService.java | 41 +++ .../impl/PythonModuleInfoServiceImpl.java | 64 +++++ .../mapper/common/PythonModuleInfoMapper.xml | 93 ++++++ .../linkis/udf/api/rpc/UdfReceiver.scala | 22 +- 8 files changed, 753 insertions(+), 3 deletions(-) create mode 100644 linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/dao/PythonModuleInfoMapper.java create mode 100644 linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/entity/PythonModuleInfo.java create mode 100644 linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/service/PythonModuleInfoService.java create mode 100644 linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/service/impl/PythonModuleInfoServiceImpl.java create mode 100644 linkis-public-enhancements/linkis-udf-service/src/main/resources/mapper/common/PythonModuleInfoMapper.xml diff --git a/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/filesystem/restful/api/FsRestfulApi.java b/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/filesystem/restful/api/FsRestfulApi.java index 4d9a3fa651..db4a46f375 100644 --- a/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/filesystem/restful/api/FsRestfulApi.java +++ b/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/filesystem/restful/api/FsRestfulApi.java @@ -1423,6 +1423,72 @@ public Message encryptPath( return Message.ok().data("data", fileMD5Str); } + @ApiOperation(value = "Python模块上传", notes = "上传Python模块文件并返回文件地址", response = Message.class) + @ApiImplicitParams({ + @ApiImplicitParam(name = "file", required = true, dataType = "MultipartFile", value = "上传的文件"), + @ApiImplicitParam(name = "fileName", required = true, dataType = "String", value = "文件名称") + }) + @RequestMapping(path = "/python-upload", method = RequestMethod.POST) + public Message pythonUpload( + HttpServletRequest req, + @RequestParam("file") MultipartFile file, + @RequestParam(value = "fileName", required = false) String fileName) + throws WorkSpaceException, IOException { + + // 获取登录用户 + String username = ModuleUserUtils.getOperationUser(req, "pythonUpload"); + + // 校验文件名称 + if (StringUtils.isBlank(fileName)) { + return Message.error("文件名称不能为空"); + } + // 获取文件名称 + String fileNameSuffix = fileName.substring(0, fileName.lastIndexOf(".")); + if (!fileNameSuffix.matches("^[a-zA-Z][a-zA-Z0-9_]{0,49}$")) { + return Message.error("模块名称错误，仅支持数字字母下划线，且以字母开头，长度最大50"); + } + + // 校验文件类型 + if (!file.getOriginalFilename().endsWith(".py") + && !file.getOriginalFilename().endsWith(".zip")) { + return Message.error("仅支持.py和.zip格式模块文件"); + } + + // 校验文件大小 + if (file.getSize() > 50 * 1024 * 1024) { + return Message.error("限制最大单个文件50M"); + } + + // 定义目录路径 + String path = "hdfs:///appcom/linkis/udf/" + username; + FsPath fsPath = new FsPath(path); + + // 获取文件系统实例 + FileSystem fileSystem = fsService.getFileSystem(username, fsPath); + + // 确认目录是否存在，不存在则创建新目录 + if (!fileSystem.exists(fsPath)) { + try { + fileSystem.mkdirs(fsPath); + } catch (IOException e) { + return Message.error("创建目录失败：" + e.getMessage()); + } + } + + // 构建新的文件路径 + String newPath = fsPath.getPath() + "/" + file.getOriginalFilename(); + FsPath fsPathNew = new FsPath(newPath); + + // 上传文件 + try (InputStream is = file.getInputStream(); + OutputStream outputStream = fileSystem.write(fsPathNew, true)) { + IOUtils.copy(is, outputStream); + } catch (IOException e) { + return Message.error("文件上传失败：" + e.getMessage()); + } + // 返回成功消息并包含文件地址 + return Message.ok().data("filePath", newPath); + } /** * * * diff --git a/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/api/UDFRestfulApi.java b/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/api/UDFRestfulApi.java index c659e4aa93..2bebc661dd 100644 --- a/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/api/UDFRestfulApi.java +++ b/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/api/UDFRestfulApi.java @@ -17,11 +17,14 @@ package org.apache.linkis.udf.api; +import org.apache.linkis.common.conf.Configuration; import org.apache.linkis.server.Message; import org.apache.linkis.server.utils.ModuleUserUtils; +import org.apache.linkis.udf.entity.PythonModuleInfo; import org.apache.linkis.udf.entity.UDFInfo; import org.apache.linkis.udf.entity.UDFTree; import org.apache.linkis.udf.excepiton.UDFException; +import org.apache.linkis.udf.service.PythonModuleInfoService; import org.apache.linkis.udf.service.UDFService; import org.apache.linkis.udf.service.UDFTreeService; import org.apache.linkis.udf.utils.ConstantVar; @@ -39,6 +42,7 @@ import org.springframework.util.StringUtils; import org.springframework.web.bind.annotation.*; +import javax.annotation.Nullable; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; @@ -49,6 +53,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import com.github.pagehelper.PageHelper; import com.github.pagehelper.PageInfo; import com.github.xiaoymin.knife4j.annotations.ApiOperationSupport; import com.google.common.collect.Lists; @@ -73,6 +78,7 @@ public class UDFRestfulApi { @Autowired private UDFService udfService; @Autowired private UDFTreeService udfTreeService; + @Autowired private PythonModuleInfoService pythonModuleInfoService; ObjectMapper mapper = new ObjectMapper(); @@ -1013,4 +1019,266 @@ public Message versionInfo( } return message; } + + /** + * Python物料查询 + * + * @param name python模块名称 + * @param engineType 引擎类型(all,spark,python) + * @param username 用户名 + * @param isLoad 是否加载（0-未加载，1-已加载） + * @param isExpire 是否过期（0-未过期，1-已过期） + * @param pageNow 页码 + * @param pageSize 每页展示数据条数 + */ + @RequestMapping(path = "/python-list", method = RequestMethod.GET) + @ApiOperation(value = "查询Python模块列表", notes = "根据条件查询Python模块信息") + @ApiImplicitParams({ + @ApiImplicitParam(name = "name", value = "Python模块名称", required = false, dataType = "String"), + @ApiImplicitParam( + name = "engineType", + value = "引擎类型（all, spark, python）", + required = false, + dataType = "String"), + @ApiImplicitParam(name = "username", value = "用户名", required = false, dataType = "String"), + @ApiImplicitParam( + name = "isLoad", + value = "是否加载（0-未加载，1-已加载）", + required = false, + dataType = "Integer"), + @ApiImplicitParam( + name = "isExpire", + value = "是否过期（0-未过期，1-已过期）", + required = false, + dataType = "Integer"), + @ApiImplicitParam(name = "pageNow", value = "页码", required = false, dataType = "Integer"), + @ApiImplicitParam(name = "pageSize", value = "每页展示数据条数", required = false, dataType = "Integer") + }) + public Message pythonList( + @RequestParam(value = "name", required = false) String name, + @RequestParam(value = "engineType", required = false) String engineType, + @RequestParam(value = "username", required = false) String username, + @RequestParam(value = "isLoad", required = false) Integer isLoad, + @RequestParam(value = "isExpire", required = false) Integer isExpire, + @RequestParam(value = "pageNow", required = false) Integer pageNow, + @RequestParam(value = "pageSize", required = false) Integer pageSize, + HttpServletRequest req) { + + // 获取登录用户 + String user = ModuleUserUtils.getOperationUser(req, "pythonList"); + + // 参数校验 + if (org.apache.commons.lang3.StringUtils.isBlank(name)) name = null; + if (org.apache.commons.lang3.StringUtils.isBlank(engineType)) engineType = null; + if (pageNow == null) pageNow = 1; + if (pageSize == null) pageSize = 10; + + // 根据管理员权限设置username + if (Configuration.isAdmin(user)) { + if (username == null) username = null; + } else { + username = user; + } + + // 分页设置 + PageHelper.startPage(pageNow, pageSize); + try { + // 执行数据库查询 + PythonModuleInfo pythonModuleInfo = new PythonModuleInfo(); + pythonModuleInfo.setName(name); + pythonModuleInfo.setEngineType(engineType); + pythonModuleInfo.setCreateUser(username); + pythonModuleInfo.setIsLoad(isLoad); + pythonModuleInfo.setIsExpire(isExpire); + List pythonList = pythonModuleInfoService.getByConditions(pythonModuleInfo); + PageInfo pageInfo = new PageInfo<>(pythonList); + // 封装返回结果 + return Message.ok().data("pythonList", pythonList).data("totalPage", pageInfo.getTotal()); + } finally { + // 关闭分页 + PageHelper.clearPage(); + } + } + + /** + * Python物料删除 + * + * @param id id + * @param isExpire 0-未过期，1-已过期 + */ + @RequestMapping(path = "/python-delete", method = RequestMethod.GET) + @ApiOperation(value = "删除Python模块", notes = "根据模块ID删除Python模块,管理员可以删除任何模块，普通用户只能删除自己创建的模块") + @ApiImplicitParams({ + @ApiImplicitParam(name = "id", value = "模块ID", required = true, dataType = "Long"), + @ApiImplicitParam( + name = "isExpire", + value = "模块是否过期（0：未过期，1：已过期）", + required = true, + dataType = "int") + }) + public Message pythonDelete( + @RequestParam(value = "id", required = false) Long id, + @RequestParam(value = "isExpire", required = false) int isExpire, + HttpServletRequest req, + HttpServletResponse resp) { + // 打印审计日志并获取登录用户 + String user = ModuleUserUtils.getOperationUser(req, "pythonDelete"); + + // 参数校验 + if (id == null) { + return Message.error("Invalid parameters: id is null"); + } + if (isExpire != 0 && isExpire != 1) { + return Message.error("Invalid parameters: isExpire must be 0 or 1"); + } + PythonModuleInfo pythonModuleInfo = new PythonModuleInfo(); + pythonModuleInfo.setId(id); + // 根据id查询Python模块信息 + PythonModuleInfo moduleInfo = pythonModuleInfoService.getByUserAndNameAndId(pythonModuleInfo); + if (moduleInfo == null) { + return Message.ok(); // 如果不存在则直接返回成功 + } + + // 判断是否是管理员 + if (!Configuration.isAdmin(user)) { + // 如果不是管理员，检查创建用户是否与当前用户一致 + if (!moduleInfo.getCreateUser().equals(user)) { + return Message.error("无权删除他人Python模块"); + } + } + + // 更新Python模块信息 + moduleInfo.setIsExpire(1); + moduleInfo.setUpdateUser(user); + moduleInfo.setUpdateTime(new Date()); + // 修改数据库中的模块名称和文件名称 + String newName = moduleInfo.getName() + "_" + System.currentTimeMillis(); + String newPath = moduleInfo.getPath() + "_" + System.currentTimeMillis(); + moduleInfo.setPath(newPath); + moduleInfo.setName(newName); + pythonModuleInfoService.updatePythonModuleInfo(moduleInfo); + return Message.ok(); + } + + /** Python物料新增/更新 */ + @ApiOperation(value = "Python物料新增/更新", notes = "根据传入的Python物料信息新增或更新") + @ApiImplicitParams({ + @ApiImplicitParam( + name = "Python物料新增/更新Request", + value = "Python物料新增/更新请求体", + required = false, + dataType = "PythonModuleInfo") + }) + @RequestMapping(value = "/python-save", method = RequestMethod.POST) + public Message request( + @Nullable @RequestBody PythonModuleInfo pythonModuleInfo, + HttpServletRequest httpReq, + HttpServletResponse httpResp) { + + // 获取登录用户 + String userName = ModuleUserUtils.getOperationUser(httpReq, "pythonSave"); + + // 入参校验 + if (org.apache.commons.lang3.StringUtils.isBlank(pythonModuleInfo.getName())) { + return Message.error("模块名称：不能为空"); + } + if (org.apache.commons.lang3.StringUtils.isBlank(pythonModuleInfo.getPath())) { + return Message.error("模块物料：不能为空"); + } + if (org.apache.commons.lang3.StringUtils.isBlank(pythonModuleInfo.getEngineType())) { + return Message.error("引擎类型：不能为空"); + } + if (pythonModuleInfo.getIsLoad() == null) { + return Message.error("是否加载：不能为空"); + } + if (pythonModuleInfo.getIsExpire() == null) { + return Message.error("是否过期：不能为空"); + } + String path = pythonModuleInfo.getPath(); + String fileName = path.substring(path.lastIndexOf("/") + 1, path.lastIndexOf(".")); + if (!pythonModuleInfo.getName().equals(fileName)) { + return Message.error("模块名称与物料文件名称必须一样"); + } + // 根据id判断是插入还是更新 + if (pythonModuleInfo.getId() == null) { + PythonModuleInfo moduleInfo = pythonModuleInfoService.getByUserAndNameAndId(pythonModuleInfo); + // 插入逻辑 + if (moduleInfo != null) { + return Message.error("模块" + moduleInfo.getName() + "已存在"); + } + pythonModuleInfo.setCreateTime(new Date()); + pythonModuleInfo.setUpdateTime(new Date()); + pythonModuleInfo.setCreateUser(userName); + pythonModuleInfo.setUpdateUser(userName); + pythonModuleInfoService.insertPythonModuleInfo(pythonModuleInfo); + return Message.ok().data("id", pythonModuleInfo.getId()); + } else { + PythonModuleInfo pythonModuleTmp = new PythonModuleInfo(); + pythonModuleTmp.setId(pythonModuleInfo.getId()); + PythonModuleInfo moduleInfo = pythonModuleInfoService.getByUserAndNameAndId(pythonModuleTmp); + // 更新逻辑 + if (moduleInfo == null) { + return Message.error("未找到该Python模块"); + } + if (!Configuration.isAdmin(userName) && !userName.equals(moduleInfo.getCreateUser())) { + return Message.error("无权编辑他人Python模块"); + } + if (moduleInfo.getIsExpire() != 0) { + return Message.error("当前模块已过期，不允许进行修改操作"); + } + pythonModuleInfo.setUpdateUser(userName); + pythonModuleInfo.setUpdateTime(new Date()); + pythonModuleInfoService.updatePythonModuleInfo(pythonModuleInfo); + } + return Message.ok(); + } + + /** + * python文件是否存在查询 + * + * @param fileName 文件名称 + */ + @RequestMapping(path = "/python-file-exist", method = RequestMethod.GET) + @ApiOperation(value = "查询Python文件是否存在", notes = "根据用户名和文件名查询Python模块信息，如果存在则返回true，否则返回false") + @ApiImplicitParams({ + @ApiImplicitParam( + name = "fileName", + value = "Python文件名", + required = true, + dataType = "string", + paramType = "query"), + @ApiImplicitParam( + name = "Authorization", + value = "Bearer token", + required = true, + dataType = "string", + paramType = "header") + }) + public Message pythonFileExist( + @RequestParam(value = "fileName", required = false) String fileName, HttpServletRequest req) { + // 审计日志打印并获取登录用户 + String userName = ModuleUserUtils.getOperationUser(req, "pythonFileExist"); + + // 参数校验 + if (org.apache.commons.lang3.StringUtils.isBlank(fileName)) { + return Message.error("参数fileName不能为空"); + } + String fileNameWithoutExtension = fileName.substring(0, fileName.lastIndexOf(".")); + if (!fileNameWithoutExtension.matches("^[a-zA-Z][a-zA-Z0-9_]{0,49}$")) { + return Message.error("只支持数字字母下划线，且以字母开头，长度最大50"); + } + + // 封装PythonModuleInfo对象并查询数据库 + PythonModuleInfo pythonModuleInfo = new PythonModuleInfo(); + pythonModuleInfo.setName(fileNameWithoutExtension); + pythonModuleInfo.setCreateUser(userName); + PythonModuleInfo moduleInfo = pythonModuleInfoService.getByUserAndNameAndId(pythonModuleInfo); + + // 根据查询结果返回相应信息 + if (moduleInfo == null) { + return Message.ok().data("result", true); + } else { + return Message.error("模块" + fileName + "已存在，如需重新上传请先删除旧的模块"); + } + } } diff --git a/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/dao/PythonModuleInfoMapper.java b/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/dao/PythonModuleInfoMapper.java new file mode 100644 index 0000000000..f32432ef18 --- /dev/null +++ b/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/dao/PythonModuleInfoMapper.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.udf.dao; + +import org.apache.linkis.udf.entity.PythonModuleInfo; + +import org.apache.ibatis.annotations.Mapper; + +import java.util.List; + +@Mapper +public interface PythonModuleInfoMapper { + + // SQL 1: 模糊查询 + List selectByConditions(PythonModuleInfo pythonModuleInfo); + + // SQL 2: 更新 + int updatePythonModuleInfo(PythonModuleInfo pythonModuleInfo); + + // SQL 3: 新增 + Long insertPythonModuleInfo(PythonModuleInfo pythonModuleInfo); + + // SQL 4: 带有判断的查询 + PythonModuleInfo selectByUserAndNameAndId(PythonModuleInfo pythonModuleInfo); + + // SQL 5: 查询包含多个引擎类型的hdfs路径 + List selectPathsByUsernameAndEnginetypes( + String username, List enginetypes); +} diff --git a/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/entity/PythonModuleInfo.java b/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/entity/PythonModuleInfo.java new file mode 100644 index 0000000000..727b323cb6 --- /dev/null +++ b/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/entity/PythonModuleInfo.java @@ -0,0 +1,158 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.udf.entity; + +import java.util.Date; + +public class PythonModuleInfo { + private Long id; + private String name; + private String description; + private String path; + private String engineType; + private String createUser; + private String updateUser; + private Integer isLoad; + private Integer isExpire; + private Date createTime; + private Date updateTime; + + public PythonModuleInfo() {} + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public String getPath() { + return path; + } + + public void setPath(String path) { + this.path = path; + } + + public String getEngineType() { + return engineType; + } + + public void setEngineType(String engineType) { + this.engineType = engineType; + } + + public String getCreateUser() { + return createUser; + } + + public void setCreateUser(String createUser) { + this.createUser = createUser; + } + + public String getUpdateUser() { + return updateUser; + } + + public void setUpdateUser(String updateUser) { + this.updateUser = updateUser; + } + + public Integer getIsLoad() { + return isLoad; + } + + public void setIsLoad(Integer isLoad) { + this.isLoad = isLoad; + } + + public Integer getIsExpire() { + return isExpire; + } + + public void setIsExpire(Integer isExpire) { + this.isExpire = isExpire; + } + + public Date getCreateTime() { + return createTime; + } + + public void setCreateTime(Date createTime) { + this.createTime = createTime; + } + + public Date getUpdateTime() { + return updateTime; + } + + public void setUpdateTime(Date updateTime) { + this.updateTime = updateTime; + } + + @Override + public String toString() { + return "PythonModuleInfo{" + + "id=" + + id + + ", name='" + + name + + '\'' + + ", description='" + + description + + '\'' + + ", path='" + + path + + '\'' + + ", engineType='" + + engineType + + '\'' + + ", createUser='" + + createUser + + '\'' + + ", updateUser='" + + updateUser + + '\'' + + ", isLoad=" + + isLoad + + ", isExpire=" + + isExpire + + ", createTime=" + + createTime + + ", updateTime=" + + updateTime + + '}'; + } +} diff --git a/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/service/PythonModuleInfoService.java b/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/service/PythonModuleInfoService.java new file mode 100644 index 0000000000..0e94309baf --- /dev/null +++ b/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/service/PythonModuleInfoService.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.udf.service; + +import org.apache.linkis.udf.entity.PythonModuleInfo; + +import java.util.List; + +public interface PythonModuleInfoService { + + // SQL 1: 模糊查询 + List getByConditions(PythonModuleInfo pythonModuleInfo); + + // SQL 2: 更新 + int updatePythonModuleInfo(PythonModuleInfo pythonModuleInfo); + + // SQL 3: 新增 + Long insertPythonModuleInfo(PythonModuleInfo pythonModuleInfo); + + // SQL 4: 带有判断的查询 + PythonModuleInfo getByUserAndNameAndId(PythonModuleInfo pythonModuleInfo); + + // SQL 5: 查询包含多个引擎类型的hdfs路径 + List getPathsByUsernameAndEnginetypes( + String username, List enginetypes); +} diff --git a/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/service/impl/PythonModuleInfoServiceImpl.java b/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/service/impl/PythonModuleInfoServiceImpl.java new file mode 100644 index 0000000000..91f5839416 --- /dev/null +++ b/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/service/impl/PythonModuleInfoServiceImpl.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.udf.service.impl; + +import org.apache.linkis.udf.dao.PythonModuleInfoMapper; +import org.apache.linkis.udf.entity.PythonModuleInfo; +import org.apache.linkis.udf.service.PythonModuleInfoService; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.util.List; + +@Service +public class PythonModuleInfoServiceImpl implements PythonModuleInfoService { + + private final PythonModuleInfoMapper pythonModuleInfoMapper; + + @Autowired + public PythonModuleInfoServiceImpl(PythonModuleInfoMapper pythonModuleInfoMapper) { + this.pythonModuleInfoMapper = pythonModuleInfoMapper; + } + + @Override + public List getByConditions(PythonModuleInfo pythonModuleInfo) { + return pythonModuleInfoMapper.selectByConditions(pythonModuleInfo); + } + + @Override + public int updatePythonModuleInfo(PythonModuleInfo pythonModuleInfo) { + return pythonModuleInfoMapper.updatePythonModuleInfo(pythonModuleInfo); + } + + @Override + public Long insertPythonModuleInfo(PythonModuleInfo pythonModuleInfo) { + return pythonModuleInfoMapper.insertPythonModuleInfo(pythonModuleInfo); + } + + @Override + public PythonModuleInfo getByUserAndNameAndId(PythonModuleInfo pythonModuleInfo) { + return pythonModuleInfoMapper.selectByUserAndNameAndId(pythonModuleInfo); + } + + @Override + public List getPathsByUsernameAndEnginetypes( + String username, List enginetypes) { + return pythonModuleInfoMapper.selectPathsByUsernameAndEnginetypes(username, enginetypes); + } +} diff --git a/linkis-public-enhancements/linkis-udf-service/src/main/resources/mapper/common/PythonModuleInfoMapper.xml b/linkis-public-enhancements/linkis-udf-service/src/main/resources/mapper/common/PythonModuleInfoMapper.xml new file mode 100644 index 0000000000..0b513ba2f2 --- /dev/null +++ b/linkis-public-enhancements/linkis-udf-service/src/main/resources/mapper/common/PythonModuleInfoMapper.xml @@ -0,0 +1,93 @@ + + + + + + + + + + + + UPDATE linkis_ps_python_module_info + + name = #{name}, + description = #{description}, + path = #{path}, + engine_type = #{engineType}, + create_user = #{createUser}, + update_user = #{updateUser}, + is_load = #{isLoad}, + is_expire = #{isExpire}, + create_time = #{createTime}, + update_time = #{updateTime}, + + WHERE id = #{id} + + + + + INSERT INTO linkis_ps_python_module_info + (name, description, path, engine_type, create_user, update_user, is_load, is_expire, create_time, update_time) + VALUES + (#{name}, #{description}, #{path}, #{engineType}, #{createUser}, #{updateUser}, #{isLoad}, #{isExpire}, #{createTime}, #{updateTime}) + + SELECT LAST_INSERT_ID() + + + + + + + + + + + diff --git a/linkis-public-enhancements/linkis-udf-service/src/main/scala/org/apache/linkis/udf/api/rpc/UdfReceiver.scala b/linkis-public-enhancements/linkis-udf-service/src/main/scala/org/apache/linkis/udf/api/rpc/UdfReceiver.scala index 501e4d3bc3..662952f5d4 100644 --- a/linkis-public-enhancements/linkis-udf-service/src/main/scala/org/apache/linkis/udf/api/rpc/UdfReceiver.scala +++ b/linkis-public-enhancements/linkis-udf-service/src/main/scala/org/apache/linkis/udf/api/rpc/UdfReceiver.scala @@ -18,10 +18,9 @@ package org.apache.linkis.udf.api.rpc import org.apache.linkis.rpc.{Receiver, Sender} -import org.apache.linkis.udf.service.{UDFService, UDFTreeService} +import org.apache.linkis.udf.service.{PythonModuleInfoService, UDFService, UDFTreeService} import java.lang - import scala.concurrent.duration.Duration class UdfReceiver extends Receiver { @@ -38,6 +37,17 @@ class UdfReceiver extends Receiver { override def receive(message: Any, sender: Sender): Unit = {} + // 注⼊PythonModuleInfoService +// val pythonModuleInfoService: PythonModuleInfoService = context.system.asInstanceOf[ExtendedActorSystem].lifecycle.systemManager.asInstanceOf[SystemManager].pythonModuleInfoService +// +// def parseModuleInfoFromPath(path: String): PythonModuleInfoVO = { +// // 假设路径格式为 "username/module_name/module_version" +// val parts = path.split("/") +// var vo = PythonModuleInfoVO() +// vo.setPath(path) +// vo +// } + override def receiveAndReply(message: Any, sender: Sender): Any = { message match { case RequestUdfTree(userName, treeType, treeId, treeCategory) => @@ -46,10 +56,16 @@ class UdfReceiver extends Receiver { case RequestUdfIds(userName, udfIds, treeCategory) => val udfs = udfService.getUDFInfoByIds(udfIds.map(id => new lang.Long(id)), treeCategory) new ResponseUdfs(udfs) +// case RequestPythonModuleProtocol(userName, engineTypes) => +// // 获取Python模块路径列表 +// val paths = pythonModuleInfoService.getPathsByUsernameAndEnginetypes(userName, engineTypes) +// // 将路径列表转换为PythonModuleInfo列表 +// val pythonModuleInfoList = paths.map(parseModuleInfoFromPath) +// new ResponsePythonModuleProtocol(pythonModuleInfoList) case _ => } } override def receiveAndReply(message: Any, duration: Duration, sender: Sender): Any = {} -} +} \ No newline at end of file From 01ea07491d68897cc041236ccbe72c6a78631c75 Mon Sep 17 00:00:00 2001 From: aiceflower Date: Sat, 24 Aug 2024 18:00:15 +0800 Subject: [PATCH 06/90] =?UTF-8?q?=E4=BA=BA=E5=B7=A5=E4=BF=AE=E6=94=B9=20co?= =?UTF-8?q?de=20format=20and=20add=20rpc=20func?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../executor/hook/PythonModuleLoad.scala | 9 ++- .../hook/PythonModuleLoadEngineConnHook.scala | 2 +- .../linkis/jobhistory/util/QueryUtils.scala | 5 +- .../linkis/udf/api/rpc/UdfReceiver.scala | 58 +++++++++++++------ .../udf/api/rpc/UdfReceiverChooser.scala | 9 ++- 5 files changed, 58 insertions(+), 25 deletions(-) diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonModuleLoad.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonModuleLoad.scala index c1f6bff1ed..7c1eb23714 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonModuleLoad.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonModuleLoad.scala @@ -17,9 +17,13 @@ package org.apache.linkis.engineconn.computation.executor.hook +import org.apache.linkis.common.conf.Configuration.IS_VIEW_FS_ENV import org.apache.linkis.common.utils.{Logging, Utils} import org.apache.linkis.engineconn.computation.executor.conf.ComputationExecutorConf -import org.apache.linkis.engineconn.computation.executor.execute.{ComputationExecutor, EngineExecutionContext} +import org.apache.linkis.engineconn.computation.executor.execute.{ + ComputationExecutor, + EngineExecutionContext +} import org.apache.linkis.engineconn.core.engineconn.EngineConnManager import org.apache.linkis.engineconn.core.executor.ExecutorManager import org.apache.linkis.manager.label.entity.Label @@ -28,10 +32,11 @@ import org.apache.linkis.rpc.Sender import org.apache.linkis.udf.UDFClientConfiguration import org.apache.linkis.udf.api.rpc.{RequestPythonModuleProtocol, ResponsePythonModuleProtocol} import org.apache.linkis.udf.entity.PythonModuleInfoVO + import org.apache.commons.lang3.StringUtils -import org.apache.linkis.common.conf.Configuration.IS_VIEW_FS_ENV import java.util + import scala.collection.JavaConverters._ import scala.collection.mutable diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonModuleLoadEngineConnHook.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonModuleLoadEngineConnHook.scala index 9eb48c40cc..80eaa888b8 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonModuleLoadEngineConnHook.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonModuleLoadEngineConnHook.scala @@ -33,7 +33,7 @@ abstract class PythonModuleLoadEngineConnHook engineCreationContext: EngineCreationContext, engineConn: EngineConn ): Unit = { - Utils.tryAndWarnMsg{ + Utils.tryAndWarnMsg { val codeLanguageLabel = new CodeLanguageLabel codeLanguageLabel.setCodeType(runType.toString) logger.info(s"engineType: ${engineType}") diff --git a/linkis-public-enhancements/linkis-jobhistory/src/main/scala/org/apache/linkis/jobhistory/util/QueryUtils.scala b/linkis-public-enhancements/linkis-jobhistory/src/main/scala/org/apache/linkis/jobhistory/util/QueryUtils.scala index e82517eeeb..582183d07f 100644 --- a/linkis-public-enhancements/linkis-jobhistory/src/main/scala/org/apache/linkis/jobhistory/util/QueryUtils.scala +++ b/linkis-public-enhancements/linkis-jobhistory/src/main/scala/org/apache/linkis/jobhistory/util/QueryUtils.scala @@ -17,8 +17,6 @@ package org.apache.linkis.jobhistory.util -import org.apache.commons.io.IOUtils -import org.apache.commons.lang3.time.DateFormatUtils import org.apache.linkis.common.conf.CommonVars import org.apache.linkis.common.conf.Configuration.IS_VIEW_FS_ENV import org.apache.linkis.common.io.FsPath @@ -29,6 +27,9 @@ import org.apache.linkis.storage.FSFactory import org.apache.linkis.storage.fs.FileSystem import org.apache.linkis.storage.utils.{FileSystemUtils, StorageUtils} +import org.apache.commons.io.IOUtils +import org.apache.commons.lang3.time.DateFormatUtils + import java.io.{BufferedReader, InputStream, InputStreamReader, OutputStream} import java.text.SimpleDateFormat import java.util.Date diff --git a/linkis-public-enhancements/linkis-udf-service/src/main/scala/org/apache/linkis/udf/api/rpc/UdfReceiver.scala b/linkis-public-enhancements/linkis-udf-service/src/main/scala/org/apache/linkis/udf/api/rpc/UdfReceiver.scala index 662952f5d4..d37d274f06 100644 --- a/linkis-public-enhancements/linkis-udf-service/src/main/scala/org/apache/linkis/udf/api/rpc/UdfReceiver.scala +++ b/linkis-public-enhancements/linkis-udf-service/src/main/scala/org/apache/linkis/udf/api/rpc/UdfReceiver.scala @@ -18,9 +18,12 @@ package org.apache.linkis.udf.api.rpc import org.apache.linkis.rpc.{Receiver, Sender} +import org.apache.linkis.udf.entity.{PythonModuleInfo, PythonModuleInfoVO} import org.apache.linkis.udf.service.{PythonModuleInfoService, UDFService, UDFTreeService} -import java.lang +import java.{lang, util} + +import scala.collection.JavaConverters.asScalaBufferConverter import scala.concurrent.duration.Duration class UdfReceiver extends Receiver { @@ -29,24 +32,35 @@ class UdfReceiver extends Receiver { private var udfService: UDFService = _ + // 注⼊PythonModuleInfoService + private var pythonModuleInfoService: PythonModuleInfoService = _ + def this(udfTreeService: UDFTreeService, udfService: UDFService) = { this() this.udfTreeService = udfTreeService this.udfService = udfService } + def this( + udfTreeService: UDFTreeService, + udfService: UDFService, + pythonModuleInfoService: PythonModuleInfoService + ) = { + this(udfTreeService, udfService) + this.pythonModuleInfoService = pythonModuleInfoService + } + override def receive(message: Any, sender: Sender): Unit = {} - // 注⼊PythonModuleInfoService -// val pythonModuleInfoService: PythonModuleInfoService = context.system.asInstanceOf[ExtendedActorSystem].lifecycle.systemManager.asInstanceOf[SystemManager].pythonModuleInfoService -// -// def parseModuleInfoFromPath(path: String): PythonModuleInfoVO = { -// // 假设路径格式为 "username/module_name/module_version" -// val parts = path.split("/") -// var vo = PythonModuleInfoVO() -// vo.setPath(path) -// vo -// } + def parseModuleInfoVO(info: PythonModuleInfo): PythonModuleInfoVO = { + // 假设路径格式为 "username/module_name/module_version" + val vo = new PythonModuleInfoVO() + vo.setPath(info.getPath) + vo.setName(info.getName) + vo.setId(info.getId) + vo.setCreateUser(info.getCreateUser) + vo + } override def receiveAndReply(message: Any, sender: Sender): Any = { message match { @@ -56,16 +70,24 @@ class UdfReceiver extends Receiver { case RequestUdfIds(userName, udfIds, treeCategory) => val udfs = udfService.getUDFInfoByIds(udfIds.map(id => new lang.Long(id)), treeCategory) new ResponseUdfs(udfs) -// case RequestPythonModuleProtocol(userName, engineTypes) => -// // 获取Python模块路径列表 -// val paths = pythonModuleInfoService.getPathsByUsernameAndEnginetypes(userName, engineTypes) -// // 将路径列表转换为PythonModuleInfo列表 -// val pythonModuleInfoList = paths.map(parseModuleInfoFromPath) -// new ResponsePythonModuleProtocol(pythonModuleInfoList) + case RequestPythonModuleProtocol(userName, engineType) => + // 获取Python模块路径列表 + var list = new java.util.ArrayList[String]() + list.add(engineType) + list.add("all") + val infoes: util.List[PythonModuleInfo] = + pythonModuleInfoService.getPathsByUsernameAndEnginetypes(userName, list) + // 将路径列表转换为PythonModuleInfo列表 + var voList = new java.util.ArrayList[PythonModuleInfoVO]() + infoes.asScala.foreach(info => { + val vo: PythonModuleInfoVO = parseModuleInfoVO(info) + voList.add(vo) + }) + new ResponsePythonModuleProtocol(voList) case _ => } } override def receiveAndReply(message: Any, duration: Duration, sender: Sender): Any = {} -} \ No newline at end of file +} diff --git a/linkis-public-enhancements/linkis-udf-service/src/main/scala/org/apache/linkis/udf/api/rpc/UdfReceiverChooser.scala b/linkis-public-enhancements/linkis-udf-service/src/main/scala/org/apache/linkis/udf/api/rpc/UdfReceiverChooser.scala index de75580102..e5621c3bf5 100644 --- a/linkis-public-enhancements/linkis-udf-service/src/main/scala/org/apache/linkis/udf/api/rpc/UdfReceiverChooser.scala +++ b/linkis-public-enhancements/linkis-udf-service/src/main/scala/org/apache/linkis/udf/api/rpc/UdfReceiverChooser.scala @@ -18,7 +18,7 @@ package org.apache.linkis.udf.api.rpc import org.apache.linkis.rpc.{Receiver, ReceiverChooser, RPCMessageEvent} -import org.apache.linkis.udf.service.{UDFService, UDFTreeService} +import org.apache.linkis.udf.service.{PythonModuleInfoService, UDFService, UDFTreeService} import org.springframework.beans.factory.annotation.Autowired import org.springframework.stereotype.Component @@ -34,10 +34,15 @@ class UdfReceiverChooser extends ReceiverChooser { @Autowired private var udfService: UDFService = _ + @Autowired + private var pythonModuleInfoService: PythonModuleInfoService = _ + private var udfReceiver: Option[UdfReceiver] = None @PostConstruct - def init(): Unit = udfReceiver = Some(new UdfReceiver(udfTreeService, udfService)) + def init(): Unit = udfReceiver = Some( + new UdfReceiver(udfTreeService, udfService, pythonModuleInfoService) + ) override def chooseReceiver(event: RPCMessageEvent): Option[Receiver] = event.message match { case _: UdfProtocol => udfReceiver From 3a513b21c080d64e956e2dd0a4f1755e3f3533a0 Mon Sep 17 00:00:00 2001 From: aiceflower Date: Sat, 24 Aug 2024 23:12:16 +0800 Subject: [PATCH 07/90] fix bug --- .../executor/hook/PythonModuleLoad.scala | 26 +++++++++++-------- .../api/rpc/RequestPythonModuleProtocol.scala | 1 + .../linkis/udf/api/rpc/UdfReceiver.scala | 6 ++++- 3 files changed, 21 insertions(+), 12 deletions(-) diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonModuleLoad.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonModuleLoad.scala index 7c1eb23714..34928d8525 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonModuleLoad.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonModuleLoad.scala @@ -79,21 +79,25 @@ abstract class PythonModuleLoad extends Logging { infoList = new util.ArrayList[PythonModuleInfoVO]() } - if (infoList.isEmpty) { - val pmi = new PythonModuleInfoVO() - pmi.setPath("viewfs:///apps-data/hadoop/hello_world.py") - infoList.add(pmi) - - val pmi1 = new PythonModuleInfoVO() - pmi1.setPath("viewfs:///apps-data/hadoop/redis2.zip") - infoList.add(pmi1) - } - // 替换Viewfs if (IS_VIEW_FS_ENV.getValue) { infoList.asScala.foreach { info => val path = info.getPath - info.setPath(path.replace("hdfs://", "viewfs://")) + logger.info(s"python path: ${path}") + if (path.startsWith("hdfs") || path.startsWith("viewfs")) { + info.setPath(path.replace("hdfs://", "viewfs://")) + } else { + info.setPath("viewfs://" + path) + } + } + } else { + + infoList.asScala.foreach { info => + val path = info.getPath + logger.info(s"hdfs python path: ${path}") + if (!path.startsWith("hdfs")) { + info.setPath("hdfs://" + path) + } } } diff --git a/linkis-public-enhancements/linkis-pes-common/src/main/scala/org/apache/linkis/udf/api/rpc/RequestPythonModuleProtocol.scala b/linkis-public-enhancements/linkis-pes-common/src/main/scala/org/apache/linkis/udf/api/rpc/RequestPythonModuleProtocol.scala index b581d86de5..27cd071fb7 100644 --- a/linkis-public-enhancements/linkis-pes-common/src/main/scala/org/apache/linkis/udf/api/rpc/RequestPythonModuleProtocol.scala +++ b/linkis-public-enhancements/linkis-pes-common/src/main/scala/org/apache/linkis/udf/api/rpc/RequestPythonModuleProtocol.scala @@ -25,3 +25,4 @@ case class RequestPythonModuleProtocol(userName: String, engineType: String) extends RetryableProtocol with CacheableProtocol with PythonModuleProtocol + with UdfProtocol diff --git a/linkis-public-enhancements/linkis-udf-service/src/main/scala/org/apache/linkis/udf/api/rpc/UdfReceiver.scala b/linkis-public-enhancements/linkis-udf-service/src/main/scala/org/apache/linkis/udf/api/rpc/UdfReceiver.scala index d37d274f06..a81d2c4faf 100644 --- a/linkis-public-enhancements/linkis-udf-service/src/main/scala/org/apache/linkis/udf/api/rpc/UdfReceiver.scala +++ b/linkis-public-enhancements/linkis-udf-service/src/main/scala/org/apache/linkis/udf/api/rpc/UdfReceiver.scala @@ -17,6 +17,7 @@ package org.apache.linkis.udf.api.rpc +import org.apache.linkis.common.utils.Logging import org.apache.linkis.rpc.{Receiver, Sender} import org.apache.linkis.udf.entity.{PythonModuleInfo, PythonModuleInfoVO} import org.apache.linkis.udf.service.{PythonModuleInfoService, UDFService, UDFTreeService} @@ -25,8 +26,9 @@ import java.{lang, util} import scala.collection.JavaConverters.asScalaBufferConverter import scala.concurrent.duration.Duration +import scala.tools.nsc.interactive.Logger -class UdfReceiver extends Receiver { +class UdfReceiver extends Receiver with Logging { private var udfTreeService: UDFTreeService = _ @@ -63,6 +65,7 @@ class UdfReceiver extends Receiver { } override def receiveAndReply(message: Any, sender: Sender): Any = { + logger.info(s"udfPython message: ${message.getClass.getName}") message match { case RequestUdfTree(userName, treeType, treeId, treeCategory) => val udfTree = udfTreeService.getTreeById(treeId, userName, treeType, treeCategory) @@ -71,6 +74,7 @@ class UdfReceiver extends Receiver { val udfs = udfService.getUDFInfoByIds(udfIds.map(id => new lang.Long(id)), treeCategory) new ResponseUdfs(udfs) case RequestPythonModuleProtocol(userName, engineType) => + logger.info(s"RequestPythonModuleProtocol: userName: $userName, engineType: $engineType") // 获取Python模块路径列表 var list = new java.util.ArrayList[String]() list.add(engineType) From 7bc3a82ff388216ba901f1dd54473410bb59286b Mon Sep 17 00:00:00 2001 From: v-kkhuang <62878639+v-kkhuang@users.noreply.github.com> Date: Sun, 25 Aug 2024 12:39:10 +0800 Subject: [PATCH 08/90] Dev 1.7.0 python udf manager (#586) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 代码由AI自动生成接口和AI插件生成 * 代码由AI自动生成接口和AI插件生成 * 代码由人工修改 * bug fix * bug fix --------- Co-authored-by: “v_kkhuang” <“420895376@qq.com”> --- .../main/java/org/apache/linkis/udf/api/UDFRestfulApi.java | 2 +- .../org/apache/linkis/udf/dao/PythonModuleInfoMapper.java | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/api/UDFRestfulApi.java b/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/api/UDFRestfulApi.java index 2bebc661dd..0db971d6f5 100644 --- a/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/api/UDFRestfulApi.java +++ b/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/api/UDFRestfulApi.java @@ -1201,6 +1201,7 @@ public Message request( } // 根据id判断是插入还是更新 if (pythonModuleInfo.getId() == null) { + pythonModuleInfo.setCreateUser(userName); PythonModuleInfo moduleInfo = pythonModuleInfoService.getByUserAndNameAndId(pythonModuleInfo); // 插入逻辑 if (moduleInfo != null) { @@ -1208,7 +1209,6 @@ public Message request( } pythonModuleInfo.setCreateTime(new Date()); pythonModuleInfo.setUpdateTime(new Date()); - pythonModuleInfo.setCreateUser(userName); pythonModuleInfo.setUpdateUser(userName); pythonModuleInfoService.insertPythonModuleInfo(pythonModuleInfo); return Message.ok().data("id", pythonModuleInfo.getId()); diff --git a/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/dao/PythonModuleInfoMapper.java b/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/dao/PythonModuleInfoMapper.java index f32432ef18..0f52960e17 100644 --- a/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/dao/PythonModuleInfoMapper.java +++ b/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/dao/PythonModuleInfoMapper.java @@ -17,9 +17,9 @@ package org.apache.linkis.udf.dao; -import org.apache.linkis.udf.entity.PythonModuleInfo; - import org.apache.ibatis.annotations.Mapper; +import org.apache.ibatis.annotations.Param; +import org.apache.linkis.udf.entity.PythonModuleInfo; import java.util.List; @@ -40,5 +40,5 @@ public interface PythonModuleInfoMapper { // SQL 5: 查询包含多个引擎类型的hdfs路径 List selectPathsByUsernameAndEnginetypes( - String username, List enginetypes); + @Param("username") String username, @Param("enginetypes") List enginetypes); } From 694935fa5dc948a41f54bd52b67591a0c8106eb2 Mon Sep 17 00:00:00 2001 From: aiceflower Date: Sun, 25 Aug 2024 12:46:16 +0800 Subject: [PATCH 09/90] fix bug --- .../org/apache/linkis/udf/api/UDFRestfulApi.java | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/api/UDFRestfulApi.java b/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/api/UDFRestfulApi.java index 2bebc661dd..0a84e0dcc1 100644 --- a/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/api/UDFRestfulApi.java +++ b/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/api/UDFRestfulApi.java @@ -1201,6 +1201,10 @@ public Message request( } // 根据id判断是插入还是更新 if (pythonModuleInfo.getId() == null) { + Integer newExpire = pythonModuleInfo.getIsExpire(); + pythonModuleInfo.setCreateUser(userName); + // 查询未过期的 + pythonModuleInfo.setIsExpire(0); PythonModuleInfo moduleInfo = pythonModuleInfoService.getByUserAndNameAndId(pythonModuleInfo); // 插入逻辑 if (moduleInfo != null) { @@ -1208,7 +1212,7 @@ public Message request( } pythonModuleInfo.setCreateTime(new Date()); pythonModuleInfo.setUpdateTime(new Date()); - pythonModuleInfo.setCreateUser(userName); + pythonModuleInfo.setIsExpire(newExpire); pythonModuleInfo.setUpdateUser(userName); pythonModuleInfoService.insertPythonModuleInfo(pythonModuleInfo); return Message.ok().data("id", pythonModuleInfo.getId()); @@ -1226,6 +1230,14 @@ public Message request( if (moduleInfo.getIsExpire() != 0) { return Message.error("当前模块已过期，不允许进行修改操作"); } + // 如果模块过期，则修改数据库中的模块名称和文件名称 + if (pythonModuleInfo.getIsExpire() == 1) { + // 修改数据库中的模块名称和文件名称 + String newName = moduleInfo.getName() + "_" + System.currentTimeMillis(); + String newPath = moduleInfo.getPath() + "_" + System.currentTimeMillis(); + pythonModuleInfo.setPath(newPath); + pythonModuleInfo.setName(newName); + } pythonModuleInfo.setUpdateUser(userName); pythonModuleInfo.setUpdateTime(new Date()); pythonModuleInfoService.updatePythonModuleInfo(pythonModuleInfo); From 7237beabc5d3804b5eaad79cb8132d3827879308 Mon Sep 17 00:00:00 2001 From: aiceflower Date: Sun, 25 Aug 2024 13:33:51 +0800 Subject: [PATCH 10/90] code format --- .../java/org/apache/linkis/udf/dao/PythonModuleInfoMapper.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/dao/PythonModuleInfoMapper.java b/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/dao/PythonModuleInfoMapper.java index 0f52960e17..54c79bcee0 100644 --- a/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/dao/PythonModuleInfoMapper.java +++ b/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/dao/PythonModuleInfoMapper.java @@ -17,9 +17,10 @@ package org.apache.linkis.udf.dao; +import org.apache.linkis.udf.entity.PythonModuleInfo; + import org.apache.ibatis.annotations.Mapper; import org.apache.ibatis.annotations.Param; -import org.apache.linkis.udf.entity.PythonModuleInfo; import java.util.List; From 97f53013a7ae2929ca3af62c2c98022278f0753e Mon Sep 17 00:00:00 2001 From: aiceflower Date: Sun, 25 Aug 2024 14:33:04 +0800 Subject: [PATCH 11/90] =?UTF-8?q?=E4=BD=BF=E7=94=A8ai=E7=94=9F=E6=88=90?= =?UTF-8?q?=E5=8D=95=E5=85=83=E6=B5=8B=E8=AF=95=E6=A1=88=E4=BE=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../PythonModuleLoadEngineConnHookTest.scala | 5 + .../executor/hook/PythonModuleLoadTest.scala | 5 + .../hook/PythonSparkEngineHookTest.scala | 48 ++++++ .../udf/api/PythonModuleRestfulApiTest.java | 135 +++++++++++++++++ .../udf/dao/PythonModuleInfoMapperTest.java | 121 +++++++++++++++ .../service/PythonModuleInfoServiceTest.java | 143 ++++++++++++++++++ 6 files changed, 457 insertions(+) create mode 100644 linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/test/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonModuleLoadEngineConnHookTest.scala create mode 100644 linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/test/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonModuleLoadTest.scala create mode 100644 linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/test/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonSparkEngineHookTest.scala create mode 100644 linkis-public-enhancements/linkis-udf-service/src/test/java/org/apache/linkis/udf/api/PythonModuleRestfulApiTest.java create mode 100644 linkis-public-enhancements/linkis-udf-service/src/test/java/org/apache/linkis/udf/dao/PythonModuleInfoMapperTest.java create mode 100644 linkis-public-enhancements/linkis-udf-service/src/test/java/org/apache/linkis/udf/service/PythonModuleInfoServiceTest.java diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/test/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonModuleLoadEngineConnHookTest.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/test/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonModuleLoadEngineConnHookTest.scala new file mode 100644 index 0000000000..64602aa338 --- /dev/null +++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/test/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonModuleLoadEngineConnHookTest.scala @@ -0,0 +1,5 @@ +package org.apache.linkis.engineconn.computation.executor.hook + +class PythonModuleLoadEngineConnHook { + +} diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/test/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonModuleLoadTest.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/test/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonModuleLoadTest.scala new file mode 100644 index 0000000000..8af788cc5d --- /dev/null +++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/test/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonModuleLoadTest.scala @@ -0,0 +1,5 @@ +package org.apache.linkis.engineconn.computation.executor.hook + +class PythonModuleLoadTest { + +} diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/test/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonSparkEngineHookTest.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/test/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonSparkEngineHookTest.scala new file mode 100644 index 0000000000..d36e2b151d --- /dev/null +++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/test/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonSparkEngineHookTest.scala @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineconn.computation.executor.hook + +import org.mockito.Mockito + +import scala.tools.fusesource_embedded.jansi.AnsiRenderer.test + + +class PythonSparkEngineHookTest extends AnyFunSuite { + + test("constructCode 返回正确的加载代码") { + val pythonModuleInfo = new PythonModuleInfoVO + pythonModuleInfo.setPath("file:///path/to/module.py") + + val hook = new PythonSparkEngineHook + val result = hook.constructCode(pythonModuleInfo) + + assert(result == "sc.addPyFile('file:///path/to/module.py')") + } + + test("constructCode 返回正确的日志信息") { + val pythonModuleInfo = new PythonModuleInfoVO + pythonModuleInfo.setPath("file:///path/to/module.py") + + val hook = new PythonSparkEngineHook + val logger = Mockito.mock(classOf[org.slf4j.Logger]) + hook.logger = logger + + hook.constructCode(pythonModuleInfo) + + val expectedLog = "pythonLoadCode: sc.addPyFile('file:///path/to/module.py')" + Mockito.verify(logger).info(expectedLog) + } +} diff --git a/linkis-public-enhancements/linkis-udf-service/src/test/java/org/apache/linkis/udf/api/PythonModuleRestfulApiTest.java b/linkis-public-enhancements/linkis-udf-service/src/test/java/org/apache/linkis/udf/api/PythonModuleRestfulApiTest.java new file mode 100644 index 0000000000..e1b621882b --- /dev/null +++ b/linkis-public-enhancements/linkis-udf-service/src/test/java/org/apache/linkis/udf/api/PythonModuleRestfulApiTest.java @@ -0,0 +1,135 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.udf.api; + +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.MediaType; +import org.springframework.test.web.servlet.MockMvc; + +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +/** + * PythonModuleRestfulApiTest 类用于对 PythonModuleRestfulApi 进行单元测试。 + */ + +public class PythonModuleRestfulApiTest { + @Autowired + protected MockMvc mockMvc; + /** + * 测试Python模块列表功能 + */ + @Test + public void testPythonList() throws Exception { + // 测试获取Python模块列表 + mockMvc.perform(get("/python-list") + .param("name", "testModule") + .param("engineType", "spark") + .param("username", "testUser") + .param("isLoad", "0") + .param("isExpire", "1") + .param("pageNow", "1") + .param("pageSize", "10")) + .andExpect(status().isOk()); + + // 测试获取Python模块列表（无参数） + mockMvc.perform(get("/python-list")) + .andExpect(status().isOk()); + + // 测试获取Python模块列表（空参数） + mockMvc.perform(get("/python-list") + .param("name", "") + .param("engineType", "") + .param("username", "") + .param("isLoad", "") + .param("isExpire", "") + .param("pageNow", "") + .param("pageSize", "")) + .andExpect(status().isOk()); + } + + /** + * 测试删除Python模块功能 + */ + @Test + public void testPythonDelete() throws Exception { + // 测试删除Python模块 + mockMvc.perform(get("/python-delete") + .param("id", "1") + .param("isExpire", "0")) + .andExpect(status().isOk()); + + // 测试删除不存在的Python模块 + mockMvc.perform(get("/python-delete") + .param("id", "999") + .param("isExpire", "0")) + .andExpect(status().isNotFound()); + + // 测试删除Python模块时传入无效参数 + mockMvc.perform(get("/python-delete") + .param("id", "1") + .param("isExpire", "2")) + .andExpect(status().isBadRequest()); + } + + /** + * 测试保存Python模块功能 + */ + @Test + public void testPythonSave() throws Exception { + // 测试保存Python模块 + mockMvc.perform(post("/python-save") + .contentType(MediaType.APPLICATION_JSON) + .content("{\"name\":\"testModule\",\"path\":\"/path/to/module.py\",\"engineType\":\"python\",\"isLoad\":1,\"isExpire\":0}")) + .andExpect(status().isOk()); + + // 测试保存Python模块时传入空名称 + mockMvc.perform(post("/python-save") + .contentType(MediaType.APPLICATION_JSON) + .content("{\"name\":\"\",\"path\":\"/path/to/module.py\",\"engineType\":\"python\",\"isLoad\":1,\"isExpire\":0}")) + .andExpect(status().isBadRequest()); + + // 测试保存Python模块时传入空路径 + mockMvc.perform(post("/python-save") + .contentType(MediaType.APPLICATION_JSON) + .content("{\"name\":\"testModule\",\"path\":\"\",\"engineType\":\"python\",\"isLoad\":1,\"isExpire\":0}")) + .andExpect(status().isBadRequest()); + } + + /** + * 测试检查Python模块文件是否存在功能 + */ + @Test + public void testPythonFileExist() throws Exception { + // 测试检查Python模块文件是否存在 + mockMvc.perform(get("/python-file-exist") + .param("fileName", "testModule.py")) + .andExpect(status().isOk()); + + // 测试检查Python模块文件是否存在时传入空文件名 + mockMvc.perform(get("/python-file-exist") + .param("fileName", "")) + .andExpect(status().isBadRequest()); + + // 测试检查Python模块文件是否存在时未传入文件名 + mockMvc.perform(get("/python-file-exist")) + .andExpect(status().isBadRequest()); + } +} \ No newline at end of file diff --git a/linkis-public-enhancements/linkis-udf-service/src/test/java/org/apache/linkis/udf/dao/PythonModuleInfoMapperTest.java b/linkis-public-enhancements/linkis-udf-service/src/test/java/org/apache/linkis/udf/dao/PythonModuleInfoMapperTest.java new file mode 100644 index 0000000000..55aacbff4f --- /dev/null +++ b/linkis-public-enhancements/linkis-udf-service/src/test/java/org/apache/linkis/udf/dao/PythonModuleInfoMapperTest.java @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.udf.dao; + +import org.apache.linkis.udf.entity.PythonModuleInfo; +import org.junit.jupiter.api.Test; +import org.springframework.test.context.event.annotation.BeforeTestClass; + +import java.util.Arrays; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * PythonModuleInfoMapperTest 类用于测试 PythonModuleInfoMapper 的功能。 + */ +public class PythonModuleInfoMapperTest { + + private PythonModuleInfoMapper pythonModuleInfoMapper; // PythonModuleInfoMapper 的模拟对象 + + /** + * 在每个测试方法执行前执行，用于初始化测试环境。 + */ + @BeforeTestClass + public void setUp() { + pythonModuleInfoMapper = mock(PythonModuleInfoMapper.class); + } + + /** + * 测试 selectByConditions 方法的功能。 + */ + @Test + public void testSelectByConditions() { + PythonModuleInfo pythonModuleInfo = new PythonModuleInfo(); + // 设置 pythonModuleInfo 的属性 + + when(pythonModuleInfoMapper.selectByConditions(pythonModuleInfo)).thenReturn(Arrays.asList(pythonModuleInfo)); + + List result = pythonModuleInfoMapper.selectByConditions(pythonModuleInfo); + assertEquals(1, result.size()); + // 验证结果的属性 + } + + /** + * 测试 updatePythonModuleInfo 方法的功能。 + */ + @Test + public void testUpdatePythonModuleInfo() { + PythonModuleInfo pythonModuleInfo = new PythonModuleInfo(); + // 设置 pythonModuleInfo 的属性 + + when(pythonModuleInfoMapper.updatePythonModuleInfo(pythonModuleInfo)).thenReturn(1); + + int result = pythonModuleInfoMapper.updatePythonModuleInfo(pythonModuleInfo); + assertEquals(1, result); + } + + /** + * 测试 insertPythonModuleInfo 方法的功能。 + */ + @Test + public void testInsertPythonModuleInfo() { + PythonModuleInfo pythonModuleInfo = new PythonModuleInfo(); + // 设置 pythonModuleInfo 的属性 + + when(pythonModuleInfoMapper.insertPythonModuleInfo(pythonModuleInfo)).thenReturn(1L); + + Long result = pythonModuleInfoMapper.insertPythonModuleInfo(pythonModuleInfo); + assertEquals(1L, result.longValue()); + } + + /** + * 测试 selectByUserAndNameAndId 方法的功能。 + */ + @Test + public void testSelectByUserAndNameAndId() { + PythonModuleInfo pythonModuleInfo = new PythonModuleInfo(); + // 设置 pythonModuleInfo 的属性 + + when(pythonModuleInfoMapper.selectByUserAndNameAndId(pythonModuleInfo)).thenReturn(pythonModuleInfo); + + PythonModuleInfo result = pythonModuleInfoMapper.selectByUserAndNameAndId(pythonModuleInfo); + assertNotNull(result); + // 验证结果的属性 + } + + /** + * 测试 selectPathsByUsernameAndEnginetypes 方法的功能。 + */ + @Test + public void testSelectPathsByUsernameAndEnginetypes() { + String username = "testUser"; + List enginetypes = Arrays.asList("type1", "type2"); + PythonModuleInfo pythonModuleInfo = new PythonModuleInfo(); + // 设置 pythonModuleInfo 的属性 + + when(pythonModuleInfoMapper.selectPathsByUsernameAndEnginetypes(username, enginetypes)).thenReturn(Arrays.asList(pythonModuleInfo)); + + List result = pythonModuleInfoMapper.selectPathsByUsernameAndEnginetypes(username, enginetypes); + assertEquals(1, result.size()); + // 验证结果的属性 + } +} \ No newline at end of file diff --git a/linkis-public-enhancements/linkis-udf-service/src/test/java/org/apache/linkis/udf/service/PythonModuleInfoServiceTest.java b/linkis-public-enhancements/linkis-udf-service/src/test/java/org/apache/linkis/udf/service/PythonModuleInfoServiceTest.java new file mode 100644 index 0000000000..ecc902692e --- /dev/null +++ b/linkis-public-enhancements/linkis-udf-service/src/test/java/org/apache/linkis/udf/service/PythonModuleInfoServiceTest.java @@ -0,0 +1,143 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.udf.service; + +import org.apache.linkis.udf.dao.PythonModuleInfoMapper; +import org.apache.linkis.udf.entity.PythonModuleInfo; +import org.apache.linkis.udf.service.impl.PythonModuleInfoServiceImpl; +import org.aspectj.lang.annotation.Before; +import org.junit.jupiter.api.Test; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; + +import java.util.Arrays; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.Mockito.when; + +/** + * PythonModuleInfoServiceImplTest 类用于对 PythonModuleInfoServiceImpl 进行单元测试。 + */ +public class PythonModuleInfoServiceTest { + + @Mock + private PythonModuleInfoMapper pythonModuleInfoMapper; + + @InjectMocks + private PythonModuleInfoServiceImpl pythonModuleInfoServiceImpl; + + /** + * 在每个测试方法执行前执行，用于初始化测试环境。 + */ + @Before("") + public void setUp() { + MockitoAnnotations.openMocks(this); + } + + /** + * 测试 getByConditions 方法的功能。 + */ + @Test + public void testGetByConditions() { + PythonModuleInfo mockInfo = new PythonModuleInfo(); + mockInfo.setId(1L); + mockInfo.setName("TestModule"); + when(pythonModuleInfoMapper.selectByConditions(mockInfo)).thenReturn(Arrays.asList(mockInfo)); + + List result = pythonModuleInfoServiceImpl.getByConditions(mockInfo); + + assertNotNull(result); + assertEquals(1, result.size()); + assertEquals(mockInfo.getId(), result.get(0).getId()); + assertEquals(mockInfo.getName(), result.get(0).getName()); + } + + /** + * 测试 updatePythonModuleInfo 方法的功能。 + */ + @Test + public void testUpdatePythonModuleInfo() { + PythonModuleInfo mockInfo = new PythonModuleInfo(); + mockInfo.setId(1L); + mockInfo.setName("UpdatedModule"); + when(pythonModuleInfoMapper.updatePythonModuleInfo(mockInfo)).thenReturn(1); + + int result = pythonModuleInfoServiceImpl.updatePythonModuleInfo(mockInfo); + + assertEquals(1, result); + } + + /** + * 测试 insertPythonModuleInfo 方法的功能。 + */ + @Test + public void testInsertPythonModuleInfo() { + PythonModuleInfo mockInfo = new PythonModuleInfo(); + mockInfo.setId(1L); + mockInfo.setName("NewModule"); + when(pythonModuleInfoMapper.insertPythonModuleInfo(mockInfo)).thenReturn(1L); + + Long result = pythonModuleInfoServiceImpl.insertPythonModuleInfo(mockInfo); + + assertNotNull(result); + assertEquals(1L, result.longValue()); + } + + /** + * 测试 getByUserAndNameAndId 方法的功能。 + */ + @Test + public void testGetByUserAndNameAndId() { + PythonModuleInfo mockInfo = new PythonModuleInfo(); + mockInfo.setId(1L); + mockInfo.setName("UniqueModule"); + when(pythonModuleInfoMapper.selectByUserAndNameAndId(mockInfo)).thenReturn(mockInfo); + + PythonModuleInfo result = pythonModuleInfoServiceImpl.getByUserAndNameAndId(mockInfo); + + assertNotNull(result); + assertEquals(mockInfo.getId(), result.getId()); + assertEquals(mockInfo.getName(), result.getName()); + } + + /** + * 测试 getPathsByUsernameAndEnginetypes 方法的功能。 + */ + @Test + public void testGetPathsByUsernameAndEnginetypes() { + String username = "testUser"; + List enginetypes = Arrays.asList("Engine1", "Engine2"); + PythonModuleInfo mockInfo1 = new PythonModuleInfo(); + mockInfo1.setId(1L); + mockInfo1.setName("Module1"); + PythonModuleInfo mockInfo2 = new PythonModuleInfo(); + mockInfo2.setId(2L); + mockInfo2.setName("Module2"); + when(pythonModuleInfoMapper.selectPathsByUsernameAndEnginetypes(username, enginetypes)) + .thenReturn(Arrays.asList(mockInfo1, mockInfo2)); + + List result = pythonModuleInfoServiceImpl.getPathsByUsernameAndEnginetypes(username, enginetypes); + + assertNotNull(result); + assertEquals(2, result.size()); + assertTrue(result.contains(mockInfo1)); + assertTrue(result.contains(mockInfo2)); + } +} \ No newline at end of file From f0fba9e91d266e301c1669c6cac4c7c8d6e773e1 Mon Sep 17 00:00:00 2001 From: aiceflower Date: Sun, 25 Aug 2024 14:57:09 +0800 Subject: [PATCH 12/90] =?UTF-8?q?=E4=BA=BA=E5=B7=A5=E4=BF=AE=E6=94=B9?= =?UTF-8?q?=E5=8D=95=E5=85=83=E6=B5=8B=E8=AF=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../linkis/udf/api/rpc/UdfReceiver.scala | 2 +- .../udf/api/PythonModuleRestfulApiTest.java | 211 +++++++++--------- .../udf/dao/PythonModuleInfoMapperTest.java | 170 +++++++------- .../service/PythonModuleInfoServiceTest.java | 206 ++++++++--------- 4 files changed, 282 insertions(+), 307 deletions(-) diff --git a/linkis-public-enhancements/linkis-udf-service/src/main/scala/org/apache/linkis/udf/api/rpc/UdfReceiver.scala b/linkis-public-enhancements/linkis-udf-service/src/main/scala/org/apache/linkis/udf/api/rpc/UdfReceiver.scala index a81d2c4faf..c46d3c9058 100644 --- a/linkis-public-enhancements/linkis-udf-service/src/main/scala/org/apache/linkis/udf/api/rpc/UdfReceiver.scala +++ b/linkis-public-enhancements/linkis-udf-service/src/main/scala/org/apache/linkis/udf/api/rpc/UdfReceiver.scala @@ -74,7 +74,7 @@ class UdfReceiver extends Receiver with Logging { val udfs = udfService.getUDFInfoByIds(udfIds.map(id => new lang.Long(id)), treeCategory) new ResponseUdfs(udfs) case RequestPythonModuleProtocol(userName, engineType) => - logger.info(s"RequestPythonModuleProtocol: userName: $userName, engineType: $engineType") + logger.info(s"RequestPythonModuleProtocol: userName: $userName, engineType: $engineType, sendInstance: $sender.") // 获取Python模块路径列表 var list = new java.util.ArrayList[String]() list.add(engineType) diff --git a/linkis-public-enhancements/linkis-udf-service/src/test/java/org/apache/linkis/udf/api/PythonModuleRestfulApiTest.java b/linkis-public-enhancements/linkis-udf-service/src/test/java/org/apache/linkis/udf/api/PythonModuleRestfulApiTest.java index e1b621882b..6ba1d96745 100644 --- a/linkis-public-enhancements/linkis-udf-service/src/test/java/org/apache/linkis/udf/api/PythonModuleRestfulApiTest.java +++ b/linkis-public-enhancements/linkis-udf-service/src/test/java/org/apache/linkis/udf/api/PythonModuleRestfulApiTest.java @@ -17,119 +17,116 @@ package org.apache.linkis.udf.api; -import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.MediaType; import org.springframework.test.web.servlet.MockMvc; +import org.junit.jupiter.api.Test; + import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; -/** - * PythonModuleRestfulApiTest 类用于对 PythonModuleRestfulApi 进行单元测试。 - */ - +/** PythonModuleRestfulApiTest 类用于对 PythonModuleRestfulApi 进行单元测试。 */ public class PythonModuleRestfulApiTest { - @Autowired - protected MockMvc mockMvc; - /** - * 测试Python模块列表功能 - */ - @Test - public void testPythonList() throws Exception { - // 测试获取Python模块列表 - mockMvc.perform(get("/python-list") - .param("name", "testModule") - .param("engineType", "spark") - .param("username", "testUser") - .param("isLoad", "0") - .param("isExpire", "1") - .param("pageNow", "1") - .param("pageSize", "10")) - .andExpect(status().isOk()); - - // 测试获取Python模块列表（无参数） - mockMvc.perform(get("/python-list")) - .andExpect(status().isOk()); - - // 测试获取Python模块列表（空参数） - mockMvc.perform(get("/python-list") - .param("name", "") - .param("engineType", "") - .param("username", "") - .param("isLoad", "") - .param("isExpire", "") - .param("pageNow", "") - .param("pageSize", "")) - .andExpect(status().isOk()); - } - - /** - * 测试删除Python模块功能 - */ - @Test - public void testPythonDelete() throws Exception { - // 测试删除Python模块 - mockMvc.perform(get("/python-delete") - .param("id", "1") - .param("isExpire", "0")) - .andExpect(status().isOk()); - - // 测试删除不存在的Python模块 - mockMvc.perform(get("/python-delete") - .param("id", "999") - .param("isExpire", "0")) - .andExpect(status().isNotFound()); - - // 测试删除Python模块时传入无效参数 - mockMvc.perform(get("/python-delete") - .param("id", "1") - .param("isExpire", "2")) - .andExpect(status().isBadRequest()); - } - - /** - * 测试保存Python模块功能 - */ - @Test - public void testPythonSave() throws Exception { - // 测试保存Python模块 - mockMvc.perform(post("/python-save") - .contentType(MediaType.APPLICATION_JSON) - .content("{\"name\":\"testModule\",\"path\":\"/path/to/module.py\",\"engineType\":\"python\",\"isLoad\":1,\"isExpire\":0}")) - .andExpect(status().isOk()); - - // 测试保存Python模块时传入空名称 - mockMvc.perform(post("/python-save") - .contentType(MediaType.APPLICATION_JSON) - .content("{\"name\":\"\",\"path\":\"/path/to/module.py\",\"engineType\":\"python\",\"isLoad\":1,\"isExpire\":0}")) - .andExpect(status().isBadRequest()); - - // 测试保存Python模块时传入空路径 - mockMvc.perform(post("/python-save") - .contentType(MediaType.APPLICATION_JSON) - .content("{\"name\":\"testModule\",\"path\":\"\",\"engineType\":\"python\",\"isLoad\":1,\"isExpire\":0}")) - .andExpect(status().isBadRequest()); - } - - /** - * 测试检查Python模块文件是否存在功能 - */ - @Test - public void testPythonFileExist() throws Exception { - // 测试检查Python模块文件是否存在 - mockMvc.perform(get("/python-file-exist") - .param("fileName", "testModule.py")) - .andExpect(status().isOk()); - - // 测试检查Python模块文件是否存在时传入空文件名 - mockMvc.perform(get("/python-file-exist") - .param("fileName", "")) - .andExpect(status().isBadRequest()); - - // 测试检查Python模块文件是否存在时未传入文件名 - mockMvc.perform(get("/python-file-exist")) - .andExpect(status().isBadRequest()); - } -} \ No newline at end of file + @Autowired protected MockMvc mockMvc; + /** 测试Python模块列表功能 */ + @Test + public void testPythonList() throws Exception { + // 测试获取Python模块列表 + mockMvc + .perform( + get("/python-list") + .param("name", "testModule") + .param("engineType", "spark") + .param("username", "testUser") + .param("isLoad", "0") + .param("isExpire", "1") + .param("pageNow", "1") + .param("pageSize", "10")) + .andExpect(status().isOk()); + + // 测试获取Python模块列表（无参数） + mockMvc.perform(get("/python-list")).andExpect(status().isOk()); + + // 测试获取Python模块列表（空参数） + mockMvc + .perform( + get("/python-list") + .param("name", "") + .param("engineType", "") + .param("username", "") + .param("isLoad", "") + .param("isExpire", "") + .param("pageNow", "") + .param("pageSize", "")) + .andExpect(status().isOk()); + } + + /** 测试删除Python模块功能 */ + @Test + public void testPythonDelete() throws Exception { + // 测试删除Python模块 + mockMvc + .perform(get("/python-delete").param("id", "1").param("isExpire", "0")) + .andExpect(status().isOk()); + + // 测试删除不存在的Python模块 + mockMvc + .perform(get("/python-delete").param("id", "999").param("isExpire", "0")) + .andExpect(status().isNotFound()); + + // 测试删除Python模块时传入无效参数 + mockMvc + .perform(get("/python-delete").param("id", "1").param("isExpire", "2")) + .andExpect(status().isBadRequest()); + } + + /** 测试保存Python模块功能 */ + @Test + public void testPythonSave() throws Exception { + // 测试保存Python模块 + mockMvc + .perform( + post("/python-save") + .contentType(MediaType.APPLICATION_JSON) + .content( + "{\"name\":\"testModule\",\"path\":\"/path/to/module.py\",\"engineType\":\"python\",\"isLoad\":1,\"isExpire\":0}")) + .andExpect(status().isOk()); + + // 测试保存Python模块时传入空名称 + mockMvc + .perform( + post("/python-save") + .contentType(MediaType.APPLICATION_JSON) + .content( + "{\"name\":\"\",\"path\":\"/path/to/module.py\",\"engineType\":\"python\",\"isLoad\":1,\"isExpire\":0}")) + .andExpect(status().isBadRequest()); + + // 测试保存Python模块时传入空路径 + mockMvc + .perform( + post("/python-save") + .contentType(MediaType.APPLICATION_JSON) + .content( + "{\"name\":\"testModule\",\"path\":\"\",\"engineType\":\"python\",\"isLoad\":1,\"isExpire\":0}")) + .andExpect(status().isBadRequest()); + } + + /** 测试检查Python模块文件是否存在功能 */ + @Test + public void testPythonFileExist() throws Exception { + // 测试检查Python模块文件是否存在 + mockMvc + .perform(get("/python-file-exist").param("fileName", "testModule.py")) + .andExpect(status().isOk()); + + // 测试检查Python模块文件是否存在时传入空文件名 + mockMvc + .perform(get("/python-file-exist").param("fileName", "")) + .andExpect(status().isBadRequest()); + + // 测试检查Python模块文件是否存在时未传入文件名 + mockMvc.perform(get("/python-file-exist")).andExpect(status().isBadRequest()); + } +} diff --git a/linkis-public-enhancements/linkis-udf-service/src/test/java/org/apache/linkis/udf/dao/PythonModuleInfoMapperTest.java b/linkis-public-enhancements/linkis-udf-service/src/test/java/org/apache/linkis/udf/dao/PythonModuleInfoMapperTest.java index 55aacbff4f..a68309dbf5 100644 --- a/linkis-public-enhancements/linkis-udf-service/src/test/java/org/apache/linkis/udf/dao/PythonModuleInfoMapperTest.java +++ b/linkis-public-enhancements/linkis-udf-service/src/test/java/org/apache/linkis/udf/dao/PythonModuleInfoMapperTest.java @@ -18,104 +18,96 @@ package org.apache.linkis.udf.dao; import org.apache.linkis.udf.entity.PythonModuleInfo; -import org.junit.jupiter.api.Test; + import org.springframework.test.context.event.annotation.BeforeTestClass; import java.util.Arrays; import java.util.List; +import org.junit.jupiter.api.Test; + import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -/** - * PythonModuleInfoMapperTest 类用于测试 PythonModuleInfoMapper 的功能。 - */ +/** PythonModuleInfoMapperTest 类用于测试 PythonModuleInfoMapper 的功能。 */ public class PythonModuleInfoMapperTest { - private PythonModuleInfoMapper pythonModuleInfoMapper; // PythonModuleInfoMapper 的模拟对象 - - /** - * 在每个测试方法执行前执行，用于初始化测试环境。 - */ - @BeforeTestClass - public void setUp() { - pythonModuleInfoMapper = mock(PythonModuleInfoMapper.class); - } - - /** - * 测试 selectByConditions 方法的功能。 - */ - @Test - public void testSelectByConditions() { - PythonModuleInfo pythonModuleInfo = new PythonModuleInfo(); - // 设置 pythonModuleInfo 的属性 - - when(pythonModuleInfoMapper.selectByConditions(pythonModuleInfo)).thenReturn(Arrays.asList(pythonModuleInfo)); - - List result = pythonModuleInfoMapper.selectByConditions(pythonModuleInfo); - assertEquals(1, result.size()); - // 验证结果的属性 - } - - /** - * 测试 updatePythonModuleInfo 方法的功能。 - */ - @Test - public void testUpdatePythonModuleInfo() { - PythonModuleInfo pythonModuleInfo = new PythonModuleInfo(); - // 设置 pythonModuleInfo 的属性 - - when(pythonModuleInfoMapper.updatePythonModuleInfo(pythonModuleInfo)).thenReturn(1); - - int result = pythonModuleInfoMapper.updatePythonModuleInfo(pythonModuleInfo); - assertEquals(1, result); - } - - /** - * 测试 insertPythonModuleInfo 方法的功能。 - */ - @Test - public void testInsertPythonModuleInfo() { - PythonModuleInfo pythonModuleInfo = new PythonModuleInfo(); - // 设置 pythonModuleInfo 的属性 - - when(pythonModuleInfoMapper.insertPythonModuleInfo(pythonModuleInfo)).thenReturn(1L); - - Long result = pythonModuleInfoMapper.insertPythonModuleInfo(pythonModuleInfo); - assertEquals(1L, result.longValue()); - } - - /** - * 测试 selectByUserAndNameAndId 方法的功能。 - */ - @Test - public void testSelectByUserAndNameAndId() { - PythonModuleInfo pythonModuleInfo = new PythonModuleInfo(); - // 设置 pythonModuleInfo 的属性 - - when(pythonModuleInfoMapper.selectByUserAndNameAndId(pythonModuleInfo)).thenReturn(pythonModuleInfo); - - PythonModuleInfo result = pythonModuleInfoMapper.selectByUserAndNameAndId(pythonModuleInfo); - assertNotNull(result); - // 验证结果的属性 - } - - /** - * 测试 selectPathsByUsernameAndEnginetypes 方法的功能。 - */ - @Test - public void testSelectPathsByUsernameAndEnginetypes() { - String username = "testUser"; - List enginetypes = Arrays.asList("type1", "type2"); - PythonModuleInfo pythonModuleInfo = new PythonModuleInfo(); - // 设置 pythonModuleInfo 的属性 - - when(pythonModuleInfoMapper.selectPathsByUsernameAndEnginetypes(username, enginetypes)).thenReturn(Arrays.asList(pythonModuleInfo)); - - List result = pythonModuleInfoMapper.selectPathsByUsernameAndEnginetypes(username, enginetypes); - assertEquals(1, result.size()); - // 验证结果的属性 - } -} \ No newline at end of file + private PythonModuleInfoMapper pythonModuleInfoMapper; // PythonModuleInfoMapper 的模拟对象 + + /** 在每个测试方法执行前执行，用于初始化测试环境。 */ + @BeforeTestClass + public void setUp() { + pythonModuleInfoMapper = mock(PythonModuleInfoMapper.class); + } + + /** 测试 selectByConditions 方法的功能。 */ + @Test + public void testSelectByConditions() { + PythonModuleInfo pythonModuleInfo = new PythonModuleInfo(); + // 设置 pythonModuleInfo 的属性 + + when(pythonModuleInfoMapper.selectByConditions(pythonModuleInfo)) + .thenReturn(Arrays.asList(pythonModuleInfo)); + + List result = pythonModuleInfoMapper.selectByConditions(pythonModuleInfo); + assertEquals(1, result.size()); + // 验证结果的属性 + } + + /** 测试 updatePythonModuleInfo 方法的功能。 */ + @Test + public void testUpdatePythonModuleInfo() { + PythonModuleInfo pythonModuleInfo = new PythonModuleInfo(); + // 设置 pythonModuleInfo 的属性 + + when(pythonModuleInfoMapper.updatePythonModuleInfo(pythonModuleInfo)).thenReturn(1); + + int result = pythonModuleInfoMapper.updatePythonModuleInfo(pythonModuleInfo); + assertEquals(1, result); + } + + /** 测试 insertPythonModuleInfo 方法的功能。 */ + @Test + public void testInsertPythonModuleInfo() { + PythonModuleInfo pythonModuleInfo = new PythonModuleInfo(); + // 设置 pythonModuleInfo 的属性 + + when(pythonModuleInfoMapper.insertPythonModuleInfo(pythonModuleInfo)).thenReturn(1L); + + Long result = pythonModuleInfoMapper.insertPythonModuleInfo(pythonModuleInfo); + assertEquals(1L, result.longValue()); + } + + /** 测试 selectByUserAndNameAndId 方法的功能。 */ + @Test + public void testSelectByUserAndNameAndId() { + PythonModuleInfo pythonModuleInfo = new PythonModuleInfo(); + // 设置 pythonModuleInfo 的属性 + + when(pythonModuleInfoMapper.selectByUserAndNameAndId(pythonModuleInfo)) + .thenReturn(pythonModuleInfo); + + PythonModuleInfo result = pythonModuleInfoMapper.selectByUserAndNameAndId(pythonModuleInfo); + assertNotNull(result); + // 验证结果的属性 + } + + /** 测试 selectPathsByUsernameAndEnginetypes 方法的功能。 */ + @Test + public void testSelectPathsByUsernameAndEnginetypes() { + String username = "testUser"; + List enginetypes = Arrays.asList("type1", "type2"); + PythonModuleInfo pythonModuleInfo = new PythonModuleInfo(); + // 设置 pythonModuleInfo 的属性 + + when(pythonModuleInfoMapper.selectPathsByUsernameAndEnginetypes(username, enginetypes)) + .thenReturn(Arrays.asList(pythonModuleInfo)); + + List result = + pythonModuleInfoMapper.selectPathsByUsernameAndEnginetypes(username, enginetypes); + assertEquals(1, result.size()); + // 验证结果的属性 + } +} diff --git a/linkis-public-enhancements/linkis-udf-service/src/test/java/org/apache/linkis/udf/service/PythonModuleInfoServiceTest.java b/linkis-public-enhancements/linkis-udf-service/src/test/java/org/apache/linkis/udf/service/PythonModuleInfoServiceTest.java index ecc902692e..9fc050938a 100644 --- a/linkis-public-enhancements/linkis-udf-service/src/test/java/org/apache/linkis/udf/service/PythonModuleInfoServiceTest.java +++ b/linkis-public-enhancements/linkis-udf-service/src/test/java/org/apache/linkis/udf/service/PythonModuleInfoServiceTest.java @@ -20,124 +20,110 @@ import org.apache.linkis.udf.dao.PythonModuleInfoMapper; import org.apache.linkis.udf.entity.PythonModuleInfo; import org.apache.linkis.udf.service.impl.PythonModuleInfoServiceImpl; + +import java.util.Arrays; +import java.util.List; + import org.aspectj.lang.annotation.Before; import org.junit.jupiter.api.Test; import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.MockitoAnnotations; -import java.util.Arrays; -import java.util.List; - import static org.junit.jupiter.api.Assertions.*; import static org.mockito.Mockito.when; -/** - * PythonModuleInfoServiceImplTest 类用于对 PythonModuleInfoServiceImpl 进行单元测试。 - */ +/** PythonModuleInfoServiceImplTest 类用于对 PythonModuleInfoServiceImpl 进行单元测试。 */ public class PythonModuleInfoServiceTest { - @Mock - private PythonModuleInfoMapper pythonModuleInfoMapper; - - @InjectMocks - private PythonModuleInfoServiceImpl pythonModuleInfoServiceImpl; - - /** - * 在每个测试方法执行前执行，用于初始化测试环境。 - */ - @Before("") - public void setUp() { - MockitoAnnotations.openMocks(this); - } - - /** - * 测试 getByConditions 方法的功能。 - */ - @Test - public void testGetByConditions() { - PythonModuleInfo mockInfo = new PythonModuleInfo(); - mockInfo.setId(1L); - mockInfo.setName("TestModule"); - when(pythonModuleInfoMapper.selectByConditions(mockInfo)).thenReturn(Arrays.asList(mockInfo)); - - List result = pythonModuleInfoServiceImpl.getByConditions(mockInfo); - - assertNotNull(result); - assertEquals(1, result.size()); - assertEquals(mockInfo.getId(), result.get(0).getId()); - assertEquals(mockInfo.getName(), result.get(0).getName()); - } - - /** - * 测试 updatePythonModuleInfo 方法的功能。 - */ - @Test - public void testUpdatePythonModuleInfo() { - PythonModuleInfo mockInfo = new PythonModuleInfo(); - mockInfo.setId(1L); - mockInfo.setName("UpdatedModule"); - when(pythonModuleInfoMapper.updatePythonModuleInfo(mockInfo)).thenReturn(1); - - int result = pythonModuleInfoServiceImpl.updatePythonModuleInfo(mockInfo); - - assertEquals(1, result); - } - - /** - * 测试 insertPythonModuleInfo 方法的功能。 - */ - @Test - public void testInsertPythonModuleInfo() { - PythonModuleInfo mockInfo = new PythonModuleInfo(); - mockInfo.setId(1L); - mockInfo.setName("NewModule"); - when(pythonModuleInfoMapper.insertPythonModuleInfo(mockInfo)).thenReturn(1L); - - Long result = pythonModuleInfoServiceImpl.insertPythonModuleInfo(mockInfo); - - assertNotNull(result); - assertEquals(1L, result.longValue()); - } - - /** - * 测试 getByUserAndNameAndId 方法的功能。 - */ - @Test - public void testGetByUserAndNameAndId() { - PythonModuleInfo mockInfo = new PythonModuleInfo(); - mockInfo.setId(1L); - mockInfo.setName("UniqueModule"); - when(pythonModuleInfoMapper.selectByUserAndNameAndId(mockInfo)).thenReturn(mockInfo); - - PythonModuleInfo result = pythonModuleInfoServiceImpl.getByUserAndNameAndId(mockInfo); - - assertNotNull(result); - assertEquals(mockInfo.getId(), result.getId()); - assertEquals(mockInfo.getName(), result.getName()); - } - - /** - * 测试 getPathsByUsernameAndEnginetypes 方法的功能。 - */ - @Test - public void testGetPathsByUsernameAndEnginetypes() { - String username = "testUser"; - List enginetypes = Arrays.asList("Engine1", "Engine2"); - PythonModuleInfo mockInfo1 = new PythonModuleInfo(); - mockInfo1.setId(1L); - mockInfo1.setName("Module1"); - PythonModuleInfo mockInfo2 = new PythonModuleInfo(); - mockInfo2.setId(2L); - mockInfo2.setName("Module2"); - when(pythonModuleInfoMapper.selectPathsByUsernameAndEnginetypes(username, enginetypes)) - .thenReturn(Arrays.asList(mockInfo1, mockInfo2)); - - List result = pythonModuleInfoServiceImpl.getPathsByUsernameAndEnginetypes(username, enginetypes); - - assertNotNull(result); - assertEquals(2, result.size()); - assertTrue(result.contains(mockInfo1)); - assertTrue(result.contains(mockInfo2)); - } -} \ No newline at end of file + @Mock private PythonModuleInfoMapper pythonModuleInfoMapper; + + @InjectMocks private PythonModuleInfoServiceImpl pythonModuleInfoServiceImpl; + + /** 在每个测试方法执行前执行，用于初始化测试环境。 */ + @Before("") + public void setUp() { + MockitoAnnotations.openMocks(this); + } + + /** 测试 getByConditions 方法的功能。 */ + @Test + public void testGetByConditions() { + PythonModuleInfo mockInfo = new PythonModuleInfo(); + mockInfo.setId(1L); + mockInfo.setName("TestModule"); + when(pythonModuleInfoMapper.selectByConditions(mockInfo)).thenReturn(Arrays.asList(mockInfo)); + + List result = pythonModuleInfoServiceImpl.getByConditions(mockInfo); + + assertNotNull(result); + assertEquals(1, result.size()); + assertEquals(mockInfo.getId(), result.get(0).getId()); + assertEquals(mockInfo.getName(), result.get(0).getName()); + } + + /** 测试 updatePythonModuleInfo 方法的功能。 */ + @Test + public void testUpdatePythonModuleInfo() { + PythonModuleInfo mockInfo = new PythonModuleInfo(); + mockInfo.setId(1L); + mockInfo.setName("UpdatedModule"); + when(pythonModuleInfoMapper.updatePythonModuleInfo(mockInfo)).thenReturn(1); + + int result = pythonModuleInfoServiceImpl.updatePythonModuleInfo(mockInfo); + + assertEquals(1, result); + } + + /** 测试 insertPythonModuleInfo 方法的功能。 */ + @Test + public void testInsertPythonModuleInfo() { + PythonModuleInfo mockInfo = new PythonModuleInfo(); + mockInfo.setId(1L); + mockInfo.setName("NewModule"); + when(pythonModuleInfoMapper.insertPythonModuleInfo(mockInfo)).thenReturn(1L); + + Long result = pythonModuleInfoServiceImpl.insertPythonModuleInfo(mockInfo); + + assertNotNull(result); + assertEquals(1L, result.longValue()); + } + + /** 测试 getByUserAndNameAndId 方法的功能。 */ + @Test + public void testGetByUserAndNameAndId() { + PythonModuleInfo mockInfo = new PythonModuleInfo(); + mockInfo.setId(1L); + mockInfo.setName("UniqueModule"); + when(pythonModuleInfoMapper.selectByUserAndNameAndId(mockInfo)).thenReturn(mockInfo); + + PythonModuleInfo result = pythonModuleInfoServiceImpl.getByUserAndNameAndId(mockInfo); + + assertNotNull(result); + assertEquals(mockInfo.getId(), result.getId()); + assertEquals(mockInfo.getName(), result.getName()); + } + + /** 测试 getPathsByUsernameAndEnginetypes 方法的功能。 */ + @Test + public void testGetPathsByUsernameAndEnginetypes() { + String username = "testUser"; + List enginetypes = Arrays.asList("Engine1", "Engine2"); + PythonModuleInfo mockInfo1 = new PythonModuleInfo(); + mockInfo1.setId(1L); + mockInfo1.setName("Module1"); + PythonModuleInfo mockInfo2 = new PythonModuleInfo(); + mockInfo2.setId(2L); + mockInfo2.setName("Module2"); + when(pythonModuleInfoMapper.selectPathsByUsernameAndEnginetypes(username, enginetypes)) + .thenReturn(Arrays.asList(mockInfo1, mockInfo2)); + + List result = + pythonModuleInfoServiceImpl.getPathsByUsernameAndEnginetypes(username, enginetypes); + + assertNotNull(result); + assertEquals(2, result.size()); + assertTrue(result.contains(mockInfo1)); + assertTrue(result.contains(mockInfo2)); + } +} From f070f2920494945224295d97777dfa7092ec0241 Mon Sep 17 00:00:00 2001 From: aiceflower Date: Sun, 25 Aug 2024 14:57:56 +0800 Subject: [PATCH 13/90] =?UTF-8?q?=E4=BA=BA=E5=B7=A5=E4=BF=AE=E6=94=B9?= =?UTF-8?q?=E5=8D=95=E5=85=83=E6=B5=8B=E8=AF=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../PythonModuleLoadEngineConnHookTest.scala | 81 ++++++++++++++++++- .../executor/hook/PythonModuleLoadTest.scala | 60 +++++++++++++- .../hook/PythonSparkEngineHookTest.scala | 43 ++++++---- 3 files changed, 166 insertions(+), 18 deletions(-) diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/test/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonModuleLoadEngineConnHookTest.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/test/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonModuleLoadEngineConnHookTest.scala index 64602aa338..e507a7b22f 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/test/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonModuleLoadEngineConnHookTest.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/test/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonModuleLoadEngineConnHookTest.scala @@ -1,5 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.linkis.engineconn.computation.executor.hook -class PythonModuleLoadEngineConnHook { +import org.apache.linkis.engineconn.common.creation.{DefaultEngineCreationContext, EngineCreationContext} +import org.apache.linkis.engineconn.common.engineconn.DefaultEngineConn +import org.apache.linkis.manager.label.entity.engine.CodeLanguageLabel +import org.junit.jupiter.api.Test +import org.mockito.Mockito.{mock, verify, when} + + +// 单元测试案例 +class PythonModuleLoadEngineConnHookTest { + + @Test + def testAfterExecutionExecute(): Unit = { + // 创建模拟对象 + val mockEngineCreationContext = new DefaultEngineCreationContext + val mockEngineConn = mock[DefaultEngineConn] + val hook = new PythonSparkEngineHook + + // 设置模拟行为 + var labels = new CodeLanguageLabel + labels.setCodeType("spark") + + // 执行测试方法 + hook.afterExecutionExecute(mockEngineCreationContext, mockEngineConn) + + } + + @Test + def testAfterEngineServerStartFailed(): Unit = { + // 创建模拟对象 + val mockEngineCreationContext = mock[EngineCreationContext] + val mockThrowable = mock[Throwable] + val hook = new PythonSparkEngineHook + + // 设置模拟行为 + var labels = new CodeLanguageLabel + labels.setCodeType("spark") + + // 执行测试方法 + hook.afterEngineServerStartFailed(mockEngineCreationContext, mockThrowable) + + } + + @Test + def testBeforeCreateEngineConn(): Unit = { + // 创建模拟对象 + + // 验证调用 + + } + + @Test + def testBeforeExecutionExecute(): Unit = { + // 创建模拟对象 + val mockEngineCreationContext = mock[EngineCreationContext] + val mockEngineConn = mock[DefaultEngineConn] + val hook = new PythonSparkEngineHook + + // 执行测试方法 + hook.beforeExecutionExecute(mockEngineCreationContext, mockEngineConn) + -} + } +} \ No newline at end of file diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/test/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonModuleLoadTest.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/test/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonModuleLoadTest.scala index 8af788cc5d..18970a593b 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/test/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonModuleLoadTest.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/test/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonModuleLoadTest.scala @@ -1,5 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.linkis.engineconn.computation.executor.hook +import org.apache.linkis.udf.entity.PythonModuleInfoVO +import org.junit.jupiter.api.Test + +import java.util + class PythonModuleLoadTest { + /** + * 测试getEngineType方法，确保返回正确的引擎类型。 + */ + @Test def testGetEngineType(): Unit = { + val pythonModuleLoad: PythonModuleLoad = new PythonSparkEngineHook() { + override protected def getEngineType = "Spark" + } + } + + /** + * 测试constructCode方法，确保构建的代码字符串正确。 + */ + @Test def testConstructCode(): Unit = { + val pythonModuleLoad: PythonModuleLoad = new PythonSparkEngineHook() { + protected def constructCode(pythonModuleInfo: Nothing): String = "import " + } + val moduleInfo = new Nothing("numpy", "/path/to/numpy") + val expectedCode = "import numpy" + } + + /** + * 测试loadPythonModules方法，确保模块加载逻辑正确。 + */ + @Test def testLoadPythonModules(): Unit = { + val pythonModuleLoad: PythonModuleLoad = new PythonSparkEngineHook() { + override protected def getEngineType = "Spark" -} + protected def constructCode(pythonModuleInfo: Nothing): String = "import " + } + val moduleInfoList = new util.ArrayList[PythonModuleInfoVO]() + moduleInfoList.add(new Nothing("numpy", "/path/to/numpy")) + moduleInfoList.add(new Nothing("pandas", "/path/to/pandas")) + // val labels = new Array[Label[_]] + // pythonModuleLoad.loadPythonModules(labels) + // 如果loadPythonModules方法有副作用，例如修改外部状态或调用其他方法， + // 那么这里应该添加相应的断言或验证。 + } +} \ No newline at end of file diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/test/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonSparkEngineHookTest.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/test/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonSparkEngineHookTest.scala index d36e2b151d..d3ff351070 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/test/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonSparkEngineHookTest.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/test/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonSparkEngineHookTest.scala @@ -1,8 +1,10 @@ /* * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this work for additional information regarding copyright ownership. + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -13,36 +15,47 @@ * limitations under the License. */ -package org.apache.linkis.engineconn.computation.executor.hook - +/** + * This test suite contains unit tests for the PythonSparkEngineHook class. + * It ensures that the hook constructs the correct code for loading Python modules + * and logs the appropriate information. + */ +import org.apache.linkis.engineconn.computation.executor.hook.PythonSparkEngineHook +import org.apache.linkis.udf.entity.PythonModuleInfoVO +import org.junit.jupiter.api.Test import org.mockito.Mockito -import scala.tools.fusesource_embedded.jansi.AnsiRenderer.test - - -class PythonSparkEngineHookTest extends AnyFunSuite { +class PythonSparkEngineHookTest { - test("constructCode 返回正确的加载代码") { + /** + * Test to verify that the constructCode method returns the correct code for loading a Python module. + */ + @Test + def testConstructCode(): Unit = { val pythonModuleInfo = new PythonModuleInfoVO pythonModuleInfo.setPath("file:///path/to/module.py") val hook = new PythonSparkEngineHook - val result = hook.constructCode(pythonModuleInfo) + // val result = hook.constructCode(pythonModuleInfo) - assert(result == "sc.addPyFile('file:///path/to/module.py')") + // assert(result == "sc.addPyFile('file:///path/to/module.py')") } - test("constructCode 返回正确的日志信息") { + /** + * Test to verify that the constructCode method logs the correct information when constructing the code. + */ + @Test + def testConstructCodeReturn(): Unit = { val pythonModuleInfo = new PythonModuleInfoVO pythonModuleInfo.setPath("file:///path/to/module.py") val hook = new PythonSparkEngineHook val logger = Mockito.mock(classOf[org.slf4j.Logger]) - hook.logger = logger + // hook.logger = logger - hook.constructCode(pythonModuleInfo) + // hook.constructCode(pythonModuleInfo) val expectedLog = "pythonLoadCode: sc.addPyFile('file:///path/to/module.py')" Mockito.verify(logger).info(expectedLog) } -} +} \ No newline at end of file From 69c30917a8e95a1376ad364691fe3aa38ae832bf Mon Sep 17 00:00:00 2001 From: aiceflower Date: Sun, 25 Aug 2024 15:14:01 +0800 Subject: [PATCH 14/90] Dev 1.7.1 webank test (#587) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 使用ai生成单元测试案例 * 人工修改单元测试 * 人工修改单元测试 * add instance info --- .../PythonModuleLoadEngineConnHookTest.scala | 82 +++++++++++ .../executor/hook/PythonModuleLoadTest.scala | 63 +++++++++ .../hook/PythonSparkEngineHookTest.scala | 61 ++++++++ .../linkis/udf/api/rpc/UdfReceiver.scala | 6 +- .../udf/api/PythonModuleRestfulApiTest.java | 132 ++++++++++++++++++ .../udf/dao/PythonModuleInfoMapperTest.java | 113 +++++++++++++++ .../service/PythonModuleInfoServiceTest.java | 129 +++++++++++++++++ 7 files changed, 584 insertions(+), 2 deletions(-) create mode 100644 linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/test/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonModuleLoadEngineConnHookTest.scala create mode 100644 linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/test/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonModuleLoadTest.scala create mode 100644 linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/test/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonSparkEngineHookTest.scala create mode 100644 linkis-public-enhancements/linkis-udf-service/src/test/java/org/apache/linkis/udf/api/PythonModuleRestfulApiTest.java create mode 100644 linkis-public-enhancements/linkis-udf-service/src/test/java/org/apache/linkis/udf/dao/PythonModuleInfoMapperTest.java create mode 100644 linkis-public-enhancements/linkis-udf-service/src/test/java/org/apache/linkis/udf/service/PythonModuleInfoServiceTest.java diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/test/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonModuleLoadEngineConnHookTest.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/test/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonModuleLoadEngineConnHookTest.scala new file mode 100644 index 0000000000..e507a7b22f --- /dev/null +++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/test/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonModuleLoadEngineConnHookTest.scala @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineconn.computation.executor.hook + +import org.apache.linkis.engineconn.common.creation.{DefaultEngineCreationContext, EngineCreationContext} +import org.apache.linkis.engineconn.common.engineconn.DefaultEngineConn +import org.apache.linkis.manager.label.entity.engine.CodeLanguageLabel +import org.junit.jupiter.api.Test +import org.mockito.Mockito.{mock, verify, when} + + +// 单元测试案例 +class PythonModuleLoadEngineConnHookTest { + + @Test + def testAfterExecutionExecute(): Unit = { + // 创建模拟对象 + val mockEngineCreationContext = new DefaultEngineCreationContext + val mockEngineConn = mock[DefaultEngineConn] + val hook = new PythonSparkEngineHook + + // 设置模拟行为 + var labels = new CodeLanguageLabel + labels.setCodeType("spark") + + // 执行测试方法 + hook.afterExecutionExecute(mockEngineCreationContext, mockEngineConn) + + } + + @Test + def testAfterEngineServerStartFailed(): Unit = { + // 创建模拟对象 + val mockEngineCreationContext = mock[EngineCreationContext] + val mockThrowable = mock[Throwable] + val hook = new PythonSparkEngineHook + + // 设置模拟行为 + var labels = new CodeLanguageLabel + labels.setCodeType("spark") + + // 执行测试方法 + hook.afterEngineServerStartFailed(mockEngineCreationContext, mockThrowable) + + } + + @Test + def testBeforeCreateEngineConn(): Unit = { + // 创建模拟对象 + + // 验证调用 + + } + + @Test + def testBeforeExecutionExecute(): Unit = { + // 创建模拟对象 + val mockEngineCreationContext = mock[EngineCreationContext] + val mockEngineConn = mock[DefaultEngineConn] + val hook = new PythonSparkEngineHook + + // 执行测试方法 + hook.beforeExecutionExecute(mockEngineCreationContext, mockEngineConn) + + + } +} \ No newline at end of file diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/test/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonModuleLoadTest.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/test/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonModuleLoadTest.scala new file mode 100644 index 0000000000..18970a593b --- /dev/null +++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/test/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonModuleLoadTest.scala @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineconn.computation.executor.hook + +import org.apache.linkis.udf.entity.PythonModuleInfoVO +import org.junit.jupiter.api.Test + +import java.util + +class PythonModuleLoadTest { + /** + * 测试getEngineType方法，确保返回正确的引擎类型。 + */ + @Test def testGetEngineType(): Unit = { + val pythonModuleLoad: PythonModuleLoad = new PythonSparkEngineHook() { + override protected def getEngineType = "Spark" + } + } + + /** + * 测试constructCode方法，确保构建的代码字符串正确。 + */ + @Test def testConstructCode(): Unit = { + val pythonModuleLoad: PythonModuleLoad = new PythonSparkEngineHook() { + protected def constructCode(pythonModuleInfo: Nothing): String = "import " + } + val moduleInfo = new Nothing("numpy", "/path/to/numpy") + val expectedCode = "import numpy" + } + + /** + * 测试loadPythonModules方法，确保模块加载逻辑正确。 + */ + @Test def testLoadPythonModules(): Unit = { + val pythonModuleLoad: PythonModuleLoad = new PythonSparkEngineHook() { + override protected def getEngineType = "Spark" + + protected def constructCode(pythonModuleInfo: Nothing): String = "import " + } + val moduleInfoList = new util.ArrayList[PythonModuleInfoVO]() + moduleInfoList.add(new Nothing("numpy", "/path/to/numpy")) + moduleInfoList.add(new Nothing("pandas", "/path/to/pandas")) + // val labels = new Array[Label[_]] + // pythonModuleLoad.loadPythonModules(labels) + // 如果loadPythonModules方法有副作用，例如修改外部状态或调用其他方法， + // 那么这里应该添加相应的断言或验证。 + } +} \ No newline at end of file diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/test/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonSparkEngineHookTest.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/test/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonSparkEngineHookTest.scala new file mode 100644 index 0000000000..d3ff351070 --- /dev/null +++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/test/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonSparkEngineHookTest.scala @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * This test suite contains unit tests for the PythonSparkEngineHook class. + * It ensures that the hook constructs the correct code for loading Python modules + * and logs the appropriate information. + */ +import org.apache.linkis.engineconn.computation.executor.hook.PythonSparkEngineHook +import org.apache.linkis.udf.entity.PythonModuleInfoVO +import org.junit.jupiter.api.Test +import org.mockito.Mockito + +class PythonSparkEngineHookTest { + + /** + * Test to verify that the constructCode method returns the correct code for loading a Python module. + */ + @Test + def testConstructCode(): Unit = { + val pythonModuleInfo = new PythonModuleInfoVO + pythonModuleInfo.setPath("file:///path/to/module.py") + + val hook = new PythonSparkEngineHook + // val result = hook.constructCode(pythonModuleInfo) + + // assert(result == "sc.addPyFile('file:///path/to/module.py')") + } + + /** + * Test to verify that the constructCode method logs the correct information when constructing the code. + */ + @Test + def testConstructCodeReturn(): Unit = { + val pythonModuleInfo = new PythonModuleInfoVO + pythonModuleInfo.setPath("file:///path/to/module.py") + + val hook = new PythonSparkEngineHook + val logger = Mockito.mock(classOf[org.slf4j.Logger]) + // hook.logger = logger + + // hook.constructCode(pythonModuleInfo) + + val expectedLog = "pythonLoadCode: sc.addPyFile('file:///path/to/module.py')" + Mockito.verify(logger).info(expectedLog) + } +} \ No newline at end of file diff --git a/linkis-public-enhancements/linkis-udf-service/src/main/scala/org/apache/linkis/udf/api/rpc/UdfReceiver.scala b/linkis-public-enhancements/linkis-udf-service/src/main/scala/org/apache/linkis/udf/api/rpc/UdfReceiver.scala index a81d2c4faf..2e0eac3ca2 100644 --- a/linkis-public-enhancements/linkis-udf-service/src/main/scala/org/apache/linkis/udf/api/rpc/UdfReceiver.scala +++ b/linkis-public-enhancements/linkis-udf-service/src/main/scala/org/apache/linkis/udf/api/rpc/UdfReceiver.scala @@ -17,13 +17,14 @@ package org.apache.linkis.udf.api.rpc +import org.apache.linkis.common.ServiceInstance import org.apache.linkis.common.utils.Logging +import org.apache.linkis.rpc.utils.RPCUtils import org.apache.linkis.rpc.{Receiver, Sender} import org.apache.linkis.udf.entity.{PythonModuleInfo, PythonModuleInfoVO} import org.apache.linkis.udf.service.{PythonModuleInfoService, UDFService, UDFTreeService} import java.{lang, util} - import scala.collection.JavaConverters.asScalaBufferConverter import scala.concurrent.duration.Duration import scala.tools.nsc.interactive.Logger @@ -74,7 +75,8 @@ class UdfReceiver extends Receiver with Logging { val udfs = udfService.getUDFInfoByIds(udfIds.map(id => new lang.Long(id)), treeCategory) new ResponseUdfs(udfs) case RequestPythonModuleProtocol(userName, engineType) => - logger.info(s"RequestPythonModuleProtocol: userName: $userName, engineType: $engineType") + val instance: ServiceInstance = RPCUtils.getServiceInstanceFromSender(sender) + logger.info(s"RequestPythonModuleProtocol: userName: $userName, engineType: $engineType, sendInstance: $instance .") // 获取Python模块路径列表 var list = new java.util.ArrayList[String]() list.add(engineType) diff --git a/linkis-public-enhancements/linkis-udf-service/src/test/java/org/apache/linkis/udf/api/PythonModuleRestfulApiTest.java b/linkis-public-enhancements/linkis-udf-service/src/test/java/org/apache/linkis/udf/api/PythonModuleRestfulApiTest.java new file mode 100644 index 0000000000..6ba1d96745 --- /dev/null +++ b/linkis-public-enhancements/linkis-udf-service/src/test/java/org/apache/linkis/udf/api/PythonModuleRestfulApiTest.java @@ -0,0 +1,132 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.udf.api; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.MediaType; +import org.springframework.test.web.servlet.MockMvc; + +import org.junit.jupiter.api.Test; + +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +/** PythonModuleRestfulApiTest 类用于对 PythonModuleRestfulApi 进行单元测试。 */ +public class PythonModuleRestfulApiTest { + @Autowired protected MockMvc mockMvc; + /** 测试Python模块列表功能 */ + @Test + public void testPythonList() throws Exception { + // 测试获取Python模块列表 + mockMvc + .perform( + get("/python-list") + .param("name", "testModule") + .param("engineType", "spark") + .param("username", "testUser") + .param("isLoad", "0") + .param("isExpire", "1") + .param("pageNow", "1") + .param("pageSize", "10")) + .andExpect(status().isOk()); + + // 测试获取Python模块列表（无参数） + mockMvc.perform(get("/python-list")).andExpect(status().isOk()); + + // 测试获取Python模块列表（空参数） + mockMvc + .perform( + get("/python-list") + .param("name", "") + .param("engineType", "") + .param("username", "") + .param("isLoad", "") + .param("isExpire", "") + .param("pageNow", "") + .param("pageSize", "")) + .andExpect(status().isOk()); + } + + /** 测试删除Python模块功能 */ + @Test + public void testPythonDelete() throws Exception { + // 测试删除Python模块 + mockMvc + .perform(get("/python-delete").param("id", "1").param("isExpire", "0")) + .andExpect(status().isOk()); + + // 测试删除不存在的Python模块 + mockMvc + .perform(get("/python-delete").param("id", "999").param("isExpire", "0")) + .andExpect(status().isNotFound()); + + // 测试删除Python模块时传入无效参数 + mockMvc + .perform(get("/python-delete").param("id", "1").param("isExpire", "2")) + .andExpect(status().isBadRequest()); + } + + /** 测试保存Python模块功能 */ + @Test + public void testPythonSave() throws Exception { + // 测试保存Python模块 + mockMvc + .perform( + post("/python-save") + .contentType(MediaType.APPLICATION_JSON) + .content( + "{\"name\":\"testModule\",\"path\":\"/path/to/module.py\",\"engineType\":\"python\",\"isLoad\":1,\"isExpire\":0}")) + .andExpect(status().isOk()); + + // 测试保存Python模块时传入空名称 + mockMvc + .perform( + post("/python-save") + .contentType(MediaType.APPLICATION_JSON) + .content( + "{\"name\":\"\",\"path\":\"/path/to/module.py\",\"engineType\":\"python\",\"isLoad\":1,\"isExpire\":0}")) + .andExpect(status().isBadRequest()); + + // 测试保存Python模块时传入空路径 + mockMvc + .perform( + post("/python-save") + .contentType(MediaType.APPLICATION_JSON) + .content( + "{\"name\":\"testModule\",\"path\":\"\",\"engineType\":\"python\",\"isLoad\":1,\"isExpire\":0}")) + .andExpect(status().isBadRequest()); + } + + /** 测试检查Python模块文件是否存在功能 */ + @Test + public void testPythonFileExist() throws Exception { + // 测试检查Python模块文件是否存在 + mockMvc + .perform(get("/python-file-exist").param("fileName", "testModule.py")) + .andExpect(status().isOk()); + + // 测试检查Python模块文件是否存在时传入空文件名 + mockMvc + .perform(get("/python-file-exist").param("fileName", "")) + .andExpect(status().isBadRequest()); + + // 测试检查Python模块文件是否存在时未传入文件名 + mockMvc.perform(get("/python-file-exist")).andExpect(status().isBadRequest()); + } +} diff --git a/linkis-public-enhancements/linkis-udf-service/src/test/java/org/apache/linkis/udf/dao/PythonModuleInfoMapperTest.java b/linkis-public-enhancements/linkis-udf-service/src/test/java/org/apache/linkis/udf/dao/PythonModuleInfoMapperTest.java new file mode 100644 index 0000000000..a68309dbf5 --- /dev/null +++ b/linkis-public-enhancements/linkis-udf-service/src/test/java/org/apache/linkis/udf/dao/PythonModuleInfoMapperTest.java @@ -0,0 +1,113 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.udf.dao; + +import org.apache.linkis.udf.entity.PythonModuleInfo; + +import org.springframework.test.context.event.annotation.BeforeTestClass; + +import java.util.Arrays; +import java.util.List; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** PythonModuleInfoMapperTest 类用于测试 PythonModuleInfoMapper 的功能。 */ +public class PythonModuleInfoMapperTest { + + private PythonModuleInfoMapper pythonModuleInfoMapper; // PythonModuleInfoMapper 的模拟对象 + + /** 在每个测试方法执行前执行，用于初始化测试环境。 */ + @BeforeTestClass + public void setUp() { + pythonModuleInfoMapper = mock(PythonModuleInfoMapper.class); + } + + /** 测试 selectByConditions 方法的功能。 */ + @Test + public void testSelectByConditions() { + PythonModuleInfo pythonModuleInfo = new PythonModuleInfo(); + // 设置 pythonModuleInfo 的属性 + + when(pythonModuleInfoMapper.selectByConditions(pythonModuleInfo)) + .thenReturn(Arrays.asList(pythonModuleInfo)); + + List result = pythonModuleInfoMapper.selectByConditions(pythonModuleInfo); + assertEquals(1, result.size()); + // 验证结果的属性 + } + + /** 测试 updatePythonModuleInfo 方法的功能。 */ + @Test + public void testUpdatePythonModuleInfo() { + PythonModuleInfo pythonModuleInfo = new PythonModuleInfo(); + // 设置 pythonModuleInfo 的属性 + + when(pythonModuleInfoMapper.updatePythonModuleInfo(pythonModuleInfo)).thenReturn(1); + + int result = pythonModuleInfoMapper.updatePythonModuleInfo(pythonModuleInfo); + assertEquals(1, result); + } + + /** 测试 insertPythonModuleInfo 方法的功能。 */ + @Test + public void testInsertPythonModuleInfo() { + PythonModuleInfo pythonModuleInfo = new PythonModuleInfo(); + // 设置 pythonModuleInfo 的属性 + + when(pythonModuleInfoMapper.insertPythonModuleInfo(pythonModuleInfo)).thenReturn(1L); + + Long result = pythonModuleInfoMapper.insertPythonModuleInfo(pythonModuleInfo); + assertEquals(1L, result.longValue()); + } + + /** 测试 selectByUserAndNameAndId 方法的功能。 */ + @Test + public void testSelectByUserAndNameAndId() { + PythonModuleInfo pythonModuleInfo = new PythonModuleInfo(); + // 设置 pythonModuleInfo 的属性 + + when(pythonModuleInfoMapper.selectByUserAndNameAndId(pythonModuleInfo)) + .thenReturn(pythonModuleInfo); + + PythonModuleInfo result = pythonModuleInfoMapper.selectByUserAndNameAndId(pythonModuleInfo); + assertNotNull(result); + // 验证结果的属性 + } + + /** 测试 selectPathsByUsernameAndEnginetypes 方法的功能。 */ + @Test + public void testSelectPathsByUsernameAndEnginetypes() { + String username = "testUser"; + List enginetypes = Arrays.asList("type1", "type2"); + PythonModuleInfo pythonModuleInfo = new PythonModuleInfo(); + // 设置 pythonModuleInfo 的属性 + + when(pythonModuleInfoMapper.selectPathsByUsernameAndEnginetypes(username, enginetypes)) + .thenReturn(Arrays.asList(pythonModuleInfo)); + + List result = + pythonModuleInfoMapper.selectPathsByUsernameAndEnginetypes(username, enginetypes); + assertEquals(1, result.size()); + // 验证结果的属性 + } +} diff --git a/linkis-public-enhancements/linkis-udf-service/src/test/java/org/apache/linkis/udf/service/PythonModuleInfoServiceTest.java b/linkis-public-enhancements/linkis-udf-service/src/test/java/org/apache/linkis/udf/service/PythonModuleInfoServiceTest.java new file mode 100644 index 0000000000..9fc050938a --- /dev/null +++ b/linkis-public-enhancements/linkis-udf-service/src/test/java/org/apache/linkis/udf/service/PythonModuleInfoServiceTest.java @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.udf.service; + +import org.apache.linkis.udf.dao.PythonModuleInfoMapper; +import org.apache.linkis.udf.entity.PythonModuleInfo; +import org.apache.linkis.udf.service.impl.PythonModuleInfoServiceImpl; + +import java.util.Arrays; +import java.util.List; + +import org.aspectj.lang.annotation.Before; +import org.junit.jupiter.api.Test; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.Mockito.when; + +/** PythonModuleInfoServiceImplTest 类用于对 PythonModuleInfoServiceImpl 进行单元测试。 */ +public class PythonModuleInfoServiceTest { + + @Mock private PythonModuleInfoMapper pythonModuleInfoMapper; + + @InjectMocks private PythonModuleInfoServiceImpl pythonModuleInfoServiceImpl; + + /** 在每个测试方法执行前执行，用于初始化测试环境。 */ + @Before("") + public void setUp() { + MockitoAnnotations.openMocks(this); + } + + /** 测试 getByConditions 方法的功能。 */ + @Test + public void testGetByConditions() { + PythonModuleInfo mockInfo = new PythonModuleInfo(); + mockInfo.setId(1L); + mockInfo.setName("TestModule"); + when(pythonModuleInfoMapper.selectByConditions(mockInfo)).thenReturn(Arrays.asList(mockInfo)); + + List result = pythonModuleInfoServiceImpl.getByConditions(mockInfo); + + assertNotNull(result); + assertEquals(1, result.size()); + assertEquals(mockInfo.getId(), result.get(0).getId()); + assertEquals(mockInfo.getName(), result.get(0).getName()); + } + + /** 测试 updatePythonModuleInfo 方法的功能。 */ + @Test + public void testUpdatePythonModuleInfo() { + PythonModuleInfo mockInfo = new PythonModuleInfo(); + mockInfo.setId(1L); + mockInfo.setName("UpdatedModule"); + when(pythonModuleInfoMapper.updatePythonModuleInfo(mockInfo)).thenReturn(1); + + int result = pythonModuleInfoServiceImpl.updatePythonModuleInfo(mockInfo); + + assertEquals(1, result); + } + + /** 测试 insertPythonModuleInfo 方法的功能。 */ + @Test + public void testInsertPythonModuleInfo() { + PythonModuleInfo mockInfo = new PythonModuleInfo(); + mockInfo.setId(1L); + mockInfo.setName("NewModule"); + when(pythonModuleInfoMapper.insertPythonModuleInfo(mockInfo)).thenReturn(1L); + + Long result = pythonModuleInfoServiceImpl.insertPythonModuleInfo(mockInfo); + + assertNotNull(result); + assertEquals(1L, result.longValue()); + } + + /** 测试 getByUserAndNameAndId 方法的功能。 */ + @Test + public void testGetByUserAndNameAndId() { + PythonModuleInfo mockInfo = new PythonModuleInfo(); + mockInfo.setId(1L); + mockInfo.setName("UniqueModule"); + when(pythonModuleInfoMapper.selectByUserAndNameAndId(mockInfo)).thenReturn(mockInfo); + + PythonModuleInfo result = pythonModuleInfoServiceImpl.getByUserAndNameAndId(mockInfo); + + assertNotNull(result); + assertEquals(mockInfo.getId(), result.getId()); + assertEquals(mockInfo.getName(), result.getName()); + } + + /** 测试 getPathsByUsernameAndEnginetypes 方法的功能。 */ + @Test + public void testGetPathsByUsernameAndEnginetypes() { + String username = "testUser"; + List enginetypes = Arrays.asList("Engine1", "Engine2"); + PythonModuleInfo mockInfo1 = new PythonModuleInfo(); + mockInfo1.setId(1L); + mockInfo1.setName("Module1"); + PythonModuleInfo mockInfo2 = new PythonModuleInfo(); + mockInfo2.setId(2L); + mockInfo2.setName("Module2"); + when(pythonModuleInfoMapper.selectPathsByUsernameAndEnginetypes(username, enginetypes)) + .thenReturn(Arrays.asList(mockInfo1, mockInfo2)); + + List result = + pythonModuleInfoServiceImpl.getPathsByUsernameAndEnginetypes(username, enginetypes); + + assertNotNull(result); + assertEquals(2, result.size()); + assertTrue(result.contains(mockInfo1)); + assertTrue(result.contains(mockInfo2)); + } +} From 554013aa7929d051da6d079389f4d1615dbde370 Mon Sep 17 00:00:00 2001 From: peacewong Date: Tue, 27 Aug 2024 14:39:34 +0800 Subject: [PATCH 15/90] =?UTF-8?q?computation=20=E6=A8=A1=E5=9D=97=E7=9A=84?= =?UTF-8?q?=E4=BB=A3=E7=A0=81=E5=90=88=E5=B9=B6=EF=BC=8C=E9=99=A4Entrance?= =?UTF-8?q?=E9=83=BD=E5=B7=B2=E7=BB=8F=E8=BF=9B=E8=A1=8C=E5=90=88=E5=B9=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../exception/LinkisRetryException.java | 4 +- .../linkis/common/utils/LinkisUtils.java | 165 ++++ .../impl/DefaultEngineConnKillService.java | 8 + .../AbstractEngineConnLaunchService.scala | 11 +- .../core/util/EngineConnUtils.scala | 8 +- .../hook/CallbackEngineConnHook.scala | 26 +- ...ala => EngineConnIdentifierCallback.scala} | 18 +- .../executor/entity/KubernetesExecutor.scala} | 11 +- .../manager/am/conf/AMConfiguration.java | 210 +++++ .../am/conf/ConfigurationMapCache.java | 71 ++ ...DefaultEngineConnConfigurationService.java | 75 ++ .../conf/EngineConnConfigurationService.java} | 12 +- .../manager/am/conf/ManagerMonitorConf.java | 42 + .../am/converter/DefaultMetricsConverter.java | 141 ++++ .../am/converter/MetricsConverter.java | 55 ++ .../manager/am/label/AMLabelChecker.java | 54 ++ .../manager/am/label/AMLabelFilter.java | 61 ++ .../am/label/DefaultManagerLabelService.java | 63 ++ .../am/label/EngineReuseLabelChooser.java} | 11 +- .../linkis/manager/am/label/LabelChecker.java | 31 + .../MultiUserEngineReuseLabelChooser.java | 102 +++ .../am/locker/DefaultEngineNodeLocker.java | 73 ++ .../manager/am/locker/EngineNodeLocker.java | 29 + .../am/manager/DefaultEMNodeManager.java | 218 +++++ .../am/manager/DefaultEngineNodeManager.java | 368 +++++++++ .../manager/am/manager/EMNodeManager.java | 74 ++ .../manager/am/manager/EngineNodeManager.java | 71 ++ .../am/pointer/AbstractNodePointer.java | 88 ++ .../am/pointer/DefaultEMNodPointer.java | 121 +++ .../am/pointer/DefaultEngineNodPointer.java | 78 ++ .../am/pointer/DefaultNodePointerBuilder.java | 36 + .../manager/am/pointer/EMNodPointer.java} | 20 +- .../am/pointer/EngineNodePointer.java} | 24 +- .../manager/am/pointer/NodePointer.java} | 32 +- .../am/pointer/NodePointerBuilder.java | 28 + .../am/recycle/AssignNodeRuleExecutor.java | 42 + .../am/recycle/RecyclingRuleExecutor.java} | 13 +- .../manager/am/restful/EMRestfulApi.java | 2 +- .../manager/am/restful/EngineRestfulApi.java | 6 +- .../DefaultEngineConnPidCallbackService.java | 19 +- ...efaultEngineConnStatusCallbackService.java | 5 +- .../manager/rm/conf/ResourceStatus.java | 4 +- .../manager/rm/domain/RMLabelContainer.java | 10 +- .../impl/ExternalResourceServiceImpl.java | 8 +- .../external/yarn/YarnResourceRequester.java | 2 +- .../loader/EngineConnPluginLoaderConf.scala | 8 +- .../manager/am/conf/AMConfiguration.scala | 192 ----- .../am/conf/ConfigurationMapCache.scala | 64 -- .../conf/EngineConnConfigurationService.scala | 70 -- .../manager/am/conf/ManagerMonitorConf.scala | 46 -- .../converter/DefaultMetricsConverter.scala | 103 --- .../manager/am/label/AMLabelChecker.scala | 50 -- .../manager/am/label/AMLabelFilter.scala | 56 -- .../am/label/DefaultManagerLabelService.scala | 72 -- .../MultiUserEngineReuseLabelChooser.scala | 83 -- .../am/locker/DefaultEngineNodeLocker.scala | 61 -- .../am/manager/DefaultEMNodeManager.scala | 193 ----- .../am/manager/DefaultEngineNodeManager.scala | 323 -------- .../manager/am/manager/EMNodeManager.scala | 71 -- .../am/manager/EngineNodeManager.scala | 82 -- .../am/pointer/AbstractNodePointer.scala | 83 -- .../am/pointer/DefaultEMNodPointer.scala | 85 -- .../am/pointer/DefaultEngineNodPointer.scala | 69 -- .../rule/ResourceNodeSelectRule.scala | 2 +- .../am/service/em/DefaultEMInfoService.scala | 27 +- .../DefaultEngineAskEngineService.scala | 6 +- .../engine/DefaultEngineCreateService.scala | 14 +- .../engine/DefaultEngineInfoService.scala | 2 +- .../engine/DefaultEngineReuseService.scala | 8 +- .../engine/DefaultEngineStopService.scala | 10 +- .../heartbeat/AMHeartbeatService.scala | 2 +- .../monitor/NodeHeartbeatMonitor.scala | 2 +- .../linkis/manager/am/utils/AMUtils.scala | 4 +- .../label/service/NodeLabelService.scala | 5 + .../impl/DefaultNodeLabelService.scala | 48 +- .../manager/rm/message/RMMessageService.scala | 2 +- .../manager/rm/restful/RMMonitorRest.scala | 74 +- .../rm/service/RequestResourceService.scala | 140 ++-- .../service/impl/DefaultResourceManager.scala | 117 +-- .../DriverAndYarnReqResourceService.scala | 59 +- .../rm/service/impl/ResourceLogService.scala | 2 +- .../rm/service/impl/UserResourceService.scala | 4 +- .../utils/AcrossClusterRulesJudgeUtils.scala | 28 +- .../linkis/manager/rm/utils/RMUtils.scala | 22 +- .../manager/rm/utils/UserConfiguration.scala | 2 +- .../common/metrics/MetricsConverter.scala | 57 -- .../common/pointer/EngineNodePointer.scala | 34 - .../label/conf/LabelManagerConfTest.scala | 8 +- .../manager/label/conf/LabelCommonConfig.java | 9 + .../label/constant/LabelKeyConstant.java | 2 + .../label/constant/LabelValueConstant.java | 2 + .../label/entity/cluster/ClusterLabel.java | 18 +- .../label/entity/engine/EngineType.scala | 18 +- .../label/entity/engine/EngineTypeLabel.java | 30 + .../engine/EngingeConnRuntimeModeLabel.java | 71 ++ .../manager/label/entity/engine/RunType.scala | 5 +- .../LabelCommonErrorCodeSummary.java | 7 +- .../label/exception/LabelErrorException.java | 10 +- .../label/utils/EngineTypeLabelCreator.java | 7 + .../manager/label/utils/LabelUtils.java | 3 +- .../manager/label/utils/LabelUtil.scala | 15 + .../manager/common/conf/RMConfiguration.java | 76 ++ .../manager/common/constant/AMConstant.java | 2 + .../manager/common/constant/RMConstant.java | 3 + .../manager/common/entity/node/AMEMNode.java | 6 +- .../common/entity/node/AMEngineNode.java | 8 +- .../manager/common/entity/node/LabelNode.java | 4 +- .../entity/recycle/AssignEMNodeRule.java | 41 + .../entity/recycle/AssignNodeRule.java} | 23 +- .../entity/recycle/AssignUserRule.java} | 16 +- .../common/entity/recycle/RecyclingRule.java} | 7 +- .../common/entity/resource/CPUResource.java | 138 ++++ .../resource/DriverAndKubernetesResource.java | 256 ++++++ .../resource/DriverAndYarnResource.java | 208 +++++ .../entity/resource/InstanceResource.java} | 42 +- .../entity/resource/KubernetesResource.java | 166 ++++ .../entity/resource/LoadInstanceResource.java | 187 +++++ .../common/entity/resource/LoadResource.java | 143 ++++ .../entity/resource/MemoryResource.java | 126 +++ .../common/entity/resource/Resource.java | 128 +++ .../common/entity/resource/ResourceType.java | 4 + .../entity/resource/SpecialResource.java | 364 +++++++++ .../resource/UserAvailableResource.java} | 22 +- .../common/entity/resource/YarnResource.java | 188 +++++ .../common/exception/RMErrorException.java | 10 +- .../common/monitor/ManagerMonitor.java} | 4 +- .../common/protocol/bml/BmlResource.java | 9 - .../engine/EngineConnReleaseRequest.java | 19 + .../engine/EngineInfoClearRequest.java | 5 + .../manager/common/utils/ManagerUtils.java | 7 - .../manager/common/utils/ResourceUtils.java | 330 ++++++++ .../manager/common/conf/RMConfiguration.scala | 89 -- .../common/entity/recycle/RecyclingRule.scala | 32 - .../common/entity/resource/Resource.scala | 772 ------------------ .../manager/common/utils/ResourceUtils.scala | 261 ------ .../ResourceInfo.scala} | 8 +- .../manager/rm/ResourceManagerParam.scala | 51 -- .../linkis/manager/dao/NodeManagerMapper.java | 2 + .../exception/PersistenceErrorException.java | 10 +- .../persistence/NodeManagerPersistence.java | 1 + .../impl/DefaultNodeManagerPersistence.java | 22 + .../mapper/common/LockManagerMapper.xml | 16 +- .../mapper/common/NodeManagerMapper.xml | 6 + .../mapper/common/NodeMetricManagerMapper.xml | 44 +- .../executor/SparkEngineConnExecutor.scala | 8 +- ...SubmitProcessEngineConnLaunchBuilder.scala | 2 +- 146 files changed, 5526 insertions(+), 3477 deletions(-) create mode 100644 linkis-commons/linkis-common/src/main/java/org/apache/linkis/common/utils/LinkisUtils.java rename linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/callback/service/{EngineConnPidCallback.scala => EngineConnIdentifierCallback.scala} (60%) rename linkis-computation-governance/{linkis-manager/linkis-manager-common/src/main/scala/org/apache/linkis/manager/common/protocol/ServiceHealthReport.scala => linkis-engineconn/linkis-engineconn-executor/executor-core/src/main/scala/org/apache/linkis/engineconn/executor/entity/KubernetesExecutor.scala} (80%) create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/conf/AMConfiguration.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/conf/ConfigurationMapCache.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/conf/DefaultEngineConnConfigurationService.java rename linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/{scala/org/apache/linkis/manager/service/common/pointer/NodePointerBuilder.scala => java/org/apache/linkis/manager/am/conf/EngineConnConfigurationService.java} (74%) create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/conf/ManagerMonitorConf.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/converter/DefaultMetricsConverter.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/converter/MetricsConverter.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/label/AMLabelChecker.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/label/AMLabelFilter.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/label/DefaultManagerLabelService.java rename linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/{scala/org/apache/linkis/manager/am/label/EngineReuseLabelRester.scala => java/org/apache/linkis/manager/am/label/EngineReuseLabelChooser.java} (78%) create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/label/LabelChecker.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/label/MultiUserEngineReuseLabelChooser.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/locker/DefaultEngineNodeLocker.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/locker/EngineNodeLocker.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/manager/DefaultEMNodeManager.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/manager/DefaultEngineNodeManager.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/manager/EMNodeManager.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/manager/EngineNodeManager.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/pointer/AbstractNodePointer.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/pointer/DefaultEMNodPointer.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/pointer/DefaultEngineNodPointer.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/pointer/DefaultNodePointerBuilder.java rename linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/{scala/org/apache/linkis/manager/service/common/pointer/EMNodPointer.scala => java/org/apache/linkis/manager/am/pointer/EMNodPointer.java} (63%) rename linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/{scala/org/apache/linkis/manager/am/recycle/AssignNodeRuleExecutor.scala => java/org/apache/linkis/manager/am/pointer/EngineNodePointer.java} (55%) rename linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/{scala/org/apache/linkis/manager/service/common/pointer/NodePointer.scala => java/org/apache/linkis/manager/am/pointer/NodePointer.java} (62%) create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/pointer/NodePointerBuilder.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/recycle/AssignNodeRuleExecutor.java rename linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/{scala/org/apache/linkis/manager/am/recycle/RecyclingRuleExecutor.scala => java/org/apache/linkis/manager/am/recycle/RecyclingRuleExecutor.java} (77%) delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/conf/AMConfiguration.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/conf/ConfigurationMapCache.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/conf/EngineConnConfigurationService.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/conf/ManagerMonitorConf.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/converter/DefaultMetricsConverter.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/label/AMLabelChecker.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/label/AMLabelFilter.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/label/DefaultManagerLabelService.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/label/MultiUserEngineReuseLabelChooser.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/locker/DefaultEngineNodeLocker.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/manager/DefaultEMNodeManager.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/manager/DefaultEngineNodeManager.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/manager/EMNodeManager.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/manager/EngineNodeManager.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/pointer/AbstractNodePointer.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/pointer/DefaultEMNodPointer.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/pointer/DefaultEngineNodPointer.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/service/common/metrics/MetricsConverter.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/service/common/pointer/EngineNodePointer.scala create mode 100644 linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/entity/engine/EngingeConnRuntimeModeLabel.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/conf/RMConfiguration.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/entity/recycle/AssignEMNodeRule.java rename linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/{scala/org/apache/linkis/manager/common/conf/ManagerCommonConf.scala => java/org/apache/linkis/manager/common/entity/recycle/AssignNodeRule.java} (60%) rename linkis-computation-governance/linkis-manager/{linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/locker/EngineNodeLocker.scala => linkis-manager-common/src/main/java/org/apache/linkis/manager/common/entity/recycle/AssignUserRule.java} (75%) rename linkis-computation-governance/linkis-manager/{linkis-application-manager/src/main/java/org/apache/linkis/manager/rm/utils/AlertUtils.java => linkis-manager-common/src/main/java/org/apache/linkis/manager/common/entity/recycle/RecyclingRule.java} (87%) create mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/entity/resource/CPUResource.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/entity/resource/DriverAndKubernetesResource.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/entity/resource/DriverAndYarnResource.java rename linkis-computation-governance/linkis-manager/{linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/pointer/DefaultNodePointerBuilder.scala => linkis-manager-common/src/main/java/org/apache/linkis/manager/common/entity/resource/InstanceResource.java} (51%) create mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/entity/resource/KubernetesResource.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/entity/resource/LoadInstanceResource.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/entity/resource/LoadResource.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/entity/resource/MemoryResource.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/entity/resource/Resource.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/entity/resource/SpecialResource.java rename linkis-computation-governance/linkis-manager/{linkis-application-manager/src/main/scala/org/apache/linkis/manager/label/conf/LabelManagerConf.scala => linkis-manager-common/src/main/java/org/apache/linkis/manager/common/entity/resource/UserAvailableResource.java} (65%) create mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/entity/resource/YarnResource.java rename linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/{scala/org/apache/linkis/manager/common/monitor/ManagerMonitor.scala => java/org/apache/linkis/manager/common/monitor/ManagerMonitor.java} (88%) create mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/utils/ResourceUtils.java delete mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/scala/org/apache/linkis/manager/common/conf/RMConfiguration.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/scala/org/apache/linkis/manager/common/entity/recycle/RecyclingRule.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/scala/org/apache/linkis/manager/common/entity/resource/Resource.scala delete mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/scala/org/apache/linkis/manager/common/utils/ResourceUtils.scala rename linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/scala/org/apache/linkis/manager/{common/protocol/ServiceState.scala => rm/ResourceInfo.scala} (82%) delete mode 100644 linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/scala/org/apache/linkis/manager/rm/ResourceManagerParam.scala diff --git a/linkis-commons/linkis-common/src/main/java/org/apache/linkis/common/exception/LinkisRetryException.java b/linkis-commons/linkis-common/src/main/java/org/apache/linkis/common/exception/LinkisRetryException.java index b9efe6cdf9..01a95db0f3 100644 --- a/linkis-commons/linkis-common/src/main/java/org/apache/linkis/common/exception/LinkisRetryException.java +++ b/linkis-commons/linkis-common/src/main/java/org/apache/linkis/common/exception/LinkisRetryException.java @@ -17,7 +17,7 @@ package org.apache.linkis.common.exception; -public class LinkisRetryException extends LinkisException { +public class LinkisRetryException extends LinkisRuntimeException { LinkisRetryException(int errCode, String desc, String ip, int port, String serviceKind) { super(errCode, desc, ip, port, serviceKind); } @@ -27,7 +27,7 @@ public LinkisRetryException(int errCode, String desc) { } @Override - ExceptionLevel getLevel() { + public ExceptionLevel getLevel() { return ExceptionLevel.RETRY; } } diff --git a/linkis-commons/linkis-common/src/main/java/org/apache/linkis/common/utils/LinkisUtils.java b/linkis-commons/linkis-common/src/main/java/org/apache/linkis/common/utils/LinkisUtils.java new file mode 100644 index 0000000000..353f80f1da --- /dev/null +++ b/linkis-commons/linkis-common/src/main/java/org/apache/linkis/common/utils/LinkisUtils.java @@ -0,0 +1,165 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.common.utils; + +import org.apache.linkis.common.exception.ErrorException; +import org.apache.linkis.common.exception.FatalException; +import org.apache.linkis.common.exception.WarnException; + +import java.util.concurrent.Callable; +import java.util.function.Function; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class LinkisUtils { + private static final Logger logger = LoggerFactory.getLogger(LinkisUtils.class); + + public static T tryCatch(Callable tryOp, Function catchOp) { + T result = null; + try { + result = tryOp.call(); + } catch (Throwable t) { + if (t instanceof FatalException) { + logger.error("Fatal error, system exit...", t); + System.exit(((FatalException) t).getErrCode()); + } else if (t instanceof VirtualMachineError) { + logger.error("Fatal error, system exit...", t); + System.exit(-1); + } else if (null != t.getCause() + && (t.getCause() instanceof FatalException + || t.getCause() instanceof VirtualMachineError)) { + logger.error("Caused by fatal error, system exit...", t); + System.exit(-1); + } else if (t instanceof Error) { + logger.error("Throw error", t); + throw (Error) t; + } else { + result = catchOp.apply(t); + } + } + return result; + } + + public static void tryFinally(Runnable tryOp, Runnable finallyOp) { + try { + tryOp.run(); + } finally { + finallyOp.run(); + } + } + + public static T tryAndWarn(Callable tryOp, Logger log) { + return tryCatch( + tryOp, + t -> { + if (t instanceof ErrorException) { + ErrorException error = (ErrorException) t; + log.error( + "Warning code（警告码）: {}, Warning message（警告信息）: {}.", + error.getErrCode(), + error.getDesc(), + error); + + } else if (t instanceof WarnException) { + WarnException warn = (WarnException) t; + log.warn( + "Warning code（警告码）: {}, Warning message（警告信息）: {}.", + warn.getErrCode(), + warn.getDesc(), + warn); + + } else { + log.warn("", t); + } + return null; + }); + } + + public static void tryAndErrorMsg(Runnable tryOp, String message, Logger log) { + try { + tryOp.run(); + } catch (WarnException t) { + WarnException warn = (WarnException) t; + log.warn( + "Warning code（警告码）: {}, Warning message（警告信息）: {}.", warn.getErrCode(), warn.getDesc()); + log.warn(message, warn); + } catch (Exception t) { + log.warn(message, t); + } + } + + public static void tryAndWarn(Runnable tryOp, Logger log) { + try { + tryOp.run(); + } catch (Throwable error) { + if (error instanceof WarnException) { + WarnException warn = (WarnException) error; + log.warn( + "Warning code（警告码）: {}, Warning message（警告信息）: {}.", + warn.getErrCode(), + warn.getDesc(), + error); + } else { + log.warn("", error); + } + } + } + + public static void tryAndWarnMsg(Runnable tryOp, String message, Logger log) { + try { + tryOp.run(); + } catch (WarnException t) { + WarnException warn = (WarnException) t; + log.warn( + "Warning code（警告码）: {}, Warning message（警告信息）: {}.", warn.getErrCode(), warn.getDesc()); + log.warn(message, warn); + } catch (Exception t) { + log.warn(message, t); + } + } + + public static T tryAndWarnMsg(Callable tryOp, String message, Logger log) { + return tryCatch( + tryOp, + t -> { + if (t instanceof ErrorException) { + ErrorException error = (ErrorException) t; + log.warn( + "Warning code（警告码）: {}, Warning message（警告信息）: {}.", + error.getErrCode(), + error.getDesc()); + log.warn(message, error); + } else if (t instanceof WarnException) { + WarnException warn = (WarnException) t; + log.warn( + "Warning code（警告码）: {}, Warning message（警告信息）: {}.", + warn.getErrCode(), + warn.getDesc()); + log.warn(message, warn); + } else { + log.warn(message, t); + } + return null; + }); + } + + public static String getJvmUser() { + return System.getProperty("user.name"); + } +} diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/java/org/apache/linkis/ecm/server/service/impl/DefaultEngineConnKillService.java b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/java/org/apache/linkis/ecm/server/service/impl/DefaultEngineConnKillService.java index 50039eefd5..2e351b00df 100644 --- a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/java/org/apache/linkis/ecm/server/service/impl/DefaultEngineConnKillService.java +++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/java/org/apache/linkis/ecm/server/service/impl/DefaultEngineConnKillService.java @@ -96,6 +96,13 @@ public EngineStopResponse dealEngineConnStop(EngineStopRequest engineStopRequest killYarnAppIdOfOneEc(engineStopRequest); } + if (AMConstant.CLUSTER_PROCESS_MARK.equals(engineStopRequest.getIdentifierType()) + && engineStopRequest.getIdentifier() != null) { + List appIds = new ArrayList<>(); + appIds.add(engineStopRequest.getIdentifier()); + GovernanceUtils.killYarnJobApp(appIds); + } + if (!response.getStopStatus()) { EngineSuicideRequest request = new EngineSuicideRequest( @@ -184,6 +191,7 @@ private String getYarnAppRegexByEngineType(String engineType) { case "sqoop": regex = EngineConnConf.SQOOP_ENGINE_CONN_YARN_APP_ID_PARSE_REGEX().getValue(); break; + case "seatunnel": case "flink": case "hive": regex = EngineConnConf.HIVE_ENGINE_CONN_YARN_APP_ID_PARSE_REGEX().getValue(); diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/AbstractEngineConnLaunchService.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/AbstractEngineConnLaunchService.scala index 49e75fb6b8..df00ed4960 100644 --- a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/AbstractEngineConnLaunchService.scala +++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/AbstractEngineConnLaunchService.scala @@ -38,6 +38,7 @@ import org.apache.linkis.manager.common.protocol.engine.{ EngineStopRequest } import org.apache.linkis.manager.engineplugin.common.launch.entity.EngineConnLaunchRequest +import org.apache.linkis.manager.label.constant.LabelValueConstant import org.apache.linkis.manager.label.utils.LabelUtil import org.apache.linkis.rpc.Sender @@ -145,11 +146,19 @@ abstract class AbstractEngineConnLaunchService extends EngineConnLaunchService w throw t } LoggerUtils.removeJobIdMDC() + val label = LabelUtil.getEngingeConnRuntimeModeLabel(request.labels) + val isYarnClusterMode: Boolean = + if (null != label && label.getModeValue.equals(LabelValueConstant.YARN_CLUSTER_VALUE)) true + else false val engineNode = new AMEngineNode() engineNode.setLabels(conn.getLabels) engineNode.setServiceInstance(conn.getServiceInstance) engineNode.setOwner(request.user) - engineNode.setMark(AMConstant.PROCESS_MARK) + if (isYarnClusterMode) { + engineNode.setMark(AMConstant.CLUSTER_PROCESS_MARK) + } else { + engineNode.setMark(AMConstant.PROCESS_MARK) + } engineNode } diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-core/src/main/scala/org/apache/linkis/engineconn/core/util/EngineConnUtils.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-core/src/main/scala/org/apache/linkis/engineconn/core/util/EngineConnUtils.scala index bbace9e66f..d2247a6d2e 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-core/src/main/scala/org/apache/linkis/engineconn/core/util/EngineConnUtils.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-core/src/main/scala/org/apache/linkis/engineconn/core/util/EngineConnUtils.scala @@ -17,10 +17,14 @@ package org.apache.linkis.engineconn.core.util -import com.google.gson.Gson +import com.google.gson.{GsonBuilder, ToNumberPolicy} object EngineConnUtils { - val GSON = new Gson() + val GSON = new GsonBuilder() + .setDateFormat("yyyy-MM-dd'T'HH:mm:ssZ") + .serializeNulls + .setObjectToNumberStrategy(ToNumberPolicy.LAZILY_PARSED_NUMBER) + .create } diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/callback/hook/CallbackEngineConnHook.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/callback/hook/CallbackEngineConnHook.scala index a5bf02f84f..dfe1137084 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/callback/hook/CallbackEngineConnHook.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/callback/hook/CallbackEngineConnHook.scala @@ -23,16 +23,19 @@ import org.apache.linkis.common.utils.{Logging, Utils} import org.apache.linkis.engineconn.acessible.executor.entity.AccessibleExecutor import org.apache.linkis.engineconn.callback.service.{ EngineConnAfterStartCallback, - EngineConnPidCallback + EngineConnIdentifierCallback } import org.apache.linkis.engineconn.common.conf.EngineConnConf import org.apache.linkis.engineconn.common.creation.EngineCreationContext import org.apache.linkis.engineconn.common.engineconn.EngineConn import org.apache.linkis.engineconn.common.hook.EngineConnHook +import org.apache.linkis.engineconn.core.EngineConnObject import org.apache.linkis.engineconn.core.executor.ExecutorManager import org.apache.linkis.engineconn.core.hook.ShutdownHook import org.apache.linkis.manager.common.entity.enumeration.NodeStatus import org.apache.linkis.manager.common.protocol.engine.EngineConnStatusCallback +import org.apache.linkis.manager.label.constant.LabelValueConstant +import org.apache.linkis.manager.label.utils.LabelUtil import org.apache.linkis.rpc.Sender import org.apache.linkis.server.conf.ServerConfiguration @@ -60,16 +63,31 @@ class CallbackEngineConnHook extends EngineConnHook with Logging { val newMap = map.++(parser.getSpringConfMap) newMap.put("spring.mvc.servlet.path", ServerConfiguration.BDP_SERVER_RESTFUL_URI.getValue) DataWorkCloudApplication.main(DWCArgumentsParser.formatSpringOptions(newMap.toMap)) + val context = EngineConnObject.getEngineCreationContext - val engineConnPidCallBack = new EngineConnPidCallback() - Utils.tryAndError(engineConnPidCallBack.callback()) + val label = LabelUtil.getEngingeConnRuntimeModeLabel(context.getLabels()) + if (null != label && label.getModeValue.equals(LabelValueConstant.YARN_CLUSTER_VALUE)) { + logger.info("cluster mode call back will be invoke in beforeExecutionExecute") + } else { + val engineConnPidCallBack = new EngineConnIdentifierCallback() + Utils.tryAndError(engineConnPidCallBack.callback()) + } logger.info("<--------------------SpringBoot App init succeed-------------------->") } override def beforeExecutionExecute( engineCreationContext: EngineCreationContext, engineConn: EngineConn - ): Unit = {} + ): Unit = { + val context = EngineConnObject.getEngineCreationContext + + val label = LabelUtil.getEngingeConnRuntimeModeLabel(context.getLabels()) + if (null != label && label.getModeValue.equals(LabelValueConstant.YARN_CLUSTER_VALUE)) { + logger.info("cluster mode call back be invoke") + val engineConnPidCallBack = new EngineConnIdentifierCallback() + Utils.tryAndError(engineConnPidCallBack.callback()) + } + } override def afterExecutionExecute( engineCreationContext: EngineCreationContext, diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/callback/service/EngineConnPidCallback.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/callback/service/EngineConnIdentifierCallback.scala similarity index 60% rename from linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/callback/service/EngineConnPidCallback.scala rename to linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/callback/service/EngineConnIdentifierCallback.scala index f0995c0b99..8b9e3ad36e 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/callback/service/EngineConnPidCallback.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/callback/service/EngineConnIdentifierCallback.scala @@ -18,18 +18,30 @@ package org.apache.linkis.engineconn.callback.service import org.apache.linkis.engineconn.core.EngineConnObject +import org.apache.linkis.engineconn.core.executor.ExecutorManager +import org.apache.linkis.engineconn.executor.entity.YarnExecutor import org.apache.linkis.governance.common.protocol.task.ResponseEngineConnPid +import org.apache.linkis.manager.label.constant.LabelValueConstant +import org.apache.linkis.manager.label.utils.LabelUtil import org.apache.linkis.rpc.Sender import java.lang.management.ManagementFactory -class EngineConnPidCallback extends AbstractEngineConnStartUpCallback { +class EngineConnIdentifierCallback extends AbstractEngineConnStartUpCallback { override def callback(): Unit = { - val pid = ManagementFactory.getRuntimeMXBean.getName.split("@")(0) + + var identifier = ManagementFactory.getRuntimeMXBean.getName.split("@")(0) val instance = Sender.getThisServiceInstance val context = EngineConnObject.getEngineCreationContext - callback(ResponseEngineConnPid(instance, pid, context.getTicketId)) + + val label = LabelUtil.getEngingeConnRuntimeModeLabel(context.getLabels()) + if (null != label && label.getModeValue.equals(LabelValueConstant.YARN_CLUSTER_VALUE)) { + identifier = ExecutorManager.getInstance.getReportExecutor match { + case cluster: YarnExecutor => cluster.getApplicationId + } + } + callback(ResponseEngineConnPid(instance, identifier, context.getTicketId)) } } diff --git a/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/scala/org/apache/linkis/manager/common/protocol/ServiceHealthReport.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/executor-core/src/main/scala/org/apache/linkis/engineconn/executor/entity/KubernetesExecutor.scala similarity index 80% rename from linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/scala/org/apache/linkis/manager/common/protocol/ServiceHealthReport.scala rename to linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/executor-core/src/main/scala/org/apache/linkis/engineconn/executor/entity/KubernetesExecutor.scala index 5b25c9dea0..2c220e1314 100644 --- a/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/scala/org/apache/linkis/manager/common/protocol/ServiceHealthReport.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/executor-core/src/main/scala/org/apache/linkis/engineconn/executor/entity/KubernetesExecutor.scala @@ -15,13 +15,14 @@ * limitations under the License. */ -package org.apache.linkis.manager.common.protocol +package org.apache.linkis.engineconn.executor.entity -abstract class ServiceHealthReport { +trait KubernetesExecutor extends Executor { - def getReportTime: Long - def setReportTime(reportTime: Long): Unit + def getKubernetesClusterID: String - def setServiceState + def getApplicationURL: String + + def getNamespace: String } diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/conf/AMConfiguration.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/conf/AMConfiguration.java new file mode 100644 index 0000000000..5164542445 --- /dev/null +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/conf/AMConfiguration.java @@ -0,0 +1,210 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.manager.am.conf; + +import org.apache.linkis.common.conf.CommonVars; +import org.apache.linkis.common.conf.TimeType; +import org.apache.linkis.common.utils.Utils; +import org.apache.linkis.manager.common.entity.enumeration.MaintainType; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; + +public class AMConfiguration { + + // The configuration key for the YARN queue name. + public static final String YARN_QUEUE_NAME_CONFIG_KEY = "wds.linkis.rm.yarnqueue"; + + // Identifier for cross-queue tasks. + public static final String CROSS_QUEUE = "crossQueue"; + + // Identifier for across-cluster tasks. + public static final String ACROSS_CLUSTER_TASK = "acrossClusterTask"; + + // Identifier for priority clusters. + public static final String PRIORITY_CLUSTER = "priorityCluster"; + + // Target identifier for distinguishing target clusters. + public static final String PRIORITY_CLUSTER_TARGET = "bdp"; + + // Origin identifier for distinguishing source clusters. + public static final String PRIORITY_CLUSTER_ORIGIN = "bdap"; + + // Configuration key for the target cluster CPU threshold. + public static final String TARGET_CPU_THRESHOLD = "targetCPUThreshold"; + + // Configuration key for the target cluster memory threshold. + public static final String TARGET_MEMORY_THRESHOLD = "targetMemoryThreshold"; + + // Configuration key for the target cluster CPU percentage threshold. + public static final String TARGET_CPU_PERCENTAGE_THRESHOLD = "targetCPUPercentageThreshold"; + + // Configuration key for the target cluster memory percentage threshold. + public static final String TARGET_MEMORY_PERCENTAGE_THRESHOLD = "targetMemoryPercentageThreshold"; + + // Configuration key for the origin cluster CPU percentage threshold. + public static final String ORIGIN_CPU_PERCENTAGE_THRESHOLD = "originCPUPercentageThreshold"; + + // Configuration key for the origin cluster memory percentage threshold. + public static final String ORIGIN_MEMORY_PERCENTAGE_THRESHOLD = "originMemoryPercentageThreshold"; + + public static final double ACROSS_CLUSTER_TOTAL_MEMORY_PERCENTAGE_THRESHOLD = + CommonVars.apply("linkis.yarn.across.cluster.memory.threshold", 0.8).getValue(); + + public static final double ACROSS_CLUSTER_TOTAL_CPU_PERCENTAGE_THRESHOLD = + CommonVars.apply("linkis.yarn.across.cluster.cpu.threshold", 0.8).getValue(); + + public static final CommonVars ECM_ADMIN_OPERATIONS = + CommonVars.apply("wds.linkis.governance.admin.operations", ""); + + public static final CommonVars ENGINE_START_MAX_TIME = + CommonVars.apply("wds.linkis.manager.am.engine.start.max.time", new TimeType("11m")); + + public static final CommonVars ENGINE_CONN_START_REST_MAX_WAIT_TIME = + CommonVars.apply("wds.linkis.manager.am.engine.rest.start.max.time", new TimeType("40s")); + + public static final CommonVars ENGINE_REUSE_MAX_TIME = + CommonVars.apply("wds.linkis.manager.am.engine.reuse.max.time", new TimeType("5m")); + + public static final Integer ENGINE_REUSE_COUNT_LIMIT = + CommonVars.apply("wds.linkis.manager.am.engine.reuse.count.limit", 2).getValue(); + + public static final CommonVars DEFAULT_NODE_OWNER = + CommonVars.apply("wds.linkis.manager.am.default.node.owner", "hadoop"); + + public static final CommonVars EM_NEW_WAIT_MILLS = + CommonVars.apply("wds.linkis.manager.am.em.new.wait.mills", 1000 * 60L); + + public static final CommonVars MULTI_USER_ENGINE_TYPES = + CommonVars.apply( + "wds.linkis.multi.user.engine.types", + "jdbc,es,presto,io_file,appconn,openlookeng,trino,jobserver,nebula,hbase,doris"); + + public static final CommonVars ALLOW_BATCH_KILL_ENGINE_TYPES = + CommonVars.apply("wds.linkis.allow.batch.kill.engine.types", "spark,hive,python"); + + public static final CommonVars UNALLOW_BATCH_KILL_ENGINE_TYPES = + CommonVars.apply("wds.linkis.allow.batch.kill.engine.types", "trino,appconn,io_file"); + public static final CommonVars MULTI_USER_ENGINE_USER = + CommonVars.apply("wds.linkis.multi.user.engine.user", getDefaultMultiEngineUser()); + + public static final CommonVars ENGINE_LOCKER_MAX_TIME = + CommonVars.apply("wds.linkis.manager.am.engine.locker.max.time", 1000 * 60 * 5); + + public static final String AM_CAN_RETRY_LOGS = + CommonVars.apply( + "wds.linkis.manager.am.can.retry.logs", "already in use;Cannot allocate memory") + .getValue(); + + public static final int ASK_ENGINE_ASYNC_MAX_THREAD_SIZE = + CommonVars.apply("wds.linkis.ecm.launch.max.thread.size", 200).getValue(); + + public static final int ASYNC_STOP_ENGINE_MAX_THREAD_SIZE = + CommonVars.apply("wds.linkis.async.stop.engine.size", 20).getValue(); + + public static final CommonVars EC_MAINTAIN_TIME_STR = + CommonVars.apply("wds.linkis.ec.maintain.time.key", MaintainType.Default.toString()); + + public static final int EC_MAINTAIN_WORK_START_TIME = + CommonVars.apply("wds.linkis.ec.maintain.time.work.start.time", 8).getValue(); + + public static final int EC_MAINTAIN_WORK_END_TIME = + CommonVars.apply("wds.linkis.ec.maintain.time.work.end.time", 19).getValue(); + + public static final Boolean NODE_SELECT_HOTSPOT_EXCLUSION_RULE = + CommonVars.apply("linkis.node.select.hotspot.exclusion.rule.enable", true).getValue(); + + public static final boolean EC_REUSE_WITH_RESOURCE_RULE_ENABLE = + CommonVars.apply("linkis.ec.reuse.with.resource.rule.enable", false).getValue(); + + public static final String EC_REUSE_WITH_RESOURCE_WITH_ECS = + CommonVars.apply("linkis.ec.reuse.with.resource.with.ecs", "spark,hive,shell,python") + .getValue(); + + public static final String SUPPORT_CLUSTER_RULE_EC_TYPES = + CommonVars.apply("linkis.support.cluster.rule.ec.types", "").getValue(); + + public static final boolean HIVE_CLUSTER_EC_EXECUTE_ONCE_RULE_ENABLE = + CommonVars.apply("linkis.hive.cluster.ec.execute.once.rule.enable", true).getValue(); + + public static final String LONG_LIVED_LABEL = + CommonVars.apply("linkis.label.node.long.lived.label.keys", "tenant|yarnCluster").getValue(); + + public static final String TMP_LIVED_LABEL = + CommonVars.apply("linkis.label.node.tmp.lived.label.keys", "taskId").getValue(); + + public static final boolean COMBINED_WITHOUT_YARN_DEFAULT = + CommonVars.apply("linkis.combined.without.yarn.default", true).getValue(); + + public static final Map AM_ENGINE_ASK_MAX_NUMBER = new HashMap<>(); + + static { + String keyValue = + CommonVars.apply("linkis.am.engine.ask.max.number", "appconn=5,trino=10").getValue(); + String[] keyValuePairs = keyValue.split(","); + for (String pair : keyValuePairs) { + String[] array = pair.split("="); + if (array.length != 2) { + throw new IllegalArgumentException( + "linkis.am.engine.ask.max.number value is illegal, value is " + pair); + } else { + AM_ENGINE_ASK_MAX_NUMBER.put(array[0], Integer.parseInt(array[1])); + } + } + } + + public static final boolean AM_ECM_RESET_RESOURCE = + CommonVars.apply("linkis.am.ecm.reset.resource.enable", true).getValue(); + + public static final boolean AM_USER_RESET_RESOURCE = + CommonVars.apply("linkis.am.user.reset.resource.enable", true).getValue(); + + public static String getDefaultMultiEngineUser() { + String jvmUser = Utils.getJvmUser(); + return String.format( + "{jdbc:\"%s\", es: \"%s\", presto:\"%s\", appconn:\"%s\", openlookeng:\"%s\", trino:\"%s\", nebula:\"%s\",doris:\"%s\", hbase:\"%s\", jobserver:\"%s\",io_file:\"root\"}", + jvmUser, jvmUser, jvmUser, jvmUser, jvmUser, jvmUser, jvmUser, jvmUser, jvmUser, jvmUser); + } + + public static boolean isMultiUserEngine(String engineType) { + String[] multiUserEngine = AMConfiguration.MULTI_USER_ENGINE_TYPES.getValue().split(","); + Optional findResult = + Arrays.stream(multiUserEngine).filter(e -> e.equalsIgnoreCase(engineType)).findFirst(); + return findResult.isPresent(); + } + + public static boolean isAllowKilledEngineType(String engineType) { + String[] allowBatchKillEngine = + AMConfiguration.ALLOW_BATCH_KILL_ENGINE_TYPES.getValue().split(","); + Optional findResult = + Arrays.stream(allowBatchKillEngine).filter(e -> e.equalsIgnoreCase(engineType)).findFirst(); + return findResult.isPresent(); + } + + public static boolean isUnAllowKilledEngineType(String engineType) { + String[] unAllowBatchKillEngine = + AMConfiguration.UNALLOW_BATCH_KILL_ENGINE_TYPES.getValue().split(","); + Optional findResult = + Arrays.stream(unAllowBatchKillEngine) + .filter(e -> e.equalsIgnoreCase(engineType)) + .findFirst(); + return findResult.isPresent(); + } +} diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/conf/ConfigurationMapCache.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/conf/ConfigurationMapCache.java new file mode 100644 index 0000000000..fa9843d955 --- /dev/null +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/conf/ConfigurationMapCache.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.manager.am.conf; + +import org.apache.linkis.common.conf.Configuration; +import org.apache.linkis.governance.common.protocol.conf.RequestQueryEngineConfigWithGlobalConfig; +import org.apache.linkis.governance.common.protocol.conf.RequestQueryGlobalConfig; +import org.apache.linkis.governance.common.protocol.conf.ResponseQueryConfig; +import org.apache.linkis.manager.label.entity.engine.EngineTypeLabel; +import org.apache.linkis.manager.label.entity.engine.UserCreatorLabel; +import org.apache.linkis.protocol.CacheableProtocol; +import org.apache.linkis.rpc.RPCMapCache; + +import java.util.Map; + +import scala.Tuple2; + +public class ConfigurationMapCache { + + public static RPCMapCache globalMapCache = + new RPCMapCache( + Configuration.CLOUD_CONSOLE_CONFIGURATION_SPRING_APPLICATION_NAME().getValue()) { + @Override + public CacheableProtocol createRequest(UserCreatorLabel userCreatorLabel) { + return new RequestQueryGlobalConfig(userCreatorLabel.getUser()); + } + + @Override + public Map createMap(Object any) { + if (any instanceof ResponseQueryConfig) { + return ((ResponseQueryConfig) any).getKeyAndValue(); + } + return null; + } + }; + + public static RPCMapCache, String, String> + engineMapCache = + new RPCMapCache, String, String>( + Configuration.CLOUD_CONSOLE_CONFIGURATION_SPRING_APPLICATION_NAME().getValue()) { + @Override + public CacheableProtocol createRequest( + Tuple2 labelTuple) { + return new RequestQueryEngineConfigWithGlobalConfig( + labelTuple._1(), labelTuple._2(), null); + } + + @Override + public Map createMap(Object any) { + if (any instanceof ResponseQueryConfig) { + return ((ResponseQueryConfig) any).getKeyAndValue(); + } + return null; + } + }; +} diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/conf/DefaultEngineConnConfigurationService.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/conf/DefaultEngineConnConfigurationService.java new file mode 100644 index 0000000000..1492c6569f --- /dev/null +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/conf/DefaultEngineConnConfigurationService.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.manager.am.conf; + +import org.apache.linkis.common.utils.LinkisUtils; +import org.apache.linkis.manager.label.entity.Label; +import org.apache.linkis.manager.label.entity.engine.EngineTypeLabel; +import org.apache.linkis.manager.label.entity.engine.UserCreatorLabel; + +import org.springframework.stereotype.Component; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import scala.Tuple2; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@Component +class DefaultEngineConnConfigurationService implements EngineConnConfigurationService { + + private static final Logger logger = + LoggerFactory.getLogger(DefaultEngineConnConfigurationService.class); + + @Override + public Map getConsoleConfiguration(List> labelList) { + Map properties = new HashMap<>(); + + Optional userCreatorLabelOption = + labelList.stream() + .filter(l -> l instanceof UserCreatorLabel) + .map(l -> (UserCreatorLabel) l) + .findFirst(); + + Optional engineTypeLabelOption = + labelList.stream() + .filter(l -> l instanceof EngineTypeLabel) + .map(l -> (EngineTypeLabel) l) + .findFirst(); + userCreatorLabelOption.ifPresent( + userCreatorLabel -> { + engineTypeLabelOption.ifPresent( + engineTypeLabel -> { + Map engineConfig = + LinkisUtils.tryAndWarn( + () -> + ConfigurationMapCache.engineMapCache.getCacheMap( + new Tuple2(userCreatorLabel, engineTypeLabel)), + logger); + if (engineConfig != null) { + properties.putAll(engineConfig); + } + }); + }); + return properties; + } +} diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/service/common/pointer/NodePointerBuilder.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/conf/EngineConnConfigurationService.java similarity index 74% rename from linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/service/common/pointer/NodePointerBuilder.scala rename to linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/conf/EngineConnConfigurationService.java index f8f478e7d1..2d2d01d057 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/service/common/pointer/NodePointerBuilder.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/conf/EngineConnConfigurationService.java @@ -15,14 +15,14 @@ * limitations under the License. */ -package org.apache.linkis.manager.service.common.pointer +package org.apache.linkis.manager.am.conf; -import org.apache.linkis.manager.common.entity.node.{EMNode, EngineNode} +import org.apache.linkis.manager.label.entity.Label; -trait NodePointerBuilder { +import java.util.List; +import java.util.Map; - def buildEMNodePointer(node: EMNode): EMNodPointer - - def buildEngineNodePointer(node: EngineNode): EngineNodePointer +public interface EngineConnConfigurationService { + Map getConsoleConfiguration(List> labelList); } diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/conf/ManagerMonitorConf.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/conf/ManagerMonitorConf.java new file mode 100644 index 0000000000..bce581a2e9 --- /dev/null +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/conf/ManagerMonitorConf.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.manager.am.conf; + +import org.apache.linkis.common.conf.CommonVars; +import org.apache.linkis.common.conf.TimeType; + +public class ManagerMonitorConf { + + public static final CommonVars NODE_MAX_CREATE_TIME = + CommonVars.apply("wds.linkis.manager.am.node.create.time", new TimeType("12m")); + + public static final CommonVars NODE_HEARTBEAT_MAX_UPDATE_TIME = + CommonVars.apply("wds.linkis.manager.am.node.heartbeat", new TimeType("12m")); + + public static final CommonVars MANAGER_MONITOR_ASYNC_POLL_SIZE = + CommonVars.apply("wds.linkis.manager.monitor.async.poll.size", 5); + + public static final CommonVars MONITOR_SWITCH_ON = + CommonVars.apply("wds.linkis.manager.am.monitor.switch.on", true); + + public static final CommonVars ECM_HEARTBEAT_MAX_UPDATE_TIME = + CommonVars.apply("wds.linkis.manager.am.ecm.heartbeat", new TimeType("5m")); + + public static final CommonVars ACROSS_QUEUES_RESOURCE_SHOW_SWITCH_ON = + CommonVars.apply("wds.linkis.manager.across.resource.show.switch.on", false); +} diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/converter/DefaultMetricsConverter.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/converter/DefaultMetricsConverter.java new file mode 100644 index 0000000000..c7620157a1 --- /dev/null +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/converter/DefaultMetricsConverter.java @@ -0,0 +1,141 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.manager.am.converter; + +import org.apache.linkis.manager.common.entity.enumeration.NodeStatus; +import org.apache.linkis.manager.common.entity.metrics.NodeHealthyInfo; +import org.apache.linkis.manager.common.entity.metrics.NodeMetrics; +import org.apache.linkis.manager.common.entity.metrics.NodeOverLoadInfo; +import org.apache.linkis.manager.common.entity.metrics.NodeTaskInfo; +import org.apache.linkis.manager.common.entity.node.AMNode; +import org.apache.linkis.server.BDPJettyServerHelper; + +import org.apache.commons.lang3.StringUtils; + +import org.springframework.stereotype.Component; + +import java.io.IOException; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@Component +public class DefaultMetricsConverter implements MetricsConverter { + + private static final Logger logger = LoggerFactory.getLogger(DefaultMetricsConverter.class); + + @Override + public NodeTaskInfo parseTaskInfo(NodeMetrics nodeMetrics) { + String msg = nodeMetrics.getHeartBeatMsg(); + if (StringUtils.isNotBlank(msg)) { + try { + JsonNode jsonNode = BDPJettyServerHelper.jacksonJson().readTree(msg); + if (jsonNode != null && jsonNode.has("taskInfo")) { + NodeTaskInfo taskInfo = + BDPJettyServerHelper.jacksonJson() + .readValue(jsonNode.get("taskInfo").asText(), NodeTaskInfo.class); + return taskInfo; + } + } catch (IOException e) { + logger.warn("parse task info failed", e); + } + } + return null; + } + + @Override + public NodeHealthyInfo parseHealthyInfo(NodeMetrics nodeMetrics) { + String healthyInfo = nodeMetrics.getHealthy(); + if (StringUtils.isNotBlank(healthyInfo)) { + try { + return BDPJettyServerHelper.jacksonJson().readValue(healthyInfo, NodeHealthyInfo.class); + } catch (IOException e) { + logger.warn("parse healthy info failed", e); + } + } + return null; + } + + @Override + public NodeOverLoadInfo parseOverLoadInfo(NodeMetrics nodeMetrics) { + String overLoad = nodeMetrics.getOverLoad(); + if (StringUtils.isNotBlank(overLoad)) { + try { + return BDPJettyServerHelper.jacksonJson().readValue(overLoad, NodeOverLoadInfo.class); + } catch (IOException e) { + logger.warn("parse over load info failed", e); + } + } + return null; + } + + @Override + public NodeStatus parseStatus(NodeMetrics nodeMetrics) { + return NodeStatus.values()[nodeMetrics.getStatus()]; + } + + @Override + public String convertTaskInfo(NodeTaskInfo nodeTaskInfo) { + try { + return BDPJettyServerHelper.jacksonJson().writeValueAsString(nodeTaskInfo); + } catch (JsonProcessingException e) { + logger.warn("convert task info failed", e); + } + return null; + } + + @Override + public String convertHealthyInfo(NodeHealthyInfo nodeHealthyInfo) { + try { + return BDPJettyServerHelper.jacksonJson().writeValueAsString(nodeHealthyInfo); + } catch (JsonProcessingException e) { + logger.warn("convert healthy info failed", e); + } + return null; + } + + @Override + public String convertOverLoadInfo(NodeOverLoadInfo nodeOverLoadInfo) { + try { + return BDPJettyServerHelper.jacksonJson().writeValueAsString(nodeOverLoadInfo); + } catch (JsonProcessingException e) { + logger.warn("convert over load info failed", e); + } + return null; + } + + @Override + public int convertStatus(NodeStatus nodeStatus) { + return nodeStatus.ordinal(); + } + + @Override + public AMNode fillMetricsToNode(AMNode amNode, NodeMetrics metrics) { + if (metrics == null) { + return amNode; + } + amNode.setNodeStatus(parseStatus(metrics)); + amNode.setNodeTaskInfo(parseTaskInfo(metrics)); + amNode.setNodeHealthyInfo(parseHealthyInfo(metrics)); + amNode.setNodeOverLoadInfo(parseOverLoadInfo(metrics)); + amNode.setUpdateTime(metrics.getUpdateTime()); + return amNode; + } +} diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/converter/MetricsConverter.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/converter/MetricsConverter.java new file mode 100644 index 0000000000..e84b577f45 --- /dev/null +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/converter/MetricsConverter.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.manager.am.converter; + +import org.apache.linkis.common.ServiceInstance; +import org.apache.linkis.manager.common.entity.enumeration.NodeHealthy; +import org.apache.linkis.manager.common.entity.enumeration.NodeStatus; +import org.apache.linkis.manager.common.entity.metrics.*; +import org.apache.linkis.manager.common.entity.node.AMNode; + +public interface MetricsConverter { + NodeTaskInfo parseTaskInfo(NodeMetrics nodeMetrics); + + NodeHealthyInfo parseHealthyInfo(NodeMetrics nodeMetrics); + + NodeOverLoadInfo parseOverLoadInfo(NodeMetrics nodeMetrics); + + NodeStatus parseStatus(NodeMetrics nodeMetrics); + + String convertTaskInfo(NodeTaskInfo nodeTaskInfo); + + String convertHealthyInfo(NodeHealthyInfo nodeHealthyInfo); + + String convertOverLoadInfo(NodeOverLoadInfo nodeOverLoadInfo); + + int convertStatus(NodeStatus nodeStatus); + + AMNode fillMetricsToNode(AMNode amNode, NodeMetrics metrics); + + default NodeMetrics getInitMetric(ServiceInstance serviceInstance) { + AMNodeMetrics nodeMetrics = new AMNodeMetrics(); + nodeMetrics.setStatus(NodeStatus.Starting.ordinal()); + + NodeHealthyInfo nodeHealthyInfo = new NodeHealthyInfo(); + nodeHealthyInfo.setNodeHealthy(NodeHealthy.Healthy); + nodeMetrics.setHealthy(convertHealthyInfo(nodeHealthyInfo)); + nodeMetrics.setServiceInstance(serviceInstance); + return nodeMetrics; + } +} diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/label/AMLabelChecker.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/label/AMLabelChecker.java new file mode 100644 index 0000000000..13e45832c8 --- /dev/null +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/label/AMLabelChecker.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.manager.am.label; + +import org.apache.linkis.manager.label.entity.Label; +import org.apache.linkis.manager.label.entity.em.EMInstanceLabel; +import org.apache.linkis.manager.label.entity.engine.EngineTypeLabel; +import org.apache.linkis.manager.label.entity.engine.UserCreatorLabel; + +import org.springframework.stereotype.Component; + +import java.util.Arrays; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; + +@Component +class AMLabelChecker implements LabelChecker { + + @Override + public boolean checkEngineLabel(List> labelList) { + return checkCorrespondingLabel(labelList, EngineTypeLabel.class, UserCreatorLabel.class); + } + + @Override + public boolean checkEMLabel(List> labelList) { + return checkCorrespondingLabel(labelList, EMInstanceLabel.class); + } + + @Override + public boolean checkCorrespondingLabel(List> labelList, Class... clazz) { + List> classes = Arrays.asList(clazz); + return labelList.stream() + .filter(Objects::nonNull) + .map(Label::getClass) + .collect(Collectors.toList()) + .containsAll(classes); + } +} diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/label/AMLabelFilter.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/label/AMLabelFilter.java new file mode 100644 index 0000000000..8820bc0be8 --- /dev/null +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/label/AMLabelFilter.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.manager.am.label; + +import org.apache.linkis.governance.common.conf.GovernanceCommonConf; +import org.apache.linkis.manager.label.entity.EMNodeLabel; +import org.apache.linkis.manager.label.entity.EngineNodeLabel; +import org.apache.linkis.manager.label.entity.Label; +import org.apache.linkis.manager.label.entity.node.AliasServiceInstanceLabel; +import org.apache.linkis.manager.service.common.label.LabelFilter; + +import org.springframework.stereotype.Component; + +import java.util.List; +import java.util.stream.Collectors; + +@Component +public class AMLabelFilter implements LabelFilter { + + @Override + public List> choseEngineLabel(List> labelList) { + return labelList.stream() + .filter( + label -> + label instanceof EngineNodeLabel + || (label instanceof AliasServiceInstanceLabel + && ((AliasServiceInstanceLabel) label) + .getAlias() + .equals(GovernanceCommonConf.ENGINE_CONN_SPRING_NAME().getValue()))) + .collect(Collectors.toList()); + } + + @Override + public List> choseEMLabel(List> labelList) { + return labelList.stream() + .filter( + label -> + label instanceof EMNodeLabel + || (label instanceof AliasServiceInstanceLabel + && ((AliasServiceInstanceLabel) label) + .getAlias() + .equals( + GovernanceCommonConf.ENGINE_CONN_MANAGER_SPRING_NAME().getValue()))) + .collect(Collectors.toList()); + } +} diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/label/DefaultManagerLabelService.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/label/DefaultManagerLabelService.java new file mode 100644 index 0000000000..50c7e1ec6e --- /dev/null +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/label/DefaultManagerLabelService.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.manager.am.label; + +import org.apache.linkis.common.ServiceInstance; +import org.apache.linkis.manager.label.entity.Label; +import org.apache.linkis.manager.label.entity.em.EMInstanceLabel; +import org.apache.linkis.manager.label.entity.engine.EngineInstanceLabel; +import org.apache.linkis.manager.label.service.NodeLabelService; +import org.apache.linkis.manager.service.common.label.ManagerLabelService; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.util.List; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@Service +public class DefaultManagerLabelService implements ManagerLabelService { + + private static final Logger logger = LoggerFactory.getLogger(DefaultManagerLabelService.class); + + @Autowired private NodeLabelService nodeLabelService; + + @Override + public boolean isEngine(ServiceInstance serviceInstance) { + List> labelList = nodeLabelService.getNodeLabels(serviceInstance); + return isEngine(labelList); + } + + @Override + public boolean isEM(ServiceInstance serviceInstance) { + List> labelList = nodeLabelService.getNodeLabels(serviceInstance); + boolean isEngine = labelList.stream().anyMatch(label -> label instanceof EngineInstanceLabel); + if (!isEngine) { + return labelList.stream().anyMatch(label -> label instanceof EMInstanceLabel); + } else { + return false; + } + } + + @Override + public boolean isEngine(List> labels) { + return labels.stream().anyMatch(label -> label instanceof EngineInstanceLabel); + } +} diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/label/EngineReuseLabelRester.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/label/EngineReuseLabelChooser.java similarity index 78% rename from linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/label/EngineReuseLabelRester.scala rename to linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/label/EngineReuseLabelChooser.java index e890fab04f..edbaf638fa 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/label/EngineReuseLabelRester.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/label/EngineReuseLabelChooser.java @@ -15,14 +15,13 @@ * limitations under the License. */ -package org.apache.linkis.manager.am.label +package org.apache.linkis.manager.am.label; -import org.apache.linkis.manager.label.entity.Label +import org.apache.linkis.manager.label.entity.Label; -import java.util +import java.util.List; -trait EngineReuseLabelChooser { - - def chooseLabels(labelList: util.List[Label[_]]): util.List[Label[_]] +public interface EngineReuseLabelChooser { + List> chooseLabels(List> labelList); } diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/label/LabelChecker.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/label/LabelChecker.java new file mode 100644 index 0000000000..0c1e27d086 --- /dev/null +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/label/LabelChecker.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.manager.am.label; + +import org.apache.linkis.manager.label.entity.Label; + +import java.util.List; + +public interface LabelChecker { + + boolean checkEngineLabel(List> labelList); + + boolean checkEMLabel(List> labelList); + + boolean checkCorrespondingLabel(List> labelList, Class... clazz); +} diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/label/MultiUserEngineReuseLabelChooser.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/label/MultiUserEngineReuseLabelChooser.java new file mode 100644 index 0000000000..b8ed766072 --- /dev/null +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/label/MultiUserEngineReuseLabelChooser.java @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.manager.am.label; + +import org.apache.linkis.manager.am.conf.AMConfiguration; +import org.apache.linkis.manager.am.exception.AMErrorCode; +import org.apache.linkis.manager.am.exception.AMErrorException; +import org.apache.linkis.manager.label.entity.Label; +import org.apache.linkis.manager.label.entity.engine.EngineTypeLabel; +import org.apache.linkis.manager.label.entity.engine.UserCreatorLabel; +import org.apache.linkis.server.BDPJettyServerHelper; + +import org.apache.commons.lang3.StringUtils; + +import org.springframework.stereotype.Component; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Stream; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@Component +public class MultiUserEngineReuseLabelChooser implements EngineReuseLabelChooser { + private static final Logger logger = + LoggerFactory.getLogger(MultiUserEngineReuseLabelChooser.class); + + private final String[] multiUserEngine = + AMConfiguration.MULTI_USER_ENGINE_TYPES.getValue().split(","); + private final Map userMap = getMultiUserEngineUserMap(); + + private Map getMultiUserEngineUserMap() { + String userJson = AMConfiguration.MULTI_USER_ENGINE_USER.getValue(); + if (StringUtils.isNotBlank(userJson)) { + Map userMap = BDPJettyServerHelper.gson().fromJson(userJson, Map.class); + return userMap; + } else { + throw new AMErrorException( + AMErrorCode.AM_CONF_ERROR.getErrorCode(), + String.format( + "Multi-user engine parameter configuration error, please check key %s", + AMConfiguration.MULTI_USER_ENGINE_USER.key())); + } + } + + /** + * Filter out UserCreator Label that supports multi-user engine + * + * @param labelList + * @return + */ + @Override + public List> chooseLabels(List> labelList) { + List> labels = new ArrayList<>(labelList); + Optional engineTypeLabelOption = + labels.stream() + .filter(label -> label instanceof EngineTypeLabel) + .map(label -> (EngineTypeLabel) label) + .findFirst(); + if (engineTypeLabelOption.isPresent()) { + EngineTypeLabel engineTypeLabel = engineTypeLabelOption.get(); + Optional maybeString = + Stream.of(multiUserEngine) + .filter(engineTypeLabel.getEngineType()::equalsIgnoreCase) + .findFirst(); + Optional userCreatorLabelOption = + labels.stream() + .filter(label -> label instanceof UserCreatorLabel) + .map(label -> (UserCreatorLabel) label) + .findFirst(); + if (maybeString.isPresent() && userCreatorLabelOption.isPresent()) { + String userAdmin = userMap.get(engineTypeLabel.getEngineType()); + UserCreatorLabel userCreatorLabel = userCreatorLabelOption.get(); + logger.info( + String.format( + "For multi user engine to reset userCreatorLabel user %s to Admin %s", + userCreatorLabel.getUser(), userAdmin)); + userCreatorLabel.setUser(userAdmin); + return labels; + } + } + return labelList; + } +} diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/locker/DefaultEngineNodeLocker.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/locker/DefaultEngineNodeLocker.java new file mode 100644 index 0000000000..fe93a03411 --- /dev/null +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/locker/DefaultEngineNodeLocker.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.manager.am.locker; + +import org.apache.linkis.manager.am.pointer.NodePointerBuilder; +import org.apache.linkis.manager.common.entity.node.AMEngineNode; +import org.apache.linkis.manager.common.entity.node.EngineNode; +import org.apache.linkis.manager.common.protocol.RequestEngineLock; +import org.apache.linkis.manager.common.protocol.RequestEngineUnlock; +import org.apache.linkis.manager.common.protocol.RequestManagerUnlock; +import org.apache.linkis.manager.common.protocol.engine.EngineLockType; +import org.apache.linkis.rpc.message.annotation.Receiver; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import java.util.Optional; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@Component +public class DefaultEngineNodeLocker implements EngineNodeLocker { + private static final Logger logger = LoggerFactory.getLogger(DefaultEngineNodeLocker.class); + + @Autowired private NodePointerBuilder nodeBuilder; + + @Override + public Optional lockEngine(EngineNode engineNode, long timeout) { + return nodeBuilder + .buildEngineNodePointer(engineNode) + .lockEngine(new RequestEngineLock(timeout, EngineLockType.Timed)); + } + + @Override + public void releaseLock(EngineNode engineNode, String lock) { + nodeBuilder.buildEngineNodePointer(engineNode).releaseLock(new RequestEngineUnlock(lock)); + } + + @Receiver + public void releaseLock(RequestManagerUnlock requestManagerUnlock) { + try { + logger.info( + String.format( + "client%s Start to unlock engine %s", + requestManagerUnlock.clientInstance(), requestManagerUnlock.engineInstance())); + AMEngineNode engineNode = new AMEngineNode(); + engineNode.setServiceInstance(requestManagerUnlock.engineInstance()); + releaseLock(engineNode, requestManagerUnlock.lock()); + logger.info( + String.format( + "client%s Finished to unlock engine %s", + requestManagerUnlock.clientInstance(), requestManagerUnlock.engineInstance())); + } catch (Exception e) { + logger.error("release lock failed", e); + } + } +} diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/locker/EngineNodeLocker.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/locker/EngineNodeLocker.java new file mode 100644 index 0000000000..171714b333 --- /dev/null +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/locker/EngineNodeLocker.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.manager.am.locker; + +import org.apache.linkis.manager.common.entity.node.EngineNode; + +import java.util.Optional; + +public interface EngineNodeLocker { + + Optional lockEngine(EngineNode engineNode, long timeout); + + void releaseLock(EngineNode engineNode, String lock); +} diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/manager/DefaultEMNodeManager.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/manager/DefaultEMNodeManager.java new file mode 100644 index 0000000000..691aa635a4 --- /dev/null +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/manager/DefaultEMNodeManager.java @@ -0,0 +1,218 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.manager.am.manager; + +import org.apache.linkis.common.ServiceInstance; +import org.apache.linkis.manager.am.converter.MetricsConverter; +import org.apache.linkis.manager.am.pointer.NodePointerBuilder; +import org.apache.linkis.manager.common.entity.metrics.NodeMetrics; +import org.apache.linkis.manager.common.entity.node.*; +import org.apache.linkis.manager.common.entity.persistence.PersistenceNodeEntity; +import org.apache.linkis.manager.common.protocol.em.ECMOperateRequest; +import org.apache.linkis.manager.common.protocol.em.ECMOperateResponse; +import org.apache.linkis.manager.common.protocol.engine.EngineStopRequest; +import org.apache.linkis.manager.engineplugin.common.launch.entity.EngineConnLaunchRequest; +import org.apache.linkis.manager.exception.NodeInstanceDuplicateException; +import org.apache.linkis.manager.persistence.NodeManagerPersistence; +import org.apache.linkis.manager.persistence.NodeMetricManagerPersistence; +import org.apache.linkis.manager.rm.ResourceInfo; +import org.apache.linkis.manager.rm.service.ResourceManager; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import java.util.*; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@Component +public class DefaultEMNodeManager implements EMNodeManager { + private static final Logger logger = LoggerFactory.getLogger(DefaultEMNodeManager.class); + + @Autowired private NodeManagerPersistence nodeManagerPersistence; + + @Autowired private NodeMetricManagerPersistence nodeMetricManagerPersistence; + + @Autowired private MetricsConverter metricsConverter; + + @Autowired private NodePointerBuilder nodePointerBuilder; + + @Autowired private ResourceManager resourceManager; + + @Override + public void emRegister(EMNode emNode) { + nodeManagerPersistence.addNodeInstance(emNode); + // init metric + nodeMetricManagerPersistence.addOrupdateNodeMetrics( + metricsConverter.getInitMetric(emNode.getServiceInstance())); + } + + @Override + public void addEMNodeInstance(EMNode emNode) { + try { + nodeManagerPersistence.addNodeInstance(emNode); + } catch (NodeInstanceDuplicateException e) { + logger.warn("em instance had exists, {}.", emNode); + nodeManagerPersistence.updateEngineNode(emNode.getServiceInstance(), emNode); + } + } + + @Override + public void initEMNodeMetrics(EMNode emNode) { + nodeMetricManagerPersistence.addOrupdateNodeMetrics( + metricsConverter.getInitMetric(emNode.getServiceInstance())); + } + + @Override + public List listEngines(EMNode emNode) { + List result = new ArrayList<>(); + List nodes = nodeManagerPersistence.getEngineNodeByEM(emNode.getServiceInstance()); + Map metricses = new HashMap<>(); + for (NodeMetrics m : nodeMetricManagerPersistence.getNodeMetrics(nodes)) { + metricses.put(m.getServiceInstance().toString(), m); + } + for (EngineNode node : nodes) { + NodeMetrics metrics = metricses.get(node.getServiceInstance().toString()); + if (metrics != null) { + metricsConverter.fillMetricsToNode(node, metrics); + } + result.add(node); + } + return result; + } + + @Override + public List listUserEngines(EMNode emNode, String user) { + List result = new ArrayList<>(); + for (EngineNode node : listEngines(emNode)) { + if (node.getOwner().equals(user)) { + result.add(node); + } + } + return result; + } + + @Override + public List listUserNodes(String user) { + return nodeManagerPersistence.getNodes(user); + } + + /** + * Get detailed em information from the persistence TODO add label to node ? + * + * @param scoreServiceInstances + * @return + */ + @Override + public EMNode[] getEMNodes(ScoreServiceInstance[] scoreServiceInstances) { + if (null == scoreServiceInstances || scoreServiceInstances.length == 0) { + return null; + } + EMNode[] emNodes = + Arrays.stream(scoreServiceInstances) + .map( + scoreServiceInstance -> { + AMEMNode emNode = new AMEMNode(); + emNode.setScore(scoreServiceInstance.getScore()); + emNode.setServiceInstance(scoreServiceInstance.getServiceInstance()); + return emNode; + }) + .toArray(EMNode[]::new); + + // 1. add nodeMetrics 2 add RM info + ResourceInfo resourceInfo = + resourceManager.getResourceInfo( + Arrays.stream(scoreServiceInstances) + .map(ScoreServiceInstance::getServiceInstance) + .toArray(ServiceInstance[]::new)); + List nodeMetrics = + nodeMetricManagerPersistence.getNodeMetrics(Arrays.asList(emNodes)); + + for (EMNode emNode : emNodes) { + Optional optionMetrics = + nodeMetrics.stream() + .filter(metrics -> metrics.getServiceInstance().equals(emNode.getServiceInstance())) + .findFirst(); + Optional optionRMNode = + resourceInfo.resourceInfo().stream() + .filter(rmNode -> rmNode.getServiceInstance().equals(emNode.getServiceInstance())) + .findFirst(); + + optionMetrics.ifPresent(metrics -> metricsConverter.fillMetricsToNode(emNode, metrics)); + optionRMNode.ifPresent(rmNode -> emNode.setNodeResource(rmNode.getNodeResource())); + } + return emNodes; + } + + @Override + public EMNode getEM(ServiceInstance serviceInstance) { + Node node = nodeManagerPersistence.getNode(serviceInstance); + if (null == node) { + logger.info("This em of " + serviceInstance + " not exists in db"); + return null; + } + AMEMNode emNode = new AMEMNode(); + emNode.setOwner(node.getOwner()); + emNode.setServiceInstance(node.getServiceInstance()); + if (node instanceof PersistenceNodeEntity) { + emNode.setStartTime(node.getStartTime()); + } + emNode.setMark(emNode.getMark()); + metricsConverter.fillMetricsToNode(emNode, nodeMetricManagerPersistence.getNodeMetrics(emNode)); + return emNode; + } + + @Override + public void stopEM(EMNode emNode) { + nodePointerBuilder.buildEMNodePointer(emNode).stopNode(); + } + + @Override + public void deleteEM(EMNode emNode) { + nodeManagerPersistence.removeNodeInstance(emNode); + logger.info("Finished to clear emNode instance(" + emNode.getServiceInstance() + ") info "); + nodeMetricManagerPersistence.deleteNodeMetrics(emNode); + logger.info("Finished to clear emNode(" + emNode.getServiceInstance() + ") metrics info"); + } + + @Override + public void pauseEM(ServiceInstance serviceInstance) {} + + /** + * 1. request engineManager to launch engine + * + * @param engineConnLaunchRequest + * @param emNode + * @return + */ + @Override + public EngineNode createEngine(EngineConnLaunchRequest engineConnLaunchRequest, EMNode emNode) { + return nodePointerBuilder.buildEMNodePointer(emNode).createEngine(engineConnLaunchRequest); + } + + @Override + public void stopEngine(EngineStopRequest engineStopRequest, EMNode emNode) { + nodePointerBuilder.buildEMNodePointer(emNode).stopEngine(engineStopRequest); + } + + @Override + public ECMOperateResponse executeOperation(EMNode ecmNode, ECMOperateRequest request) { + return nodePointerBuilder.buildEMNodePointer(ecmNode).executeOperation(request); + } +} diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/manager/DefaultEngineNodeManager.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/manager/DefaultEngineNodeManager.java new file mode 100644 index 0000000000..02b143d5cd --- /dev/null +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/manager/DefaultEngineNodeManager.java @@ -0,0 +1,368 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.manager.am.manager; + +import org.apache.linkis.common.ServiceInstance; +import org.apache.linkis.common.exception.LinkisRetryException; +import org.apache.linkis.manager.am.conf.AMConfiguration; +import org.apache.linkis.manager.am.converter.MetricsConverter; +import org.apache.linkis.manager.am.exception.AMErrorCode; +import org.apache.linkis.manager.am.exception.AMErrorException; +import org.apache.linkis.manager.am.locker.EngineNodeLocker; +import org.apache.linkis.manager.am.pointer.EngineNodePointer; +import org.apache.linkis.manager.am.pointer.NodePointerBuilder; +import org.apache.linkis.manager.common.constant.AMConstant; +import org.apache.linkis.manager.common.entity.enumeration.NodeStatus; +import org.apache.linkis.manager.common.entity.metrics.NodeMetrics; +import org.apache.linkis.manager.common.entity.node.*; +import org.apache.linkis.manager.common.entity.persistence.PersistenceLabel; +import org.apache.linkis.manager.common.protocol.engine.EngineOperateRequest; +import org.apache.linkis.manager.common.protocol.engine.EngineOperateResponse; +import org.apache.linkis.manager.common.protocol.node.NodeHeartbeatMsg; +import org.apache.linkis.manager.label.builder.factory.LabelBuilderFactory; +import org.apache.linkis.manager.label.builder.factory.LabelBuilderFactoryContext; +import org.apache.linkis.manager.label.entity.engine.EngineInstanceLabel; +import org.apache.linkis.manager.persistence.LabelManagerPersistence; +import org.apache.linkis.manager.persistence.NodeManagerPersistence; +import org.apache.linkis.manager.persistence.NodeMetricManagerPersistence; +import org.apache.linkis.manager.rm.ResourceInfo; +import org.apache.linkis.manager.rm.service.ResourceManager; + +import org.apache.commons.lang3.StringUtils; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.retry.annotation.Backoff; +import org.springframework.retry.annotation.Retryable; +import org.springframework.stereotype.Service; + +import java.lang.reflect.UndeclaredThrowableException; +import java.util.*; +import java.util.stream.Collectors; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@Service +public class DefaultEngineNodeManager implements EngineNodeManager { + private static final Logger logger = LoggerFactory.getLogger(DefaultEngineNodeManager.class); + + @Autowired private EngineNodeLocker engineLocker; + + @Autowired private NodeManagerPersistence nodeManagerPersistence; + + @Autowired private NodeMetricManagerPersistence nodeMetricManagerPersistence; + + @Autowired private MetricsConverter metricsConverter; + + @Autowired private NodePointerBuilder nodePointerBuilder; + + @Autowired private ResourceManager resourceManager; + + @Autowired private LabelManagerPersistence labelManagerPersistence; + + private final LabelBuilderFactory labelBuilderFactory = + LabelBuilderFactoryContext.getLabelBuilderFactory(); + + @Override + public List listEngines(String user) { + List userNodes = nodeManagerPersistence.getNodes(user); + + List nodes = + userNodes.stream() + .map(Node::getServiceInstance) + .map(nodeManagerPersistence::getEngineNode) + .collect(Collectors.toList()); + + List nodeMetrics = nodeMetricManagerPersistence.getNodeMetrics(nodes); + Map metricses = + nodeMetrics.stream() + .collect( + Collectors.toMap( + m -> m.getServiceInstance().toString(), + m -> m, + (existingValue, newValue) -> newValue)); + + nodes.forEach( + node -> { + Optional nodeMetricsOptional = + Optional.ofNullable(metricses.get(node.getServiceInstance().toString())); + nodeMetricsOptional.ifPresent(m -> metricsConverter.fillMetricsToNode(node, m)); + }); + return nodes; + } + + @Retryable( + value = {feign.RetryableException.class, UndeclaredThrowableException.class}, + maxAttempts = 5, + backoff = @Backoff(delay = 10000)) + @Override + public EngineNode getEngineNodeInfo(EngineNode engineNode) { + EngineNodePointer engine = nodePointerBuilder.buildEngineNodePointer(engineNode); + NodeHeartbeatMsg heartMsg = engine.getNodeHeartbeatMsg(); + engineNode.setNodeHealthyInfo(heartMsg.getHealthyInfo()); + engineNode.setNodeOverLoadInfo(heartMsg.getOverLoadInfo()); + engineNode.setNodeStatus(heartMsg.getStatus()); + return engineNode; + } + + @Override + public EngineNode getEngineNodeInfoByDB(EngineNode engineNode) { + EngineNode dbEngineNode = nodeManagerPersistence.getEngineNode(engineNode.getServiceInstance()); + if (null == dbEngineNode) { + throw new LinkisRetryException( + AMConstant.ENGINE_ERROR_CODE, engineNode + " not exists in db"); + } + metricsConverter.fillMetricsToNode( + dbEngineNode, nodeMetricManagerPersistence.getNodeMetrics(dbEngineNode)); + return dbEngineNode; + } + + @Override + public EngineNode getEngineNodeInfoByTicketId(String ticketId) { + EngineNode dbEngineNode = nodeManagerPersistence.getEngineNodeByTicketId(ticketId); + if (null == dbEngineNode) { + throw new LinkisRetryException(AMConstant.ENGINE_ERROR_CODE, ticketId + " not exists in db"); + } + metricsConverter.fillMetricsToNode( + dbEngineNode, nodeMetricManagerPersistence.getNodeMetrics(dbEngineNode)); + return dbEngineNode; + } + + @Override + public void updateEngineStatus( + ServiceInstance serviceInstance, NodeStatus fromState, NodeStatus toState) {} + + @Override + public void updateEngine(EngineNode engineNode) { + nodeManagerPersistence.updateNodeInstance(engineNode); + } + + @Override + public EngineNode switchEngine(EngineNode engineNode) { + return null; + } + + @Override + public EngineNode reuseEngine(EngineNode engineNode) { + EngineNode node = getEngineNodeInfo(engineNode); + if (node == null || !NodeStatus.isAvailable(node.getNodeStatus())) { + return null; + } + if (!NodeStatus.isLocked(node.getNodeStatus())) { + Optional lockStr = + engineLocker.lockEngine(node, (long) AMConfiguration.ENGINE_LOCKER_MAX_TIME.getValue()); + if (!lockStr.isPresent()) { + throw new LinkisRetryException( + AMConstant.ENGINE_ERROR_CODE, + String.format( + "Failed to request lock from engine by reuse %s", node.getServiceInstance())); + } + node.setLock(lockStr.get()); + return node; + } else { + return null; + } + } + + /** + * TODO use Engine需要考虑流式引擎的场景，后续需要通过Label加额外的处理 + * + * @param engineNode + * @param timeout + * @return + */ + @Override + public EngineNode useEngine(EngineNode engineNode, long timeout) { + // wait until engine to be available + EngineNode node = getEngineNodeInfo(engineNode); + if (node == null || !NodeStatus.isAvailable(node.getNodeStatus())) { + return null; + } + if (!NodeStatus.isLocked(node.getNodeStatus())) { + Optional lockStr = engineLocker.lockEngine(node, timeout); + if (!lockStr.isPresent()) { + throw new LinkisRetryException( + AMConstant.ENGINE_ERROR_CODE, + String.format("Failed to request lock from engine %s", node.getServiceInstance())); + } + node.setLock(lockStr.get()); + return node; + } else { + return null; + } + } + + @Override + public EngineNode useEngine(EngineNode engineNode) { + return useEngine(engineNode, AMConfiguration.ENGINE_LOCKER_MAX_TIME.getValue()); + } + + /** + * Get detailed engine information from the persistence + * + * @param scoreServiceInstances + * @return + */ + @Override + public EngineNode[] getEngineNodes(ScoreServiceInstance[] scoreServiceInstances) { + if (scoreServiceInstances == null || scoreServiceInstances.length == 0) { + return null; + } + List scoreServiceInstancesList = Arrays.asList(scoreServiceInstances); + EngineNode[] engineNodes = + scoreServiceInstancesList.stream() + .map( + scoreServiceInstance -> { + AMEngineNode engineNode = new AMEngineNode(); + engineNode.setScore(scoreServiceInstance.getScore()); + engineNode.setServiceInstance(scoreServiceInstance.getServiceInstance()); + return engineNode; + }) + .toArray(EngineNode[]::new); + + List serviceInstancesList = + scoreServiceInstancesList.stream() + .map(ScoreServiceInstance::getServiceInstance) + .collect(Collectors.toList()); + + try { + ResourceInfo resourceInfo = + resourceManager.getResourceInfo(serviceInstancesList.toArray(new ServiceInstance[0])); + + if (serviceInstancesList.isEmpty()) { + throw new LinkisRetryException( + AMConstant.ENGINE_ERROR_CODE, "Service instances cannot be empty."); + } + + List nodeMetrics = + nodeMetricManagerPersistence.getNodeMetrics(Arrays.asList(engineNodes)); + + for (EngineNode engineNode : engineNodes) { + Optional optionMetrics = + nodeMetrics.stream() + .filter( + nodeMetric -> + nodeMetric.getServiceInstance().equals(engineNode.getServiceInstance())) + .findFirst(); + + Optional optionRMNode = + resourceInfo.resourceInfo().stream() + .filter( + resourceNode -> + resourceNode.getServiceInstance().equals(engineNode.getServiceInstance())) + .findFirst(); + + optionMetrics.ifPresent(metrics -> metricsConverter.fillMetricsToNode(engineNode, metrics)); + optionRMNode.ifPresent(rmNode -> engineNode.setNodeResource(rmNode.getNodeResource())); + } + } catch (Exception e) { + LinkisRetryException linkisRetryException = + new LinkisRetryException(AMConstant.ENGINE_ERROR_CODE, "Failed to process data."); + linkisRetryException.initCause(e); + throw linkisRetryException; + } + return engineNodes; + } + + /** + * add info to persistence + * + * @param engineNode + */ + @Override + public void addEngineNode(EngineNode engineNode) { + nodeManagerPersistence.addEngineNode(engineNode); + // init metric + nodeMetricManagerPersistence.addOrupdateNodeMetrics( + metricsConverter.getInitMetric(engineNode.getServiceInstance())); + } + + /** + * delete info to persistence + * + * @param engineNode + */ + @Override + public void deleteEngineNode(EngineNode engineNode) { + nodeManagerPersistence.deleteEngineNode(engineNode); + } + + @Override + public EngineNode getEngineNode(ServiceInstance serviceInstance) { + return nodeManagerPersistence.getEngineNode(serviceInstance); + } + + /** + * 1.serviceInstance中取出instance（实际是ticketId） 2.update serviceInstance 表，包括 + * instance替换，替换mark，owner，updator，creator的空值，更新updateTime 3.update engine_em关联表 4.update label + * ticket_id ==> instance + * + * @param serviceInstance + * @param engineNode + */ + @Override + public void updateEngineNode(ServiceInstance serviceInstance, EngineNode engineNode) { + nodeManagerPersistence.updateEngineNode(serviceInstance, engineNode); + nodeMetricManagerPersistence.deleteNodeMetrics(engineNode); + + EngineInstanceLabel engineLabel = labelBuilderFactory.createLabel(EngineInstanceLabel.class); + engineLabel.setInstance(engineNode.getServiceInstance().getInstance()); + engineLabel.setServiceName(engineNode.getServiceInstance().getApplicationName()); + + EngineInstanceLabel oldEngineLabel = labelBuilderFactory.createLabel(EngineInstanceLabel.class); + oldEngineLabel.setInstance(serviceInstance.getInstance()); + oldEngineLabel.setServiceName(engineNode.getServiceInstance().getApplicationName()); + PersistenceLabel oldPersistenceLabel = + labelBuilderFactory.convertLabel(oldEngineLabel, PersistenceLabel.class); + PersistenceLabel label = + labelManagerPersistence.getLabelByKeyValue( + oldPersistenceLabel.getLabelKey(), oldPersistenceLabel.getStringValue()); + + PersistenceLabel persistenceLabel = + labelBuilderFactory.convertLabel(engineLabel, PersistenceLabel.class); + persistenceLabel.setLabelValueSize(persistenceLabel.getValue().size()); + labelManagerPersistence.updateLabel(label.getId(), persistenceLabel); + } + + public EngineOperateResponse executeOperation( + EngineNode engineNode, EngineOperateRequest request) { + EngineNodePointer engine = nodePointerBuilder.buildEngineNodePointer(engineNode); + return engine.executeOperation(request); + } + + public EngineNode getEngineNodeInfo(ServiceInstance serviceInstance) { + EngineNode engineNode = getEngineNode(serviceInstance); + if (Objects.isNull(engineNode)) { + throw new AMErrorException( + AMErrorCode.NOT_EXISTS_ENGINE_CONN.getErrorCode(), + AMErrorCode.NOT_EXISTS_ENGINE_CONN.getErrorDesc()); + } + NodeMetrics nodeMetric = nodeMetricManagerPersistence.getNodeMetrics(engineNode); + if (engineNode.getNodeStatus() == null) { + if (null != nodeMetric && null != nodeMetric.getStatus()) { + engineNode.setNodeStatus(NodeStatus.values()[nodeMetric.getStatus()]); + } else { + engineNode.setNodeStatus(NodeStatus.Starting); + } + if (null != nodeMetric && StringUtils.isNotBlank(nodeMetric.getHeartBeatMsg())) { + engineNode.setEcMetrics(nodeMetric.getHeartBeatMsg()); + } + } + return engineNode; + } +} diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/manager/EMNodeManager.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/manager/EMNodeManager.java new file mode 100644 index 0000000000..cf540036a3 --- /dev/null +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/manager/EMNodeManager.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.manager.am.manager; + +import org.apache.linkis.common.ServiceInstance; +import org.apache.linkis.manager.common.entity.node.EMNode; +import org.apache.linkis.manager.common.entity.node.EngineNode; +import org.apache.linkis.manager.common.entity.node.Node; +import org.apache.linkis.manager.common.entity.node.ScoreServiceInstance; +import org.apache.linkis.manager.common.protocol.em.ECMOperateRequest; +import org.apache.linkis.manager.common.protocol.em.ECMOperateResponse; +import org.apache.linkis.manager.common.protocol.engine.EngineStopRequest; +import org.apache.linkis.manager.engineplugin.common.launch.entity.EngineConnLaunchRequest; + +import java.util.List; + +public interface EMNodeManager { + + void emRegister(EMNode emNode); + + List listEngines(EMNode emNode); + + List listUserEngines(EMNode emNode, String user); + + List listUserNodes(String user); + + /** + * Get detailed em information from the persistence + * + * @param scoreServiceInstances + * @return + */ + EMNode[] getEMNodes(ScoreServiceInstance[] scoreServiceInstances); + + EMNode getEM(ServiceInstance serviceInstance); + + void stopEM(EMNode emNode); + + void deleteEM(EMNode emNode); + + void pauseEM(ServiceInstance serviceInstance); + + /** + * 1. request engineManager to launch engine 2. persist engine info + * + * @param engineConnLaunchRequest engine launch request + * @param emNode ecm node + * @return engine node + */ + EngineNode createEngine(EngineConnLaunchRequest engineConnLaunchRequest, EMNode emNode); + + void stopEngine(EngineStopRequest engineStopRequest, EMNode emNode); + + void addEMNodeInstance(EMNode emNode); + + void initEMNodeMetrics(EMNode emNode); + + ECMOperateResponse executeOperation(EMNode ecmNode, ECMOperateRequest request); +} diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/manager/EngineNodeManager.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/manager/EngineNodeManager.java new file mode 100644 index 0000000000..7c3f64efee --- /dev/null +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/manager/EngineNodeManager.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.manager.am.manager; + +import org.apache.linkis.common.ServiceInstance; +import org.apache.linkis.manager.common.entity.enumeration.NodeStatus; +import org.apache.linkis.manager.common.entity.node.EngineNode; +import org.apache.linkis.manager.common.entity.node.ScoreServiceInstance; +import org.apache.linkis.manager.common.protocol.engine.EngineOperateRequest; +import org.apache.linkis.manager.common.protocol.engine.EngineOperateResponse; + +import java.util.List; + +public interface EngineNodeManager { + + List listEngines(String user); + + EngineNode getEngineNode(ServiceInstance serviceInstance); + + EngineNode getEngineNodeInfo(EngineNode engineNode); + + EngineNode getEngineNodeInfo(ServiceInstance serviceInstance); + + EngineNode getEngineNodeInfoByDB(EngineNode engineNode); + + EngineNode getEngineNodeInfoByTicketId(String ticketId); + + /** + * Get detailed engine information from the persistence + * + * @param scoreServiceInstances + * @return + */ + EngineNode[] getEngineNodes(ScoreServiceInstance[] scoreServiceInstances); + + void updateEngineStatus( + ServiceInstance serviceInstance, NodeStatus fromState, NodeStatus toState); + + void addEngineNode(EngineNode engineNode); + + void updateEngineNode(ServiceInstance serviceInstance, EngineNode engineNode); + + void updateEngine(EngineNode engineNode); + + void deleteEngineNode(EngineNode engineNode); + + EngineNode switchEngine(EngineNode engineNode); + + EngineNode reuseEngine(EngineNode engineNode); + + EngineNode useEngine(EngineNode engineNode, long timeout); + + EngineNode useEngine(EngineNode engineNode); + + EngineOperateResponse executeOperation(EngineNode engineNode, EngineOperateRequest request); +} diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/pointer/AbstractNodePointer.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/pointer/AbstractNodePointer.java new file mode 100644 index 0000000000..fc0a1fca24 --- /dev/null +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/pointer/AbstractNodePointer.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.manager.am.pointer; + +import org.apache.linkis.manager.common.entity.enumeration.NodeStatus; +import org.apache.linkis.manager.common.entity.node.Node; +import org.apache.linkis.manager.common.protocol.node.*; +import org.apache.linkis.manager.label.entity.Label; +import org.apache.linkis.rpc.Sender; + +public abstract class AbstractNodePointer implements NodePointer { + + protected Sender getSender() { + return Sender.getSender(getNode().getServiceInstance()); + } + + /** + * 向对应的Node发送请求获取节点状态 + * + * @return + */ + @Override + public NodeStatus getNodeStatus() { + Sender sender = getSender(); + ResponseNodeStatus responseStatus = (ResponseNodeStatus) sender.ask(new RequestNodeStatus()); + return responseStatus.getNodeStatus(); + } + + /** + * 向对应的Node发送请求获取节点心跳信息 + * + * @return + */ + @Override + public NodeHeartbeatMsg getNodeHeartbeatMsg() { + Sender sender = getSender(); + NodeHeartbeatMsg heartbeatMsg = (NodeHeartbeatMsg) sender.ask(new NodeHeartbeatRequest()); + return heartbeatMsg; + } + + /** + * 向对应的Node发送Kill 请求 + * + * @return + */ + @Override + public void stopNode() { + Sender sender = getSender(); + sender.send(new StopNodeRequest()); + } + + /** + * 向对应的Node Label 更新请求 + * + * @return + */ + @Override + public void updateLabels(Label[] labels) {} + + @Override + public void updateNodeHealthyRequest(NodeHealthyRequest nodeHealthyRequest) { + getSender().send(nodeHealthyRequest); + } + + @Override + public boolean equals(Object obj) { + if (obj instanceof Node) { + Node nodeB = (Node) obj; + return getNode().getServiceInstance().equals(nodeB.getServiceInstance()); + } + return false; + } +} diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/pointer/DefaultEMNodPointer.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/pointer/DefaultEMNodPointer.java new file mode 100644 index 0000000000..07097fcb0b --- /dev/null +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/pointer/DefaultEMNodPointer.java @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.manager.am.pointer; + +import org.apache.linkis.common.exception.LinkisRetryException; +import org.apache.linkis.manager.am.exception.AMErrorException; +import org.apache.linkis.manager.common.constant.AMConstant; +import org.apache.linkis.manager.common.entity.node.EngineNode; +import org.apache.linkis.manager.common.entity.node.Node; +import org.apache.linkis.manager.common.protocol.em.ECMOperateRequest; +import org.apache.linkis.manager.common.protocol.em.ECMOperateResponse; +import org.apache.linkis.manager.common.protocol.engine.EngineStopRequest; +import org.apache.linkis.manager.common.protocol.engine.EngineStopResponse; +import org.apache.linkis.manager.engineplugin.common.launch.entity.EngineConnLaunchRequest; +import org.apache.linkis.server.BDPJettyServerHelper; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class DefaultEMNodPointer extends AbstractNodePointer implements EMNodPointer { + private static final Logger logger = LoggerFactory.getLogger(DefaultEMNodPointer.class); + + private Node node; + + public DefaultEMNodPointer(Node node) { + this.node = node; + } + + /** + * 与该远程指针关联的node信息 + * + * @return + */ + @Override + public Node getNode() { + return node; + } + + @Override + public EngineNode createEngine(EngineConnLaunchRequest engineConnLaunchRequest) { + logger.info("Start to createEngine ask em " + getNode().getServiceInstance()); + Object result = getSender().ask(engineConnLaunchRequest); + if (result instanceof EngineNode) { + EngineNode engineNode = (EngineNode) result; + logger.info( + "Succeed to createEngine ask em " + + getNode().getServiceInstance() + + ", engineNode " + + engineNode); + return engineNode; + } else { + throw new LinkisRetryException( + AMConstant.ENGINE_ERROR_CODE, + "Failed to createEngine ask em " + getNode().getServiceInstance() + "result: " + result); + } + } + + @Override + public void stopEngine(EngineStopRequest engineStopRequest) { + try { + Object result = getSender().ask(engineStopRequest); + if (result instanceof EngineStopResponse) { + EngineStopResponse engineStopResponse = (EngineStopResponse) result; + if (!engineStopResponse.getStopStatus()) { + logger.info( + "Kill engine : " + + engineStopRequest.getServiceInstance().toString() + + " failed, because " + + engineStopResponse.getMsg() + + " . Will ask engine to suicide."); + } else { + logger.info( + "Succeed to kill engine " + engineStopRequest.getServiceInstance().toString() + "."); + } + } else { + logger.warn( + "Ask em : " + + getNode().getServiceInstance().toString() + + " to kill engine : " + + engineStopRequest.getServiceInstance().toString() + + " failed, response is : " + + BDPJettyServerHelper.gson().toJson(result) + + "."); + } + } catch (Exception e) { + logger.warn( + "Ask em : " + + getNode().getServiceInstance().toString() + + " to kill engine : " + + engineStopRequest.getServiceInstance().toString() + + " failed, exception is : " + + e.getMessage() + + "."); + } + } + + @Override + public ECMOperateResponse executeOperation(ECMOperateRequest ecmOperateRequest) { + Object result = getSender().ask(ecmOperateRequest); + if (result instanceof ECMOperateResponse) { + return (ECMOperateResponse) result; + } else { + throw new AMErrorException(AMConstant.ENGINE_ERROR_CODE, "Failed to execute ECM operation."); + } + } +} diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/pointer/DefaultEngineNodPointer.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/pointer/DefaultEngineNodPointer.java new file mode 100644 index 0000000000..cdbbcbbf09 --- /dev/null +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/pointer/DefaultEngineNodPointer.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.manager.am.pointer; + +import org.apache.linkis.manager.am.exception.AMErrorException; +import org.apache.linkis.manager.common.entity.node.Node; +import org.apache.linkis.manager.common.protocol.RequestEngineLock; +import org.apache.linkis.manager.common.protocol.RequestEngineUnlock; +import org.apache.linkis.manager.common.protocol.ResponseEngineLock; +import org.apache.linkis.manager.common.protocol.engine.EngineOperateRequest; +import org.apache.linkis.manager.common.protocol.engine.EngineOperateResponse; + +import java.util.Optional; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class DefaultEngineNodPointer extends AbstractNodePointer implements EngineNodePointer { + private static final Logger logger = LoggerFactory.getLogger(DefaultEngineNodPointer.class); + + private Node node; + + public DefaultEngineNodPointer(Node node) { + this.node = node; + } + + @Override + public Node getNode() { + return node; + } + + @Override + public Optional lockEngine(RequestEngineLock requestEngineLock) { + Object result = getSender().ask(requestEngineLock); + if (result instanceof ResponseEngineLock) { + ResponseEngineLock responseEngineLock = (ResponseEngineLock) result; + if (responseEngineLock.lockStatus()) { + return Optional.of(responseEngineLock.lock()); + } else { + logger.info( + "Failed to get locker," + node.getServiceInstance() + ": " + responseEngineLock.msg()); + return Optional.empty(); + } + } else { + return Optional.empty(); + } + } + + @Override + public void releaseLock(RequestEngineUnlock requestEngineUnlock) { + getSender().send(requestEngineUnlock); + } + + @Override + public EngineOperateResponse executeOperation(EngineOperateRequest engineOperateRequest) { + Object result = getSender().ask(engineOperateRequest); + if (result instanceof EngineOperateResponse) { + return (EngineOperateResponse) result; + } else { + throw new AMErrorException(-1, "Illegal response of operation."); + } + } +} diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/pointer/DefaultNodePointerBuilder.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/pointer/DefaultNodePointerBuilder.java new file mode 100644 index 0000000000..58ec1ae940 --- /dev/null +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/pointer/DefaultNodePointerBuilder.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.manager.am.pointer; + +import org.apache.linkis.manager.common.entity.node.EMNode; +import org.apache.linkis.manager.common.entity.node.EngineNode; + +import org.springframework.stereotype.Component; + +@Component +public class DefaultNodePointerBuilder implements NodePointerBuilder { + @Override + public EMNodPointer buildEMNodePointer(EMNode node) { + return new DefaultEMNodPointer(node); + } + + @Override + public EngineNodePointer buildEngineNodePointer(EngineNode node) { + return new DefaultEngineNodPointer(node); + } +} diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/service/common/pointer/EMNodPointer.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/pointer/EMNodPointer.java similarity index 63% rename from linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/service/common/pointer/EMNodPointer.scala rename to linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/pointer/EMNodPointer.java index 803c191a0f..a85bf6eaca 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/service/common/pointer/EMNodPointer.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/pointer/EMNodPointer.java @@ -15,19 +15,19 @@ * limitations under the License. */ -package org.apache.linkis.manager.service.common.pointer +package org.apache.linkis.manager.am.pointer; -import org.apache.linkis.manager.common.entity.node.EngineNode -import org.apache.linkis.manager.common.protocol.em.{ECMOperateRequest, ECMOperateResponse} -import org.apache.linkis.manager.common.protocol.engine.EngineStopRequest -import org.apache.linkis.manager.engineplugin.common.launch.entity.EngineConnLaunchRequest +import org.apache.linkis.manager.common.entity.node.EngineNode; +import org.apache.linkis.manager.common.protocol.em.ECMOperateRequest; +import org.apache.linkis.manager.common.protocol.em.ECMOperateResponse; +import org.apache.linkis.manager.common.protocol.engine.EngineStopRequest; +import org.apache.linkis.manager.engineplugin.common.launch.entity.EngineConnLaunchRequest; -trait EMNodPointer extends NodePointer { +public interface EMNodPointer extends NodePointer { - def createEngine(engineConnLaunchRequest: EngineConnLaunchRequest): EngineNode + EngineNode createEngine(EngineConnLaunchRequest engineConnLaunchRequest); - def stopEngine(engineStopRequest: EngineStopRequest): Unit - - def executeOperation(request: ECMOperateRequest): ECMOperateResponse + void stopEngine(EngineStopRequest engineStopRequest); + ECMOperateResponse executeOperation(ECMOperateRequest request); } diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/recycle/AssignNodeRuleExecutor.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/pointer/EngineNodePointer.java similarity index 55% rename from linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/recycle/AssignNodeRuleExecutor.scala rename to linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/pointer/EngineNodePointer.java index 340995a52e..8be00a09c2 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/recycle/AssignNodeRuleExecutor.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/pointer/EngineNodePointer.java @@ -15,24 +15,20 @@ * limitations under the License. */ -package org.apache.linkis.manager.am.recycle +package org.apache.linkis.manager.am.pointer; -import org.apache.linkis.common.ServiceInstance -import org.apache.linkis.manager.common.entity.recycle.{AssignNodeRule, RecyclingRule} +import org.apache.linkis.manager.common.protocol.RequestEngineLock; +import org.apache.linkis.manager.common.protocol.RequestEngineUnlock; +import org.apache.linkis.manager.common.protocol.engine.EngineOperateRequest; +import org.apache.linkis.manager.common.protocol.engine.EngineOperateResponse; -import org.springframework.stereotype.Component +import java.util.Optional; -@Component -class AssignNodeRuleExecutor extends RecyclingRuleExecutor { +public interface EngineNodePointer extends NodePointer { - override def ifAccept(recyclingRule: RecyclingRule): Boolean = - recyclingRule.isInstanceOf[AssignNodeRule] + Optional lockEngine(RequestEngineLock requestEngineLock); - override def executeRule(recyclingRule: RecyclingRule): Array[ServiceInstance] = - recyclingRule match { - case AssignNodeRule(serviceInstance, user) => - Array(serviceInstance) - case _ => null - } + void releaseLock(RequestEngineUnlock requestEngineUnlock); + EngineOperateResponse executeOperation(EngineOperateRequest engineOperateRequest); } diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/service/common/pointer/NodePointer.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/pointer/NodePointer.java similarity index 62% rename from linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/service/common/pointer/NodePointer.scala rename to linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/pointer/NodePointer.java index 5f8f0b5979..e5d519873e 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/service/common/pointer/NodePointer.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/pointer/NodePointer.java @@ -15,56 +15,50 @@ * limitations under the License. */ -package org.apache.linkis.manager.service.common.pointer +package org.apache.linkis.manager.am.pointer; -import org.apache.linkis.manager.common.entity.enumeration.NodeStatus -import org.apache.linkis.manager.common.entity.node.Node -import org.apache.linkis.manager.common.protocol.node.{NodeHealthyRequest, NodeHeartbeatMsg} -import org.apache.linkis.manager.label.entity.Label +import org.apache.linkis.manager.common.entity.enumeration.NodeStatus; +import org.apache.linkis.manager.common.entity.node.Node; +import org.apache.linkis.manager.common.protocol.node.NodeHealthyRequest; +import org.apache.linkis.manager.common.protocol.node.NodeHeartbeatMsg; +import org.apache.linkis.manager.label.entity.Label; -trait NodePointer { +public interface NodePointer { /** * 与该远程指针关联的node信息 * * @return */ - def getNode(): Node + Node getNode(); /** * 向对应的Node发送请求获取节点状态 * * @return */ - def getNodeStatus(): NodeStatus + NodeStatus getNodeStatus(); /** * 向对应的Node发送请求获取节点心跳信息 * * @return */ - def getNodeHeartbeatMsg(): NodeHeartbeatMsg + NodeHeartbeatMsg getNodeHeartbeatMsg(); /** * 向对应的Node发送Kill 请求 * * @return */ - def stopNode(): Unit + void stopNode(); /** * 向对应的Node发送Label更新 请求 * * @return */ - def updateLabels(labels: Array[Label[_]]): Unit + void updateLabels(Label[] labels); - def updateNodeHealthyRequest(nodeHealthyRequest: NodeHealthyRequest): Unit - - override def equals(obj: Any): Boolean = obj match { - case nodeB: Node => getNode().getServiceInstance.equals(nodeB.getServiceInstance) - case _ => false - } - - override def hashCode(): Int = super.hashCode() + void updateNodeHealthyRequest(NodeHealthyRequest nodeHealthyRequest); } diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/pointer/NodePointerBuilder.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/pointer/NodePointerBuilder.java new file mode 100644 index 0000000000..649e142399 --- /dev/null +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/pointer/NodePointerBuilder.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.manager.am.pointer; + +import org.apache.linkis.manager.common.entity.node.EMNode; +import org.apache.linkis.manager.common.entity.node.EngineNode; + +public interface NodePointerBuilder { + + EMNodPointer buildEMNodePointer(EMNode node); + + EngineNodePointer buildEngineNodePointer(EngineNode node); +} diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/recycle/AssignNodeRuleExecutor.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/recycle/AssignNodeRuleExecutor.java new file mode 100644 index 0000000000..f9a3acce4f --- /dev/null +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/recycle/AssignNodeRuleExecutor.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.manager.am.recycle; + +import org.apache.linkis.common.ServiceInstance; +import org.apache.linkis.manager.common.entity.recycle.AssignNodeRule; +import org.apache.linkis.manager.common.entity.recycle.RecyclingRule; + +import org.springframework.stereotype.Component; + +@Component +public class AssignNodeRuleExecutor implements RecyclingRuleExecutor { + @Override + public boolean ifAccept(RecyclingRule recyclingRule) { + return recyclingRule instanceof AssignNodeRule; + } + + @Override + public ServiceInstance[] executeRule(RecyclingRule recyclingRule) { + if (recyclingRule instanceof AssignNodeRule) { + AssignNodeRule assignNodeRule = (AssignNodeRule) recyclingRule; + return new ServiceInstance[] {assignNodeRule.serviceInstance()}; + } else { + return null; + } + } +} diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/recycle/RecyclingRuleExecutor.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/recycle/RecyclingRuleExecutor.java similarity index 77% rename from linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/recycle/RecyclingRuleExecutor.scala rename to linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/recycle/RecyclingRuleExecutor.java index 8a4db6fb03..ab53aa8281 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/recycle/RecyclingRuleExecutor.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/recycle/RecyclingRuleExecutor.java @@ -15,15 +15,14 @@ * limitations under the License. */ -package org.apache.linkis.manager.am.recycle +package org.apache.linkis.manager.am.recycle; -import org.apache.linkis.common.ServiceInstance -import org.apache.linkis.manager.common.entity.recycle.RecyclingRule +import org.apache.linkis.common.ServiceInstance; +import org.apache.linkis.manager.common.entity.recycle.RecyclingRule; -trait RecyclingRuleExecutor { +public interface RecyclingRuleExecutor { - def ifAccept(recyclingRule: RecyclingRule): Boolean - - def executeRule(recyclingRule: RecyclingRule): Array[ServiceInstance] + boolean ifAccept(RecyclingRule recyclingRule); + ServiceInstance[] executeRule(RecyclingRule recyclingRule); } diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/restful/EMRestfulApi.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/restful/EMRestfulApi.java index c0b481d318..fbebd6813b 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/restful/EMRestfulApi.java +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/restful/EMRestfulApi.java @@ -123,7 +123,7 @@ public class EMRestfulApi { private Logger logger = LoggerFactory.getLogger(EMRestfulApi.class); - private String[] adminOperations = AMConfiguration.ECM_ADMIN_OPERATIONS().getValue().split(","); + private String[] adminOperations = AMConfiguration.ECM_ADMIN_OPERATIONS.getValue().split(","); private void checkAdmin(String userName) throws AMErrorException { if (Configuration.isNotAdmin(userName)) { diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/restful/EngineRestfulApi.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/restful/EngineRestfulApi.java index e9210326ef..4f68937e63 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/restful/EngineRestfulApi.java +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/restful/EngineRestfulApi.java @@ -124,7 +124,7 @@ public Message askEngineConn( engineAskRequest.setUser(userName); long timeout = engineAskRequest.getTimeOut(); if (timeout <= 0) { - timeout = AMConfiguration.ENGINE_CONN_START_REST_MAX_WAIT_TIME().getValue().toLong(); + timeout = AMConfiguration.ENGINE_CONN_START_REST_MAX_WAIT_TIME.getValue().toLong(); engineAskRequest.setTimeOut(timeout); } Map retEngineNode = new HashMap<>(); @@ -208,7 +208,7 @@ public Object call() { EngineNode createNode = engineCreateService.createEngine(engineCreateRequest, sender); long timeout = 0L; if (engineCreateRequest.getTimeout() <= 0) { - timeout = AMConfiguration.ENGINE_START_MAX_TIME().getValue().toLong(); + timeout = AMConfiguration.ENGINE_START_MAX_TIME.getValue().toLong(); } else { timeout = engineCreateRequest.getTimeout(); } @@ -301,7 +301,7 @@ public Message createEngineConn( engineCreateRequest.setUser(userName); long timeout = engineCreateRequest.getTimeout(); if (timeout <= 0) { - timeout = AMConfiguration.ENGINE_CONN_START_REST_MAX_WAIT_TIME().getValue().toLong(); + timeout = AMConfiguration.ENGINE_CONN_START_REST_MAX_WAIT_TIME.getValue().toLong(); engineCreateRequest.setTimeout(timeout); } logger.info( diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/impl/DefaultEngineConnPidCallbackService.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/impl/DefaultEngineConnPidCallbackService.java index 3d199fe29c..5fbbb7c32a 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/impl/DefaultEngineConnPidCallbackService.java +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/impl/DefaultEngineConnPidCallbackService.java @@ -17,10 +17,14 @@ package org.apache.linkis.manager.am.service.impl; +import org.apache.linkis.common.ServiceInstance; import org.apache.linkis.governance.common.protocol.task.ResponseEngineConnPid; import org.apache.linkis.manager.am.manager.DefaultEngineNodeManager; import org.apache.linkis.manager.am.service.EngineConnPidCallbackService; +import org.apache.linkis.manager.am.service.engine.AbstractEngineService; +import org.apache.linkis.manager.common.constant.AMConstant; import org.apache.linkis.manager.common.entity.node.EngineNode; +import org.apache.linkis.manager.label.service.NodeLabelService; import org.apache.linkis.rpc.message.annotation.Receiver; import org.springframework.beans.factory.annotation.Autowired; @@ -30,12 +34,15 @@ import org.slf4j.LoggerFactory; @Service -public class DefaultEngineConnPidCallbackService implements EngineConnPidCallbackService { +public class DefaultEngineConnPidCallbackService extends AbstractEngineService + implements EngineConnPidCallbackService { private static final Logger logger = LoggerFactory.getLogger(DefaultEngineConnPidCallbackService.class); @Autowired private DefaultEngineNodeManager defaultEngineNodeManager; + @Autowired private NodeLabelService nodeLabelService; + @Receiver @Override public void dealPid(ResponseEngineConnPid protocol) { @@ -47,7 +54,8 @@ public void dealPid(ResponseEngineConnPid protocol) { protocol.pid(), protocol.ticketId()); - EngineNode engineNode = defaultEngineNodeManager.getEngineNode(protocol.serviceInstance()); + EngineNode engineNode = + defaultEngineNodeManager.getEngineNodeInfoByTicketId(protocol.ticketId()); if (engineNode == null) { logger.error( "DefaultEngineConnPidCallbackService dealPid failed, engineNode is null, serviceInstance:{}", @@ -56,6 +64,13 @@ public void dealPid(ResponseEngineConnPid protocol) { } engineNode.setIdentifier(protocol.pid()); + ServiceInstance oldServiceInstance = engineNode.getServiceInstance(); + if (engineNode.getMark().equals(AMConstant.CLUSTER_PROCESS_MARK)) { + ServiceInstance serviceInstance = protocol.serviceInstance(); + engineNode.setServiceInstance(serviceInstance); + getEngineNodeManager().updateEngineNode(oldServiceInstance, engineNode); + nodeLabelService.labelsFromInstanceToNewInstance(oldServiceInstance, serviceInstance); + } defaultEngineNodeManager.updateEngine(engineNode); } } diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/impl/DefaultEngineConnStatusCallbackService.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/impl/DefaultEngineConnStatusCallbackService.java index 7efeabd3e8..7b3d53f4ca 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/impl/DefaultEngineConnStatusCallbackService.java +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/impl/DefaultEngineConnStatusCallbackService.java @@ -18,6 +18,7 @@ package org.apache.linkis.manager.am.service.impl; import org.apache.linkis.manager.am.conf.AMConfiguration; +import org.apache.linkis.manager.am.converter.MetricsConverter; import org.apache.linkis.manager.am.service.EngineConnStatusCallbackService; import org.apache.linkis.manager.am.service.engine.EngineStopService; import org.apache.linkis.manager.common.constant.AMConstant; @@ -26,7 +27,6 @@ import org.apache.linkis.manager.common.protocol.engine.EngineConnStatusCallback; import org.apache.linkis.manager.common.protocol.engine.EngineConnStatusCallbackToAM; import org.apache.linkis.manager.persistence.NodeMetricManagerPersistence; -import org.apache.linkis.manager.service.common.metrics.MetricsConverter; import org.apache.linkis.rpc.message.annotation.Receiver; import org.apache.linkis.server.BDPJettyServerHelper; @@ -56,8 +56,7 @@ public class DefaultEngineConnStatusCallbackService implements EngineConnStatusC @Autowired private EngineStopService engineStopService; - private static final String[] canRetryLogs = - AMConfiguration.AM_CAN_RETRY_LOGS().getValue().split(";"); + private static final String[] canRetryLogs = AMConfiguration.AM_CAN_RETRY_LOGS.split(";"); @Receiver public void dealEngineConnStatusCallback(EngineConnStatusCallback protocol) { diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/rm/conf/ResourceStatus.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/rm/conf/ResourceStatus.java index 13b58cc379..9639639a9b 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/rm/conf/ResourceStatus.java +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/rm/conf/ResourceStatus.java @@ -38,9 +38,9 @@ public enum ResourceStatus { public static ResourceStatus measure(Resource leftResource, Resource maxResource) { if (leftResource != null && maxResource != null) { - if (leftResource.$less$eq(maxResource.$times(warnThreshold))) { + if (!leftResource.moreThan(maxResource.multiplied(warnThreshold))) { return critical; - } else if (leftResource.$less$eq(maxResource.$times(normalThreshold))) { + } else if (!leftResource.moreThan(maxResource.multiplied(normalThreshold))) { return warn; } else { return normal; diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/rm/domain/RMLabelContainer.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/rm/domain/RMLabelContainer.java index 1aa81e9026..4d15ce200b 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/rm/domain/RMLabelContainer.java +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/rm/domain/RMLabelContainer.java @@ -50,7 +50,7 @@ public class RMLabelContainer { private EngineTypeLabel engineTypeLabel; private UserCreatorLabel userCreatorLabel; private EngineInstanceLabel engineInstanceLabel; - private CombinedLabel combinedUserCreatorEngineTypeLabel; + private CombinedLabel combinedResourceLabel; private Label currentLabel; public RMLabelContainer(List> labels) { @@ -58,11 +58,11 @@ public RMLabelContainer(List> labels) { this.lockedLabels = Lists.newArrayList(); try { if (getUserCreatorLabel() != null && getEngineTypeLabel() != null) { - this.combinedUserCreatorEngineTypeLabel = + this.combinedResourceLabel = (CombinedLabel) combinedLabelBuilder.build( "", Lists.newArrayList(getUserCreatorLabel(), getEngineTypeLabel())); - this.labels.add(combinedUserCreatorEngineTypeLabel); + this.labels.add(combinedResourceLabel); } } catch (Exception e) { logger.warn("failed to get combinedUserCreatorEngineTypeLabel", e); @@ -157,8 +157,8 @@ public EngineInstanceLabel getEngineInstanceLabel() throws RMErrorException { return null; } - public CombinedLabel getCombinedUserCreatorEngineTypeLabel() { - return combinedUserCreatorEngineTypeLabel; + public CombinedLabel getCombinedResourceLabel() { + return combinedResourceLabel; } public Label getCurrentLabel() { diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/rm/external/service/impl/ExternalResourceServiceImpl.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/rm/external/service/impl/ExternalResourceServiceImpl.java index 73d785643f..a5823853ec 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/rm/external/service/impl/ExternalResourceServiceImpl.java +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/rm/external/service/impl/ExternalResourceServiceImpl.java @@ -111,7 +111,7 @@ public NodeResource getResource( NodeResource resource = (NodeResource) retry( - (Integer) RMConfiguration.EXTERNAL_RETRY_NUM().getValue(), + RMConfiguration.EXTERNAL_RETRY_NUM.getValue(), (i) -> externalResourceRequester.requestResourceInfo(identifier, provider), (i) -> externalResourceRequester.reloadExternalResourceAddress(provider)); return resource; @@ -136,7 +136,7 @@ public List getAppInfo( List appInfos = (List) retry( - (Integer) RMConfiguration.EXTERNAL_RETRY_NUM().getValue(), + RMConfiguration.EXTERNAL_RETRY_NUM.getValue(), (i) -> externalResourceRequester.requestAppInfo(identifier, provider), (i) -> externalResourceRequester.reloadExternalResourceAddress(provider)); return appInfos; @@ -177,8 +177,8 @@ public ExternalResourceProvider chooseProvider( if (label == null) { realClusterLabel = LabelBuilderFactoryContext.getLabelBuilderFactory().createLabel(ClusterLabel.class); - realClusterLabel.setClusterName(RMConfiguration.DEFAULT_YARN_CLUSTER_NAME().getValue()); - realClusterLabel.setClusterType(RMConfiguration.DEFAULT_YARN_TYPE().getValue()); + realClusterLabel.setClusterName(RMConfiguration.DEFAULT_YARN_CLUSTER_NAME.getValue()); + realClusterLabel.setClusterType(RMConfiguration.DEFAULT_YARN_TYPE.getValue()); } else { realClusterLabel = (ClusterLabel) label; } diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/rm/external/yarn/YarnResourceRequester.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/rm/external/yarn/YarnResourceRequester.java index cd78c561ef..92f5018493 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/rm/external/yarn/YarnResourceRequester.java +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/rm/external/yarn/YarnResourceRequester.java @@ -159,7 +159,7 @@ public static JsonNode getChildQueues(JsonNode resp) { && !queues.isNull() && !queues.isMissingNode() && queues.isArray() - && ((ArrayNode) queues).size() > 0) { + && queues.size() > 0) { return queues; } else { return resp.get("childQueues"); diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/engineplugin/loader/EngineConnPluginLoaderConf.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/engineplugin/loader/EngineConnPluginLoaderConf.scala index 06ddd1552d..96b1258d9e 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/engineplugin/loader/EngineConnPluginLoaderConf.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/engineplugin/loader/EngineConnPluginLoaderConf.scala @@ -21,14 +21,8 @@ import org.apache.linkis.common.conf.{CommonVars, Configuration} object EngineConnPluginLoaderConf { - val ENGINE_PLUGIN_RESOURCE_ID_NAME_PREFIX: String = - "wds.linkis.engineConn.plugin.loader.resource-id." - - val CLASS_LOADER_CLASS_NAME: CommonVars[String] = - CommonVars("wds.linkis.engineconn.plugin.loader.classname", "") - val ENGINE_PLUGIN_LOADER_DEFAULT_USER: CommonVars[String] = - CommonVars("wds.linkis.engineconn.plugin.loader.defaultUser", "hadoop") + CommonVars("linkis.engineconn.plugin.loader.defaultUser", "hadoop") val ENGINE_PLUGIN_STORE_PATH: CommonVars[String] = CommonVars( "wds.linkis.engineconn.plugin.loader.store.path", diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/conf/AMConfiguration.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/conf/AMConfiguration.scala deleted file mode 100644 index 8ee323f445..0000000000 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/conf/AMConfiguration.scala +++ /dev/null @@ -1,192 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.manager.am.conf - -import org.apache.linkis.common.conf.{CommonVars, Configuration, TimeType} -import org.apache.linkis.common.utils.Utils -import org.apache.linkis.manager.common.entity.enumeration.MaintainType - -object AMConfiguration { - - val YARN_QUEUE_NAME_CONFIG_KEY = "wds.linkis.rm.yarnqueue" - - val CROSS_QUEUE = "crossQueue" - - val ACROSS_CLUSTER_TASK = "acrossClusterTask" - - val PRIORITY_CLUSTER = "priorityCluster" - - val PRIORITY_CLUSTER_TARGET = "bdp" - - val PRIORITY_CLUSTER_ORIGIN = "bdap" - - val TARGET_CPU_THRESHOLD = "targetCPUThreshold" - - val TARGET_MEMORY_THRESHOLD = "targetMemoryThreshold" - - val TARGET_CPU_PERCENTAGE_THRESHOLD = "targetCPUPercentageThreshold" - - val TARGET_MEMORY_PERCENTAGE_THRESHOLD = "targetMemoryPercentageThreshold" - - val ORIGIN_CPU_PERCENTAGE_THRESHOLD = "originCPUPercentageThreshold" - - val ORIGIN_MEMORY_PERCENTAGE_THRESHOLD = "originMemoryPercentageThreshold" - - val ACROSS_CLUSTER_TOTAL_MEMORY_PERCENTAGE_THRESHOLD: Double = - CommonVars("linkis.yarn.across.cluster.memory.threshold", 0.8).getValue - - val ACROSS_CLUSTER_TOTAL_CPU_PERCENTAGE_THRESHOLD: Double = - CommonVars("linkis.yarn.across.cluster.cpu.threshold", 0.8).getValue - - val ECM_ADMIN_OPERATIONS = CommonVars("wds.linkis.governance.admin.operations", "") - - val ENGINE_START_MAX_TIME = - CommonVars("wds.linkis.manager.am.engine.start.max.time", new TimeType("11m")) - - val ENGINE_CONN_START_REST_MAX_WAIT_TIME = - CommonVars("wds.linkis.manager.am.engine.rest.start.max.time", new TimeType("40s")) - - val ENGINE_REUSE_MAX_TIME = - CommonVars("wds.linkis.manager.am.engine.reuse.max.time", new TimeType("5m")) - - val ENGINE_REUSE_COUNT_LIMIT = CommonVars("wds.linkis.manager.am.engine.reuse.count.limit", 2) - - val NODE_STATUS_HEARTBEAT_TIME = - CommonVars("wds.linkis.manager.am.node.heartbeat", new TimeType("3m")) - - val NODE_HEARTBEAT_MAX_UPDATE_TIME = - CommonVars("wds.linkis.manager.am.node.heartbeat", new TimeType("5m")) - - val DEFAULT_NODE_OWNER = CommonVars("wds.linkis.manager.am.default.node.owner", "hadoop") - - val STOP_ENGINE_WAIT = CommonVars("wds.linkis.manager.am.stop.engine.wait", new TimeType("5m")) - - val STOP_EM_WAIT = CommonVars("wds.linkis.manager.am.stop.em.wait", new TimeType("5m")) - - val EM_LABEL_INIT_WAIT = - CommonVars("wds.linkis.manager.am.em.label.init.wait", new TimeType("5m")) - - val EM_NEW_WAIT_MILLS = CommonVars("wds.linkis.manager.am.em.new.wait.mills", 1000 * 60L) - - val ENGINECONN_DEBUG_ENABLED = CommonVars("wds.linkis.engineconn.debug.mode.enable", false) - - val MULTI_USER_ENGINE_TYPES = CommonVars( - "wds.linkis.multi.user.engine.types", - "jdbc,es,presto,io_file,appconn,openlookeng,trino,jobserver" - ) - - val ALLOW_BATCH_KILL_ENGINE_TYPES = - CommonVars("wds.linkis.allow.batch.kill.engine.types", "spark,hive,python") - - val UNALLOW_BATCH_KILL_ENGINE_TYPES = - CommonVars("wds.linkis.unallow.batch.kill.engine.types", "trino,appconn,io_file") - - val MULTI_USER_ENGINE_USER = - CommonVars("wds.linkis.multi.user.engine.user", getDefaultMultiEngineUser) - - val ENGINE_LOCKER_MAX_TIME = - CommonVars("wds.linkis.manager.am.engine.locker.max.time", 1000 * 60 * 5) - - val AM_CAN_RETRY_LOGS = - CommonVars("wds.linkis.manager.am.can.retry.logs", "already in use;Cannot allocate memory") - - val ASK_ENGINE_ASYNC_MAX_THREAD_SIZE: Int = - CommonVars("wds.linkis.ecm.launch.max.thread.size", 200).getValue - - val ASYNC_STOP_ENGINE_MAX_THREAD_SIZE: Int = - CommonVars("wds.linkis.async.stop.engine.size", 20).getValue - - val EC_MAINTAIN_TIME_STR = - CommonVars("wds.linkis.ec.maintain.time.key", MaintainType.Default.toString) - - val EC_MAINTAIN_WORK_START_TIME = - CommonVars("wds.linkis.ec.maintain.time.work.start.time", 8).getValue - - val EC_MAINTAIN_WORK_END_TIME = - CommonVars("wds.linkis.ec.maintain.time.work.end.time", 19).getValue - - val NODE_SELECT_HOTSPOT_EXCLUSION_RULE = - CommonVars("linkis.node.select.hotspot.exclusion.rule.enable", true).getValue - - val EC_REUSE_WITH_RESOURCE_RULE_ENABLE = - CommonVars("linkis.ec.reuse.with.resource.rule.enable", false).getValue - - val EC_REUSE_WITH_RESOURCE_WITH_ECS: String = - CommonVars("linkis.ec.reuse.with.resource.with.ecs", "spark,hive,shell,python").getValue - - val SUPPORT_CLUSTER_RULE_EC_TYPES: String = - CommonVars("linkis.support.cluster.rule.ec.types", "").getValue - - val HIVE_CLUSTER_EC_EXECUTE_ONCE_RULE_ENABLE = - CommonVars("linkis.hive.cluster.ec.execute.once.rule.enable", true).getValue - - val AM_ENGINE_ASK_MAX_NUMBER = - CommonVars("linkis.am.engine.ask.max.number", "appconn=5,trino=10").getValue - .split(",") - .map { keyValue => - val array = keyValue.split("=") - if (array.length != 2) { - throw new IllegalArgumentException( - s"linkis.am.engine.ask.max.number value is illegal, value is $keyValue" - ) - } else { - (array(0), array(1).toInt) - } - } - .toMap - - val AM_ECM_RESET_RESOURCE = CommonVars("linkis.am.ecm.reset.resource.enable", true).getValue - - val AM_USER_RESET_RESOURCE = CommonVars("linkis.am.user.reset.resource.enable", true).getValue - - private def getDefaultMultiEngineUser(): String = { - val jvmUser = Utils.getJvmUser - s""" {jdbc:"$jvmUser", es: "$jvmUser", presto:"$jvmUser", appconn:"$jvmUser", openlookeng:"$jvmUser", trino:"$jvmUser", io_file:"root", jobserver:"$jvmUser"}""" - } - - def isMultiUserEngine(engineType: String): Boolean = { - val multiUserEngine = AMConfiguration.MULTI_USER_ENGINE_TYPES.getValue.split(",") - val findResult = multiUserEngine.find(_.equalsIgnoreCase(engineType)) - if (findResult.isDefined) { - true - } else { - false - } - } - - def isAllowKilledEngineType(engineType: String): Boolean = { - val allowBatchKillEngine = AMConfiguration.ALLOW_BATCH_KILL_ENGINE_TYPES.getValue.split(",") - val findResult = allowBatchKillEngine.find(_.equalsIgnoreCase(engineType)) - if (findResult.isDefined) { - true - } else { - false - } - } - - def isUnAllowKilledEngineType(engineType: String): Boolean = { - val unAllowBatchKillEngine = AMConfiguration.UNALLOW_BATCH_KILL_ENGINE_TYPES.getValue.split(",") - val findResult = unAllowBatchKillEngine.find(engineType.toLowerCase().contains(_)) - if (findResult.isDefined) { - true - } else { - false - } - } - -} diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/conf/ConfigurationMapCache.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/conf/ConfigurationMapCache.scala deleted file mode 100644 index 961cc7834a..0000000000 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/conf/ConfigurationMapCache.scala +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.manager.am.conf - -import org.apache.linkis.common.conf.Configuration -import org.apache.linkis.governance.common.protocol.conf.{ - RequestQueryEngineConfig, - RequestQueryEngineConfigWithGlobalConfig, - RequestQueryGlobalConfig, - ResponseQueryConfig -} -import org.apache.linkis.manager.label.entity.engine.{EngineTypeLabel, UserCreatorLabel} -import org.apache.linkis.manager.label.utils.EngineTypeLabelCreator -import org.apache.linkis.protocol.CacheableProtocol -import org.apache.linkis.rpc.RPCMapCache - -import java.util - -object ConfigurationMapCache { - - val globalMapCache = new RPCMapCache[UserCreatorLabel, String, String]( - Configuration.CLOUD_CONSOLE_CONFIGURATION_SPRING_APPLICATION_NAME.getValue - ) { - - override protected def createRequest(userCreatorLabel: UserCreatorLabel): CacheableProtocol = - RequestQueryGlobalConfig(userCreatorLabel.getUser) - - override protected def createMap(any: Any): util.Map[String, String] = any match { - case response: ResponseQueryConfig => response.getKeyAndValue - } - - } - - val engineMapCache = new RPCMapCache[(UserCreatorLabel, EngineTypeLabel), String, String]( - Configuration.CLOUD_CONSOLE_CONFIGURATION_SPRING_APPLICATION_NAME.getValue - ) { - - override protected def createRequest( - labelTuple: (UserCreatorLabel, EngineTypeLabel) - ): CacheableProtocol = - RequestQueryEngineConfigWithGlobalConfig(labelTuple._1, labelTuple._2) - - override protected def createMap(any: Any): util.Map[String, String] = any match { - case response: ResponseQueryConfig => response.getKeyAndValue - } - - } - -} diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/conf/EngineConnConfigurationService.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/conf/EngineConnConfigurationService.scala deleted file mode 100644 index ad306ef653..0000000000 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/conf/EngineConnConfigurationService.scala +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.manager.am.conf - -import org.apache.linkis.common.utils.{Logging, Utils} -import org.apache.linkis.manager.label.entity.Label -import org.apache.linkis.manager.label.entity.engine.{EngineTypeLabel, UserCreatorLabel} -import org.apache.linkis.server.JMap - -import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean -import org.springframework.context.annotation.{Bean, Configuration} - -import java.util - -import scala.collection.JavaConverters._ - -trait EngineConnConfigurationService { - - def getConsoleConfiguration(label: util.List[Label[_]]): util.Map[String, String] - -} - -class DefaultEngineConnConfigurationService extends EngineConnConfigurationService with Logging { - - override def getConsoleConfiguration(label: util.List[Label[_]]): util.Map[String, String] = { - val properties = new JMap[String, String] - val userCreatorLabelOption = label.asScala.find(_.isInstanceOf[UserCreatorLabel]) - val engineTypeLabelOption = label.asScala.find(_.isInstanceOf[EngineTypeLabel]) - if (userCreatorLabelOption.isDefined) { - val userCreatorLabel = userCreatorLabelOption.get.asInstanceOf[UserCreatorLabel] - if (engineTypeLabelOption.isDefined) { - val engineTypeLabel = engineTypeLabelOption.get.asInstanceOf[EngineTypeLabel] - val engineConfig = Utils.tryAndWarn( - ConfigurationMapCache.engineMapCache.getCacheMap((userCreatorLabel, engineTypeLabel)) - ) - if (null != engineConfig) { - properties.putAll(engineConfig) - } - } - } - properties - } - -} - -@Configuration -class ApplicationManagerSpringConfiguration { - - @ConditionalOnMissingBean - @Bean - def getDefaultEngineConnConfigurationService: EngineConnConfigurationService = { - new DefaultEngineConnConfigurationService - } - -} diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/conf/ManagerMonitorConf.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/conf/ManagerMonitorConf.scala deleted file mode 100644 index 3516fd6770..0000000000 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/conf/ManagerMonitorConf.scala +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.manager.am.conf - -import org.apache.linkis.common.conf.{CommonVars, TimeType} - -object ManagerMonitorConf { - - val NODE_MAX_CREATE_TIME = - CommonVars("wds.linkis.manager.am.node.create.time", new TimeType("12m")) - - val NODE_HEARTBEAT_MAX_UPDATE_TIME = - CommonVars("wds.linkis.manager.am.node.heartbeat", new TimeType("12m")) - - val ENGINE_KILL_TIMEOUT = - CommonVars("wds.linkis.manager.am.engine.kill.timeout", new TimeType("2m")) - - val EM_KILL_TIMEOUT = CommonVars("wds.linkis.manager.am.em.kill.timeout", new TimeType("2m")) - - val MANAGER_MONITOR_ASYNC_POLL_SIZE = - CommonVars("wds.linkis.manager.monitor.async.poll.size", 5) - - val MONITOR_SWITCH_ON = CommonVars("wds.linkis.manager.am.monitor.switch.on", true) - - val ECM_HEARTBEAT_MAX_UPDATE_TIME = - CommonVars("wds.linkis.manager.am.ecm.heartbeat", new TimeType("5m")) - - val ACROSS_QUEUES_RESOURCE_SHOW_SWITCH_ON = - CommonVars("wds.linkis.manager.across.resource.show.switch.on", false) - -} diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/converter/DefaultMetricsConverter.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/converter/DefaultMetricsConverter.scala deleted file mode 100644 index eb6de51921..0000000000 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/converter/DefaultMetricsConverter.scala +++ /dev/null @@ -1,103 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.manager.am.converter - -import org.apache.linkis.common.utils.Logging -import org.apache.linkis.manager.common.entity.enumeration.NodeStatus -import org.apache.linkis.manager.common.entity.metrics.{ - NodeHealthyInfo, - NodeMetrics, - NodeOverLoadInfo, - NodeTaskInfo -} -import org.apache.linkis.manager.common.entity.node.AMNode -import org.apache.linkis.manager.service.common.metrics.MetricsConverter -import org.apache.linkis.server.BDPJettyServerHelper - -import org.apache.commons.lang3.StringUtils - -import org.springframework.stereotype.Component - -@Component -class DefaultMetricsConverter extends MetricsConverter with Logging { - - override def parseTaskInfo(nodeMetrics: NodeMetrics): NodeTaskInfo = { - val msg = nodeMetrics.getHeartBeatMsg - if (StringUtils.isNotBlank(msg)) { - val jsonNode = BDPJettyServerHelper.jacksonJson.readTree(msg) - if (jsonNode != null && jsonNode.has("taskInfo")) { - val taskInfo = BDPJettyServerHelper.jacksonJson.readValue( - jsonNode.get("taskInfo").asText(), - classOf[NodeTaskInfo] - ) - return taskInfo - } - } - null - } - - override def parseHealthyInfo(nodeMetrics: NodeMetrics): NodeHealthyInfo = { - val healthyInfo = nodeMetrics.getHealthy - if (StringUtils.isNotBlank(healthyInfo)) { - BDPJettyServerHelper.jacksonJson.readValue(healthyInfo, classOf[NodeHealthyInfo]) - } else { - null - } - } - - override def parseOverLoadInfo(nodeMetrics: NodeMetrics): NodeOverLoadInfo = { - val overLoad = nodeMetrics.getOverLoad - if (StringUtils.isNotBlank(overLoad)) { - BDPJettyServerHelper.jacksonJson.readValue(overLoad, classOf[NodeOverLoadInfo]) - } else { - null - } - } - - override def parseStatus(nodeMetrics: NodeMetrics): NodeStatus = { - NodeStatus.values()(nodeMetrics.getStatus) - } - - override def convertTaskInfo(nodeTaskInfo: NodeTaskInfo): String = { - BDPJettyServerHelper.jacksonJson.writeValueAsString(nodeTaskInfo) - } - - override def convertHealthyInfo(nodeHealthyInfo: NodeHealthyInfo): String = { - BDPJettyServerHelper.jacksonJson.writeValueAsString(nodeHealthyInfo) - } - - override def convertOverLoadInfo(nodeOverLoadInfo: NodeOverLoadInfo): String = { - BDPJettyServerHelper.jacksonJson.writeValueAsString(nodeOverLoadInfo) - } - - override def convertStatus(nodeStatus: NodeStatus): Int = { - nodeStatus.ordinal() - } - - override def fillMetricsToNode(amNode: AMNode, metrics: NodeMetrics): AMNode = { - if (metrics == null) return amNode - amNode.setNodeStatus(parseStatus(metrics)) - amNode.setNodeTaskInfo(parseTaskInfo(metrics)) - amNode.setNodeHealthyInfo(parseHealthyInfo(metrics)) - amNode.setNodeOverLoadInfo(parseOverLoadInfo(metrics)) - amNode.setUpdateTime(metrics.getUpdateTime) - amNode.setNodeDescription(metrics.getDescription) - amNode - } - -} diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/label/AMLabelChecker.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/label/AMLabelChecker.scala deleted file mode 100644 index 2d7f5d6e2d..0000000000 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/label/AMLabelChecker.scala +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.manager.am.label - -import org.apache.linkis.manager.label.entity.Label -import org.apache.linkis.manager.label.entity.em.EMInstanceLabel -import org.apache.linkis.manager.label.entity.engine.{EngineTypeLabel, UserCreatorLabel} -import org.apache.linkis.manager.service.common.label.LabelChecker - -import org.springframework.stereotype.Component - -import java.util - -import scala.collection.JavaConverters._ - -@Component -class AMLabelChecker extends LabelChecker { - - override def checkEngineLabel(labelList: util.List[Label[_]]): Boolean = { - checkCorrespondingLabel(labelList, classOf[EngineTypeLabel], classOf[UserCreatorLabel]) - } - - override def checkEMLabel(labelList: util.List[Label[_]]): Boolean = { - checkCorrespondingLabel(labelList, classOf[EMInstanceLabel]) - } - - override def checkCorrespondingLabel( - labelList: util.List[Label[_]], - clazz: Class[_]* - ): Boolean = { - // TODO: 是否需要做子类的判断 - labelList.asScala.filter(null != _).map(_.getClass).asJava.containsAll(clazz.asJava) - } - -} diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/label/AMLabelFilter.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/label/AMLabelFilter.scala deleted file mode 100644 index 783061c8dd..0000000000 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/label/AMLabelFilter.scala +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.manager.am.label - -import org.apache.linkis.governance.common.conf.GovernanceCommonConf._ -import org.apache.linkis.manager.label.entity.{EMNodeLabel, EngineNodeLabel, Label} -import org.apache.linkis.manager.label.entity.node.AliasServiceInstanceLabel -import org.apache.linkis.manager.service.common.label.LabelFilter - -import org.springframework.stereotype.Component - -import java.util - -import scala.collection.JavaConverters._ - -@Component -class AMLabelFilter extends LabelFilter { - - override def choseEngineLabel(labelList: util.List[Label[_]]): util.List[Label[_]] = { - labelList.asScala.filter { - case _: EngineNodeLabel => true - // TODO: magic - case label: AliasServiceInstanceLabel - if label.getAlias.equals(ENGINE_CONN_SPRING_NAME.getValue) => - true - case _ => false - } - }.asJava - - override def choseEMLabel(labelList: util.List[Label[_]]): util.List[Label[_]] = { - labelList.asScala.filter { - case _: EMNodeLabel => true - // TODO: magic - case label: AliasServiceInstanceLabel - if label.getAlias.equals(ENGINE_CONN_MANAGER_SPRING_NAME.getValue) => - true - case _ => false - } - }.asJava - -} diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/label/DefaultManagerLabelService.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/label/DefaultManagerLabelService.scala deleted file mode 100644 index cb1c9843ff..0000000000 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/label/DefaultManagerLabelService.scala +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.manager.am.label - -import org.apache.linkis.common.ServiceInstance -import org.apache.linkis.common.utils.Logging -import org.apache.linkis.manager.label.entity.Label -import org.apache.linkis.manager.label.entity.em.EMInstanceLabel -import org.apache.linkis.manager.label.entity.engine.EngineInstanceLabel -import org.apache.linkis.manager.label.service.NodeLabelService -import org.apache.linkis.manager.service.common.label.ManagerLabelService - -import org.springframework.beans.factory.annotation.Autowired -import org.springframework.stereotype.Service - -import java.util - -import scala.collection.JavaConverters._ - -@Service -class DefaultManagerLabelService extends ManagerLabelService with Logging { - - @Autowired - private var nodeLabelService: NodeLabelService = _ - - override def isEngine(serviceInstance: ServiceInstance): Boolean = { - val list = nodeLabelService.getNodeLabels(serviceInstance) - isEngine(list) - } - - override def isEM(serviceInstance: ServiceInstance): Boolean = { - val list = nodeLabelService.getNodeLabels(serviceInstance) - val isEngine = list.asScala.exists { - case _: EngineInstanceLabel => - true - case _ => false - } - if (!isEngine) { - list.asScala.exists { - case _: EMInstanceLabel => - true - case _ => false - } - } else { - false - } - } - - override def isEngine(labels: util.List[Label[_]]): Boolean = { - labels.asScala.exists { - case _: EngineInstanceLabel => - true - case _ => false - } - } - -} diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/label/MultiUserEngineReuseLabelChooser.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/label/MultiUserEngineReuseLabelChooser.scala deleted file mode 100644 index ae001caca8..0000000000 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/label/MultiUserEngineReuseLabelChooser.scala +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.manager.am.label - -import org.apache.linkis.common.utils.Logging -import org.apache.linkis.manager.am.conf.AMConfiguration -import org.apache.linkis.manager.am.exception.{AMErrorCode, AMErrorException} -import org.apache.linkis.manager.label.entity.Label -import org.apache.linkis.manager.label.entity.engine.{EngineTypeLabel, UserCreatorLabel} -import org.apache.linkis.server.BDPJettyServerHelper - -import org.apache.commons.lang3.StringUtils - -import org.springframework.stereotype.Component - -import java.util - -import scala.collection.JavaConverters._ - -@Component -class MultiUserEngineReuseLabelChooser extends EngineReuseLabelChooser with Logging { - - private val multiUserEngine = AMConfiguration.MULTI_USER_ENGINE_TYPES.getValue.split(",") - - private val userMap = getMultiUserEngineUserMap() - - def getMultiUserEngineUserMap(): util.Map[String, String] = { - val userJson = AMConfiguration.MULTI_USER_ENGINE_USER.getValue - if (StringUtils.isNotBlank(userJson)) { - val userMap = - BDPJettyServerHelper.gson.fromJson(userJson, classOf[java.util.Map[String, String]]) - userMap - } else { - throw new AMErrorException( - AMErrorCode.AM_CONF_ERROR.getErrorCode, - s"Multi-user engine parameter configuration error,please check key ${AMConfiguration.MULTI_USER_ENGINE_USER.key}" - ) - } - - } - - /** - * Filter out UserCreator Label that supports multi-user engine 过滤掉支持多用户引擎的UserCreator Label - * - * @param labelList - * @return - */ - override def chooseLabels(labelList: util.List[Label[_]]): util.List[Label[_]] = { - val labels = labelList.asScala - val engineTypeLabelOption = labels.find(_.isInstanceOf[EngineTypeLabel]) - if (engineTypeLabelOption.isDefined) { - val engineTypeLabel = engineTypeLabelOption.get.asInstanceOf[EngineTypeLabel] - val maybeString = multiUserEngine.find(_.equalsIgnoreCase(engineTypeLabel.getEngineType)) - val userCreatorLabelOption = labels.find(_.isInstanceOf[UserCreatorLabel]) - if (maybeString.isDefined && userCreatorLabelOption.isDefined) { - val userAdmin = userMap.get(engineTypeLabel.getEngineType) - val userCreatorLabel = userCreatorLabelOption.get.asInstanceOf[UserCreatorLabel] - logger.info( - s"For multi user engine to reset userCreatorLabel user ${userCreatorLabel.getUser} to admin $userAdmin " - ) - userCreatorLabel.setUser(userAdmin) - return labels.asJava - } - } - labelList - } - -} diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/locker/DefaultEngineNodeLocker.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/locker/DefaultEngineNodeLocker.scala deleted file mode 100644 index b582b913d0..0000000000 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/locker/DefaultEngineNodeLocker.scala +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.manager.am.locker - -import org.apache.linkis.common.utils.{Logging, Utils} -import org.apache.linkis.manager.common.entity.node.{AMEngineNode, EngineNode} -import org.apache.linkis.manager.common.protocol.{ - RequestEngineLock, - RequestEngineUnlock, - RequestManagerUnlock -} -import org.apache.linkis.manager.service.common.pointer.NodePointerBuilder -import org.apache.linkis.rpc.message.annotation.Receiver - -import org.springframework.beans.factory.annotation.Autowired -import org.springframework.stereotype.Component - -@Component -class DefaultEngineNodeLocker extends EngineNodeLocker with Logging { - - @Autowired - private var nodeBuilder: NodePointerBuilder = _ - - override def lockEngine(engineNode: EngineNode, timeout: Long): Option[String] = { - // TODO 判断engine需要的锁类型进行不同的实例化 - nodeBuilder.buildEngineNodePointer(engineNode).lockEngine(RequestEngineLock(timeout)) - } - - override def releaseLock(engineNode: EngineNode, lock: String): Unit = { - nodeBuilder.buildEngineNodePointer(engineNode).releaseLock(RequestEngineUnlock(lock)) - } - - @Receiver - def releaseLock(requestManagerUnlock: RequestManagerUnlock): Unit = Utils.tryAndWarn { - logger.info( - s"client${requestManagerUnlock.clientInstance} Start to unlock engine ${requestManagerUnlock.engineInstance}" - ) - val engineNode = new AMEngineNode() - engineNode.setServiceInstance(requestManagerUnlock.engineInstance) - releaseLock(engineNode, requestManagerUnlock.lock) - logger.info( - s"client${requestManagerUnlock.clientInstance} Finished to unlock engine ${requestManagerUnlock.engineInstance}" - ) - } - -} diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/manager/DefaultEMNodeManager.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/manager/DefaultEMNodeManager.scala deleted file mode 100644 index 95f9ebc75b..0000000000 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/manager/DefaultEMNodeManager.scala +++ /dev/null @@ -1,193 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.manager.am.manager - -import org.apache.linkis.common.ServiceInstance -import org.apache.linkis.common.utils.{Logging, Utils} -import org.apache.linkis.manager.common.entity.node._ -import org.apache.linkis.manager.common.entity.persistence.PersistenceNodeEntity -import org.apache.linkis.manager.common.protocol.em.{ECMOperateRequest, ECMOperateResponse} -import org.apache.linkis.manager.common.protocol.engine.EngineStopRequest -import org.apache.linkis.manager.engineplugin.common.launch.entity.EngineConnLaunchRequest -import org.apache.linkis.manager.exception.NodeInstanceDuplicateException -import org.apache.linkis.manager.persistence.{NodeManagerPersistence, NodeMetricManagerPersistence} -import org.apache.linkis.manager.rm.service.ResourceManager -import org.apache.linkis.manager.service.common.metrics.MetricsConverter -import org.apache.linkis.manager.service.common.pointer.NodePointerBuilder - -import org.springframework.beans.factory.annotation.Autowired -import org.springframework.stereotype.Component - -import java.util - -import scala.collection.JavaConverters._ - -@Component -class DefaultEMNodeManager extends EMNodeManager with Logging { - - @Autowired - private var nodeManagerPersistence: NodeManagerPersistence = _ - - @Autowired - private var nodeMetricManagerPersistence: NodeMetricManagerPersistence = _ - - @Autowired - private var metricsConverter: MetricsConverter = _ - - @Autowired - private var nodePointerBuilder: NodePointerBuilder = _ - - @Autowired - private var resourceManager: ResourceManager = _ - - override def emRegister(emNode: EMNode): Unit = { - nodeManagerPersistence.addNodeInstance(emNode) - // init metric - nodeMetricManagerPersistence.addOrupdateNodeMetrics( - metricsConverter.getInitMetric(emNode.getServiceInstance) - ) - } - - override def addEMNodeInstance(emNode: EMNode): Unit = { - Utils.tryCatch(nodeManagerPersistence.addNodeInstance(emNode)) { - case e: NodeInstanceDuplicateException => - logger.warn(s"em instance had exists, $emNode.") - nodeManagerPersistence.updateEngineNode(emNode.getServiceInstance, emNode) - case t: Throwable => throw t - } - } - - override def initEMNodeMetrics(emNode: EMNode): Unit = { - nodeMetricManagerPersistence.addOrupdateNodeMetrics( - metricsConverter.getInitMetric(emNode.getServiceInstance) - ) - } - - override def listEngines(emNode: EMNode): util.List[EngineNode] = { - val nodes = nodeManagerPersistence.getEngineNodeByEM(emNode.getServiceInstance) - val metricses = nodeMetricManagerPersistence - .getNodeMetrics(nodes) - .asScala - .map(m => (m.getServiceInstance.toString, m)) - .toMap - nodes.asScala.map { node => - metricses - .get(node.getServiceInstance.toString) - .foreach(metricsConverter.fillMetricsToNode(node, _)) - node - } - nodes - } - - override def listUserEngines(emNode: EMNode, user: String): util.List[EngineNode] = { - listEngines(emNode).asScala.filter(_.getOwner.equals(user)) - }.asJava - - def listUserNodes(user: String): java.util.List[Node] = { - nodeManagerPersistence.getNodes(user) - } - - /** - * Get detailed em information from the persistence TODO add label to node ? - * - * @param scoreServiceInstances - * @return - */ - override def getEMNodes(scoreServiceInstances: Array[ScoreServiceInstance]): Array[EMNode] = { - - if (null == scoreServiceInstances || scoreServiceInstances.isEmpty) { - return null - } - val emNodes = scoreServiceInstances.map { scoreServiceInstances => - val emNode = new AMEMNode() - emNode.setScore(scoreServiceInstances.getScore) - emNode.setServiceInstance(scoreServiceInstances.getServiceInstance) - emNode - } - // 1. add nodeMetrics 2 add RM info - val resourceInfo = - resourceManager.getResourceInfo(scoreServiceInstances.map(_.getServiceInstance)) - val nodeMetrics = nodeMetricManagerPersistence.getNodeMetrics(emNodes.toList.asJava) - emNodes.map { emNode => - val optionMetrics = - nodeMetrics.asScala.find(_.getServiceInstance.equals(emNode.getServiceInstance)) - val optionRMNode = - resourceInfo.resourceInfo.asScala.find( - _.getServiceInstance.equals(emNode.getServiceInstance) - ) - optionMetrics.foreach(metricsConverter.fillMetricsToNode(emNode, _)) - optionRMNode.foreach(rmNode => emNode.setNodeResource(rmNode.getNodeResource)) - emNode - } - emNodes.toArray - } - - override def getEM(serviceInstance: ServiceInstance): EMNode = { - val node = nodeManagerPersistence.getNode(serviceInstance) - if (null == node) { - logger.info(s"This em of $serviceInstance not exists in db") - return null - } - val emNode = new AMEMNode() - emNode.setOwner(node.getOwner) - emNode.setServiceInstance(node.getServiceInstance) - node match { - case a: PersistenceNodeEntity => emNode.setStartTime(a.getStartTime) - case _ => - } - emNode.setMark(emNode.getMark) - metricsConverter.fillMetricsToNode(emNode, nodeMetricManagerPersistence.getNodeMetrics(emNode)) - emNode - } - - override def stopEM(emNode: EMNode): Unit = { - nodePointerBuilder.buildEMNodePointer(emNode).stopNode() - } - - override def deleteEM(emNode: EMNode): Unit = { - nodeManagerPersistence.removeNodeInstance(emNode) - logger.info(s"Finished to clear emNode instance(${emNode.getServiceInstance}) info ") - nodeMetricManagerPersistence.deleteNodeMetrics(emNode) - logger.info(s"Finished to clear emNode(${emNode.getServiceInstance}) metrics info") - } - - override def pauseEM(serviceInstance: ServiceInstance): Unit = {} - - /** - * 1. request engineManager to launch engine - * - * @param engineBuildRequest - * @param emNode - * @return - */ - override def createEngine( - engineConnLaunchRequest: EngineConnLaunchRequest, - emNode: EMNode - ): EngineNode = { - nodePointerBuilder.buildEMNodePointer(emNode).createEngine(engineConnLaunchRequest) - } - - override def stopEngine(engineStopRequest: EngineStopRequest, emNode: EMNode): Unit = { - nodePointerBuilder.buildEMNodePointer(emNode).stopEngine(engineStopRequest) - } - - override def executeOperation(ecmNode: EMNode, request: ECMOperateRequest): ECMOperateResponse = { - nodePointerBuilder.buildEMNodePointer(ecmNode).executeOperation(request) - } - -} diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/manager/DefaultEngineNodeManager.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/manager/DefaultEngineNodeManager.scala deleted file mode 100644 index 68e7d4cbd9..0000000000 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/manager/DefaultEngineNodeManager.scala +++ /dev/null @@ -1,323 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.manager.am.manager - -import org.apache.linkis.common.ServiceInstance -import org.apache.linkis.common.exception.LinkisRetryException -import org.apache.linkis.common.utils.{Logging, RetryHandler, Utils} -import org.apache.linkis.manager.am.conf.AMConfiguration -import org.apache.linkis.manager.am.exception.{AMErrorCode, AMErrorException} -import org.apache.linkis.manager.am.locker.EngineNodeLocker -import org.apache.linkis.manager.common.constant.AMConstant -import org.apache.linkis.manager.common.entity.enumeration.NodeStatus -import org.apache.linkis.manager.common.entity.node.{ - AMEngineNode, - EngineNode, - RMNode, - ScoreServiceInstance -} -import org.apache.linkis.manager.common.entity.persistence.PersistenceLabel -import org.apache.linkis.manager.common.protocol.engine.{ - EngineOperateRequest, - EngineOperateResponse -} -import org.apache.linkis.manager.label.builder.factory.LabelBuilderFactoryContext -import org.apache.linkis.manager.label.entity.engine.EngineInstanceLabel -import org.apache.linkis.manager.persistence.{ - LabelManagerPersistence, - NodeManagerPersistence, - NodeMetricManagerPersistence -} -import org.apache.linkis.manager.rm.service.ResourceManager -import org.apache.linkis.manager.service.common.metrics.MetricsConverter -import org.apache.linkis.manager.service.common.pointer.NodePointerBuilder - -import org.apache.commons.lang3.StringUtils - -import org.springframework.beans.factory.annotation.Autowired -import org.springframework.stereotype.Service - -import java.lang.reflect.UndeclaredThrowableException -import java.util - -import scala.collection.JavaConverters._ -import scala.tools.scalap.scalax.util.StringUtil - -@Service -class DefaultEngineNodeManager extends EngineNodeManager with Logging { - - @Autowired - private var engineLocker: EngineNodeLocker = _ - - @Autowired - private var nodeManagerPersistence: NodeManagerPersistence = _ - - @Autowired - private var nodeMetricManagerPersistence: NodeMetricManagerPersistence = _ - - @Autowired - private var metricsConverter: MetricsConverter = _ - - @Autowired - private var nodePointerBuilder: NodePointerBuilder = _ - - @Autowired - private var resourceManager: ResourceManager = _ - - @Autowired - private var labelManagerPersistence: LabelManagerPersistence = _ - - private val labelBuilderFactory = LabelBuilderFactoryContext.getLabelBuilderFactory - - override def listEngines(user: String): util.List[EngineNode] = { - // TODO: user 应该是除了root，hadoop - val nodes = nodeManagerPersistence - .getNodes(user) - .asScala - .map(_.getServiceInstance) - .map(nodeManagerPersistence.getEngineNode) - val metricses = nodeMetricManagerPersistence - .getNodeMetrics(nodes.asJava) - .asScala - .map(m => (m.getServiceInstance.toString, m)) - .toMap - nodes.map { node => - metricses - .get(node.getServiceInstance.toString) - .foreach(metricsConverter.fillMetricsToNode(node, _)) - node - } - }.asJava - - override def getEngineNodeInfo(engineNode: EngineNode): EngineNode = { - - /** - * 修改为实时请求对应的EngineNode - */ - val engine = nodePointerBuilder.buildEngineNodePointer(engineNode) - val heartMsg = engine.getNodeHeartbeatMsg() - engineNode.setNodeHealthyInfo(heartMsg.getHealthyInfo) - engineNode.setNodeOverLoadInfo(heartMsg.getOverLoadInfo) - engineNode.setNodeStatus(heartMsg.getStatus) - engineNode - } - - override def getEngineNodeInfoByDB(engineNode: EngineNode): EngineNode = { - // 1. 从持久化器中获取EngineNode信息，需要获取Task信息和Status信息，方便后面使用 - metricsConverter.fillMetricsToNode( - engineNode, - nodeMetricManagerPersistence.getNodeMetrics(engineNode) - ) - engineNode - } - - override def updateEngineStatus( - serviceInstance: ServiceInstance, - fromState: NodeStatus, - toState: NodeStatus - ): Unit = {} - - override def updateEngine(engineNode: EngineNode): Unit = { - nodeManagerPersistence.updateNodeInstance(engineNode) - } - - override def switchEngine(engineNode: EngineNode): EngineNode = { - null - } - - override def reuseEngine(engineNode: EngineNode): EngineNode = { - val node = getEngineNodeInfo(engineNode) - if (node == null || !NodeStatus.isAvailable(node.getNodeStatus)) { - return null - } - if (!NodeStatus.isLocked(node.getNodeStatus)) { - val lockStr = engineLocker.lockEngine(node, AMConfiguration.ENGINE_LOCKER_MAX_TIME.getValue) - if (lockStr.isEmpty) { - throw new LinkisRetryException( - AMConstant.ENGINE_ERROR_CODE, - s"Failed to request lock from engine by reuse ${node.getServiceInstance}" - ) - } - node.setLock(lockStr.get) - node - } else { - null - } - } - - /** - * TODO use Engine需要考虑流式引擎的场景，后续需要通过Label加额外的处理 - * - * @param engineNode - * @param timeout - * @return - */ - override def useEngine(engineNode: EngineNode, timeout: Long): EngineNode = { - val retryHandler = new RetryHandler {} - retryHandler.addRetryException(classOf[feign.RetryableException]) - retryHandler.addRetryException(classOf[UndeclaredThrowableException]) - val node = retryHandler.retry[EngineNode](getEngineNodeInfo(engineNode), "getEngineNodeInfo") - // val node = getEngineNodeInfo(engineNode) - if (node == null || !NodeStatus.isAvailable(node.getNodeStatus)) { - return null - } - if (!NodeStatus.isLocked(node.getNodeStatus)) { - val lockStr = engineLocker.lockEngine(node, timeout) - if (lockStr.isEmpty) { - throw new LinkisRetryException( - AMConstant.ENGINE_ERROR_CODE, - s"Failed to request lock from engine ${node.getServiceInstance}" - ) - } - node.setLock(lockStr.get) - node - } else { - null - } - } - - override def useEngine(engineNode: EngineNode): EngineNode = { - useEngine(engineNode, AMConfiguration.ENGINE_LOCKER_MAX_TIME.getValue) - } - - /** - * Get detailed engine information from the persistence //TODO 是否增加owner到node - * - * @param scoreServiceInstances - * @return - */ - override def getEngineNodes( - scoreServiceInstances: Array[ScoreServiceInstance] - ): Array[EngineNode] = { - if (null == scoreServiceInstances || scoreServiceInstances.isEmpty) { - return null - } - val engineNodes = scoreServiceInstances.map { scoreServiceInstances => - val engineNode = new AMEngineNode() - engineNode.setScore(scoreServiceInstances.getScore) - engineNode.setServiceInstance(scoreServiceInstances.getServiceInstance) - engineNode - } - // 1. add nodeMetrics 2 add RM info - val resourceInfo = - resourceManager.getResourceInfo(scoreServiceInstances.map(_.getServiceInstance)) - val nodeMetrics = nodeMetricManagerPersistence.getNodeMetrics(engineNodes.toList.asJava) - engineNodes.map { engineNode => - val optionMetrics = - nodeMetrics.asScala.find(_.getServiceInstance.equals(engineNode.getServiceInstance)) - - val optionRMNode = - resourceInfo.resourceInfo.asScala.find( - _.getServiceInstance.equals(engineNode.getServiceInstance) - ) - - optionMetrics.foreach(metricsConverter.fillMetricsToNode(engineNode, _)) - optionRMNode.foreach(rmNode => engineNode.setNodeResource(rmNode.getNodeResource)) - - engineNode - } - } - - /** - * add info to persistence - * - * @param engineNode - */ - override def addEngineNode(engineNode: EngineNode): Unit = { - nodeManagerPersistence.addEngineNode(engineNode) - // init metric - nodeMetricManagerPersistence.addOrupdateNodeMetrics( - metricsConverter.getInitMetric(engineNode.getServiceInstance) - ) - } - - /** - * delete info to persistence - * - * @param engineNode - */ - override def deleteEngineNode(engineNode: EngineNode): Unit = { - nodeManagerPersistence.deleteEngineNode(engineNode) - - } - - override def getEngineNode(serviceInstance: ServiceInstance): EngineNode = { - nodeManagerPersistence.getEngineNode(serviceInstance) - } - - /** - * 1.serviceInstance中取出instance（实际是ticketId） 2.update serviceInstance 表，包括 - * instance替换，替换mark，owner，updator，creator的空值，更新updateTime 3.update engine_em关联表 4.update label - * ticket_id ==> instance - * - * @param serviceInstance - * @param engineNode - */ - override def updateEngineNode(serviceInstance: ServiceInstance, engineNode: EngineNode): Unit = { - nodeManagerPersistence.updateEngineNode(serviceInstance, engineNode) - Utils.tryAndWarnMsg(nodeMetricManagerPersistence.deleteNodeMetrics(engineNode))( - "Failed to clear old metrics" - ) - val engineLabel = labelBuilderFactory.createLabel(classOf[EngineInstanceLabel]) - engineLabel.setInstance(engineNode.getServiceInstance.getInstance) - engineLabel.setServiceName(engineNode.getServiceInstance.getApplicationName) - val oldEngineLabel = labelBuilderFactory.createLabel(classOf[EngineInstanceLabel]) - oldEngineLabel.setInstance(serviceInstance.getInstance) - oldEngineLabel.setServiceName(engineNode.getServiceInstance.getApplicationName) - val oldPersistenceLabel = - labelBuilderFactory.convertLabel(oldEngineLabel, classOf[PersistenceLabel]) - val label = labelManagerPersistence.getLabelByKeyValue( - oldPersistenceLabel.getLabelKey, - oldPersistenceLabel.getStringValue - ) - val persistenceLabel = - labelBuilderFactory.convertLabel(engineLabel, classOf[PersistenceLabel]) - persistenceLabel.setLabelValueSize(persistenceLabel.getValue.size()) - labelManagerPersistence.updateLabel(label.getId, persistenceLabel) - } - - override def executeOperation( - engineNode: EngineNode, - request: EngineOperateRequest - ): EngineOperateResponse = { - val engine = nodePointerBuilder.buildEngineNodePointer(engineNode) - engine.executeOperation(request) - } - - override def getEngineNodeInfo(serviceInstance: ServiceInstance): EngineNode = { - val engineNode = getEngineNode(serviceInstance) - if (engineNode != null) { - val nodeMetric = nodeMetricManagerPersistence.getNodeMetrics(engineNode) - if (engineNode.getNodeStatus == null) { - engineNode.setNodeStatus( - if (Option(nodeMetric).isDefined) NodeStatus.values()(nodeMetric.getStatus) - else NodeStatus.Starting - ) - } - if (null != nodeMetric && StringUtils.isNotBlank(nodeMetric.getHeartBeatMsg)) { - engineNode.setEcMetrics(nodeMetric.getHeartBeatMsg) - } - return engineNode - } - throw new AMErrorException( - AMErrorCode.NOT_EXISTS_ENGINE_CONN.getErrorCode, - AMErrorCode.NOT_EXISTS_ENGINE_CONN.getErrorDesc - ) - } - -} diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/manager/EMNodeManager.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/manager/EMNodeManager.scala deleted file mode 100644 index 404939a6fd..0000000000 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/manager/EMNodeManager.scala +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.manager.am.manager - -import org.apache.linkis.common.ServiceInstance -import org.apache.linkis.manager.common.entity.node.{EMNode, EngineNode, Node, ScoreServiceInstance} -import org.apache.linkis.manager.common.protocol.em.{ECMOperateRequest, ECMOperateResponse} -import org.apache.linkis.manager.common.protocol.engine.EngineStopRequest -import org.apache.linkis.manager.engineplugin.common.launch.entity.EngineConnLaunchRequest - -trait EMNodeManager { - - def emRegister(emNode: EMNode): Unit - - def listEngines(emNode: EMNode): java.util.List[EngineNode] - - def listUserEngines(emNode: EMNode, user: String): java.util.List[EngineNode] - - def listUserNodes(user: String): java.util.List[Node] - - /** - * Get detailed em information from the persistence - * - * @param scoreServiceInstances - * @return - */ - def getEMNodes(scoreServiceInstances: Array[ScoreServiceInstance]): Array[EMNode] - - def getEM(serviceInstance: ServiceInstance): EMNode - - def stopEM(emNode: EMNode): Unit - - def deleteEM(emNode: EMNode): Unit - - def pauseEM(serviceInstance: ServiceInstance): Unit - - /** - * 1. request engineManager to launch engine 2. persist engine info - * - * @param engineConnLaunchRequest - * engine launch request - * @param emNode - * ecm node - * @return - * engine node - */ - def createEngine(engineConnLaunchRequest: EngineConnLaunchRequest, emNode: EMNode): EngineNode - - def stopEngine(engineStopRequest: EngineStopRequest, emNode: EMNode): Unit - - def addEMNodeInstance(emNode: EMNode): Unit - - def initEMNodeMetrics(emNode: EMNode): Unit - - def executeOperation(ecmNode: EMNode, request: ECMOperateRequest): ECMOperateResponse -} diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/manager/EngineNodeManager.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/manager/EngineNodeManager.scala deleted file mode 100644 index f628e1e47f..0000000000 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/manager/EngineNodeManager.scala +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.manager.am.manager - -import org.apache.linkis.common.ServiceInstance -import org.apache.linkis.manager.common.entity.enumeration.NodeStatus -import org.apache.linkis.manager.common.entity.node.{EngineNode, ScoreServiceInstance} -import org.apache.linkis.manager.common.protocol.engine.{ - EngineOperateRequest, - EngineOperateResponse -} - -trait EngineNodeManager { - - def listEngines(user: String): java.util.List[EngineNode] - - def getEngineNode(serviceInstance: ServiceInstance): EngineNode - - def getEngineNodeInfo(engineNode: EngineNode): EngineNode - - def getEngineNodeInfo(serviceInstance: ServiceInstance): EngineNode - - def getEngineNodeInfoByDB(engineNode: EngineNode): EngineNode - - /** - * Get detailed engine information from the persistence - * - * @param scoreServiceInstances - * @return - */ - def getEngineNodes(scoreServiceInstances: Array[ScoreServiceInstance]): Array[EngineNode] - - def updateEngineStatus( - serviceInstance: ServiceInstance, - fromState: NodeStatus, - toState: NodeStatus - ): Unit - - /** - * add info to persistence - * - * @param engineNode - */ - def addEngineNode(engineNode: EngineNode): Unit - - def updateEngineNode(serviceInstance: ServiceInstance, engineNode: EngineNode): Unit - - def updateEngine(engineNode: EngineNode): Unit - - /** - * delete info to persistence - * - * @param engineNode - */ - def deleteEngineNode(engineNode: EngineNode): Unit - - def switchEngine(engineNode: EngineNode): EngineNode - - def reuseEngine(engineNode: EngineNode): EngineNode - - def useEngine(engineNode: EngineNode): EngineNode - - def useEngine(engineNode: EngineNode, timeout: Long): EngineNode - - def executeOperation(engineNode: EngineNode, request: EngineOperateRequest): EngineOperateResponse - -} diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/pointer/AbstractNodePointer.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/pointer/AbstractNodePointer.scala deleted file mode 100644 index 225cfef5d8..0000000000 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/pointer/AbstractNodePointer.scala +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.manager.am.pointer - -import org.apache.linkis.common.exception.WarnException -import org.apache.linkis.common.utils.Logging -import org.apache.linkis.manager.common.entity.enumeration.NodeStatus -import org.apache.linkis.manager.common.protocol.node._ -import org.apache.linkis.manager.label.entity.Label -import org.apache.linkis.manager.service.common.pointer.NodePointer -import org.apache.linkis.rpc.Sender - -abstract class AbstractNodePointer extends NodePointer with Logging { - - protected def getSender: Sender = { - Sender.getSender(getNode().getServiceInstance) - } - - /** - * 向对应的Node发送请求获取节点状态 - * - * @return - */ - override def getNodeStatus(): NodeStatus = { - val sender = getSender - sender.ask(new RequestNodeStatus) match { - case responseStatus: ResponseNodeStatus => - responseStatus.getNodeStatus - case warn: WarnException => throw warn - } - } - - /** - * 向对应的Node发送请求获取节点心跳信息 - * - * @return - */ - override def getNodeHeartbeatMsg(): NodeHeartbeatMsg = { - val sender = getSender - sender.ask(new NodeHeartbeatRequest) match { - case heartbeatMsg: NodeHeartbeatMsg => - heartbeatMsg - case warn: WarnException => throw warn - } - } - - /** - * 向对应的Node发送Kill 请求 - * - * @return - */ - override def stopNode(): Unit = { - val sender = getSender - sender.send(new StopNodeRequest) - } - - /** - * 向对应的Node Label 更新请求 - * - * @return - */ - override def updateLabels(labels: Array[Label[_]]): Unit = {} - - override def updateNodeHealthyRequest(nodeHealthyRequest: NodeHealthyRequest): Unit = { - getSender.send(nodeHealthyRequest) - } - -} diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/pointer/DefaultEMNodPointer.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/pointer/DefaultEMNodPointer.scala deleted file mode 100644 index 8fc82f6c4b..0000000000 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/pointer/DefaultEMNodPointer.scala +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.manager.am.pointer - -import org.apache.linkis.common.exception.LinkisRetryException -import org.apache.linkis.common.utils.Utils -import org.apache.linkis.manager.am.exception.AMErrorException -import org.apache.linkis.manager.am.utils.AMUtils -import org.apache.linkis.manager.common.constant.AMConstant -import org.apache.linkis.manager.common.entity.node.{EngineNode, Node} -import org.apache.linkis.manager.common.protocol.em.{ECMOperateRequest, ECMOperateResponse} -import org.apache.linkis.manager.common.protocol.engine.{EngineStopRequest, EngineStopResponse} -import org.apache.linkis.manager.engineplugin.common.launch.entity.EngineConnLaunchRequest -import org.apache.linkis.manager.service.common.pointer.EMNodPointer - -class DefaultEMNodPointer(val node: Node) extends AbstractNodePointer with EMNodPointer { - - /** - * 与该远程指针关联的node信息 - * - * @return - */ - override def getNode(): Node = node - - override def createEngine(engineConnLaunchRequest: EngineConnLaunchRequest): EngineNode = { - logger.info(s"Start to createEngine ask em ${getNode().getServiceInstance}") - val ec = getSender.ask(engineConnLaunchRequest) - ec match { - case engineNode: EngineNode => - logger.info( - s"Succeed to createEngine ask em ${getNode().getServiceInstance}, engineNode $engineNode " - ) - engineNode - case _ => - throw new LinkisRetryException( - AMConstant.ENGINE_ERROR_CODE, - s"Failed to createEngine ask em ${getNode().getServiceInstance}, for return ${ec}" - ) - } - } - - override def stopEngine(engineStopRequest: EngineStopRequest): Unit = { - Utils.tryAndWarn { - getSender.ask(engineStopRequest) match { - case engineStopResponse: EngineStopResponse => - if (!engineStopResponse.getStopStatus) { - logger.info( - s"Kill engine : ${engineStopRequest.getServiceInstance.toString} failed, because ${engineStopResponse.getMsg} . Will ask engine to suicide." - ) - } else { - logger.info(s"Succeed to kill engine ${engineStopRequest.getServiceInstance.toString}.") - } - case o: AnyRef => - logger.warn( - s"Ask em : ${getNode().getServiceInstance.toString} to kill engine : ${engineStopRequest.getServiceInstance.toString} failed, response is : ${AMUtils.GSON - .toJson(o)}. " - ) - } - } - } - - override def executeOperation(request: ECMOperateRequest): ECMOperateResponse = { - getSender.ask(request) match { - case response: ECMOperateResponse => response - case _ => - throw new AMErrorException(AMConstant.ENGINE_ERROR_CODE, "Failed to execute ECM operation.") - } - } - -} diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/pointer/DefaultEngineNodPointer.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/pointer/DefaultEngineNodPointer.scala deleted file mode 100644 index eda42b6dbf..0000000000 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/pointer/DefaultEngineNodPointer.scala +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.manager.am.pointer - -import org.apache.linkis.common.exception.WarnException -import org.apache.linkis.manager.common.entity.node.Node -import org.apache.linkis.manager.common.protocol.{ - RequestEngineLock, - RequestEngineUnlock, - ResponseEngineLock -} -import org.apache.linkis.manager.common.protocol.engine.{ - EngineOperateRequest, - EngineOperateResponse -} -import org.apache.linkis.manager.service.common.pointer.EngineNodePointer - -class DefaultEngineNodPointer(val node: Node) extends AbstractNodePointer with EngineNodePointer { - - /** - * 与该远程指针关联的node信息 - * - * @return - */ - override def getNode(): Node = node - - override def lockEngine(requestEngineLock: RequestEngineLock): Option[String] = { - getSender.ask(requestEngineLock) match { - case ResponseEngineLock(lockStatus, lock, msg) => - if (lockStatus) { - Some(lock) - } else { - logger.info(s"Failed to get locker,${node.getServiceInstance}: " + msg) - None - } - case _ => None - } - } - - override def releaseLock(requestEngineUnlock: RequestEngineUnlock): Unit = { - getSender.send(requestEngineUnlock) - } - - override def executeOperation( - engineOperateRequest: EngineOperateRequest - ): EngineOperateResponse = { - val rs = getSender.ask(engineOperateRequest) - rs match { - case response: EngineOperateResponse => response - case _ => throw new WarnException(-1, "Illegal response of operation.") - } - } - -} diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/selector/rule/ResourceNodeSelectRule.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/selector/rule/ResourceNodeSelectRule.scala index 32019593f4..28b025731c 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/selector/rule/ResourceNodeSelectRule.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/selector/rule/ResourceNodeSelectRule.scala @@ -55,7 +55,7 @@ class ResourceNodeSelectRule extends NodeSelectRule with Logging { ) { true } else { - node.getNodeResource.getLeftResource > nodeBRM.getNodeResource.getLeftResource + node.getNodeResource.getLeftResource.moreThan(nodeBRM.getNodeResource.getLeftResource) } } { t: Throwable => logger.warn(s"Failed to Compare resource ${t.getMessage}") diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/em/DefaultEMInfoService.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/em/DefaultEMInfoService.scala index 4f0f5f6aa5..b82e32cd18 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/em/DefaultEMInfoService.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/em/DefaultEMInfoService.scala @@ -21,7 +21,9 @@ import org.apache.linkis.common.ServiceInstance import org.apache.linkis.common.utils.{Logging, Utils} import org.apache.linkis.governance.common.conf.GovernanceCommonConf import org.apache.linkis.manager.am.conf.AMConfiguration +import org.apache.linkis.manager.am.converter.MetricsConverter import org.apache.linkis.manager.am.manager.EMNodeManager +import org.apache.linkis.manager.am.pointer.NodePointerBuilder import org.apache.linkis.manager.am.service.engine.EngineInfoService import org.apache.linkis.manager.common.entity.enumeration.NodeHealthy import org.apache.linkis.manager.common.entity.metrics.NodeHealthyInfo @@ -40,8 +42,6 @@ import org.apache.linkis.manager.persistence.{ ResourceManagerPersistence } import org.apache.linkis.manager.rm.service.ResourceManager -import org.apache.linkis.manager.service.common.metrics.MetricsConverter -import org.apache.linkis.manager.service.common.pointer.NodePointerBuilder import org.apache.linkis.rpc.message.annotation.Receiver import org.apache.linkis.server.toScalaBuffer @@ -99,7 +99,7 @@ class DefaultEMInfoService extends EMInfoService with Logging { val label = new AliasServiceInstanceLabel label.setAlias(GovernanceCommonConf.ENGINE_CONN_MANAGER_SPRING_NAME.getValue) val instances = nodeLabelService.getNodesByLabel(label) - val resourceInfo = resourceManager.getResourceInfo(instances.asScala.toSeq.toArray).resourceInfo + val resourceInfo = resourceManager.getResourceInfo(instances.asScala.toArray).resourceInfo val resourceInfoMap = resourceInfo.asScala.map(r => (r.getServiceInstance.toString, r)).toMap instances.asScala .map(emNodeManager.getEM) @@ -179,18 +179,19 @@ class DefaultEMInfoService extends EMInfoService with Logging { val ecmNodeResource = ecmInstance.getNodeResource // 资源对比，资源重置 if ( - (!(useResource == ecmNodeResource.getUsedResource)) || (!(lockResource == ecmNodeResource.getLockedResource)) + (!(useResource.equalsTo(ecmNodeResource.getUsedResource)) || (!(lockResource + .equalsTo(ecmNodeResource.getLockedResource)))) ) { logger.info( MessageFormat.format( "ECM:{0} resources will be reset, Record Resources:{1} ,Real Resources:{2}", ecmInstance.getServiceInstance.getInstance, - ecmNodeResource.getUsedResource + ecmNodeResource.getLockedResource, + ecmNodeResource.getUsedResource.add(ecmNodeResource.getLockedResource), realSumResource ) ) ecmNodeResource.setLockedResource(lockResource) - ecmNodeResource.setLeftResource(ecmNodeResource.getMaxResource - realSumResource) + ecmNodeResource.setLeftResource(ecmNodeResource.getMaxResource.minus(realSumResource)) ecmNodeResource.setUsedResource(useResource) val persistence = ResourceUtils.toPersistenceResource(ecmNodeResource) val resourceLabel = labelManagerPersistence.getLabelByResource(persistence) @@ -253,18 +254,20 @@ class DefaultEMInfoService extends EMInfoService with Logging { val (sumResource, uedResource, lockResource) = collectResource(userEngineNodeFilter, userResourceType) if ( - (!(uedResource == userPersistenceResource.getUsedResource)) || (!(lockResource == userPersistenceResource.getLockedResource)) + (!(uedResource.equalsTo(userPersistenceResource.getUsedResource)) || (!(lockResource + .equalsTo(userPersistenceResource.getLockedResource)))) ) { logger.info( MessageFormat.format( "LabelUser:{0} resources will be reset, Record Resources:{1} ,Real Resources:{2}", labelUser, - userPersistenceResource.getUsedResource + userPersistenceResource.getLockedResource, + userPersistenceResource.getUsedResource + .add(userPersistenceResource.getLockedResource), sumResource ) ) userPersistenceResource.setLeftResource( - userPersistenceResource.getMaxResource - sumResource + userPersistenceResource.getMaxResource.minus(sumResource) ) userPersistenceResource.setUsedResource(uedResource) userPersistenceResource.setLockedResource(lockResource) @@ -297,9 +300,9 @@ class DefaultEMInfoService extends EMInfoService with Logging { ) ) { case ((accSum, accUed, accLock), nodeResource) => ( - accSum + nodeResource.getUsedResource + nodeResource.getLockedResource, - accUed + nodeResource.getUsedResource, - accLock + nodeResource.getLockedResource + accSum.add(nodeResource.getUsedResource.add(nodeResource.getLockedResource)), + accUed.add(nodeResource.getUsedResource), + accLock.add(nodeResource.getLockedResource) ) } } diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/DefaultEngineAskEngineService.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/DefaultEngineAskEngineService.scala index f2a4f83f34..9240ec0dd4 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/DefaultEngineAskEngineService.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/DefaultEngineAskEngineService.scala @@ -294,7 +294,7 @@ class DefaultEngineAskEngineService ) val chooseLabels = multiUserEngineReuseLabelChooser.chooseLabels(labelList) val userCreatorAndEngineTypeLabel = - new RMLabelContainer(chooseLabels).getCombinedUserCreatorEngineTypeLabel + new RMLabelContainer(chooseLabels).getCombinedResourceLabel val engineCreateKey = if (null != userCreatorAndEngineTypeLabel) { userCreatorAndEngineTypeLabel.getStringValue + LabelUtil.getTenantValue(chooseLabels) } else { @@ -303,7 +303,7 @@ class DefaultEngineAskEngineService val engineType = LabelUtil.getEngineType(chooseLabels) val semaphore = if ( - AMConfiguration.AM_ENGINE_ASK_MAX_NUMBER.contains( + AMConfiguration.AM_ENGINE_ASK_MAX_NUMBER.containsKey( engineType.toLowerCase(Locale.getDefault) ) && StringUtils.isNotBlank(engineCreateKey) ) { @@ -314,7 +314,7 @@ class DefaultEngineAskEngineService engineCreateKey, new Semaphore( AMConfiguration.AM_ENGINE_ASK_MAX_NUMBER - .getOrElse(engineType.toLowerCase(Locale.getDefault), 10) + .getOrDefault(engineType.toLowerCase(Locale.getDefault), 10) ) ) } diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/DefaultEngineCreateService.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/DefaultEngineCreateService.scala index 123c4da874..52c225cf8b 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/DefaultEngineCreateService.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/DefaultEngineCreateService.scala @@ -381,10 +381,18 @@ class DefaultEngineCreateService private def ensuresIdle(engineNode: EngineNode, resourceTicketId: String): Boolean = { - val engineNodeInfo = Utils.tryAndWarnMsg( - getEngineNodeManager.getEngineNodeInfoByDB(engineNode) - )("Failed to from db get engine node info") + val engineNodeInfo = + Utils.tryAndWarnMsg(if (engineNode.getMark == AMConstant.CLUSTER_PROCESS_MARK) { + getEngineNodeManager.getEngineNodeInfoByTicketId(resourceTicketId) + } else { + getEngineNodeManager.getEngineNodeInfoByDB(engineNode) + })("Failed to from db get engine node info") + if (null == engineNodeInfo) return false + + if (engineNodeInfo.getServiceInstance != null) { + engineNode.setServiceInstance(engineNodeInfo.getServiceInstance) + } if (NodeStatus.isCompleted(engineNodeInfo.getNodeStatus)) { val metrics = nodeMetricManagerPersistence.getNodeMetrics(engineNodeInfo) val msg = if (metrics != null) metrics.getHeartBeatMsg else null diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/DefaultEngineInfoService.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/DefaultEngineInfoService.scala index 2d5b6aa6e1..cd40e8ebd2 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/DefaultEngineInfoService.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/DefaultEngineInfoService.scala @@ -26,13 +26,13 @@ import org.apache.linkis.governance.common.protocol.engineconn.{ } import org.apache.linkis.governance.common.utils.GovernanceConstant import org.apache.linkis.manager.am.manager.{EMNodeManager, EngineNodeManager} +import org.apache.linkis.manager.am.pointer.NodePointerBuilder import org.apache.linkis.manager.common.entity.enumeration.NodeHealthy import org.apache.linkis.manager.common.entity.node.{EMNode, EngineNode} import org.apache.linkis.manager.common.protocol.node.NodeHealthyRequest import org.apache.linkis.manager.label.builder.factory.LabelBuilderFactoryContext import org.apache.linkis.manager.label.service.NodeLabelService import org.apache.linkis.manager.rm.service.ResourceManager -import org.apache.linkis.manager.service.common.pointer.NodePointerBuilder import org.apache.linkis.rpc.message.annotation.Receiver import org.springframework.beans.factory.annotation.Autowired diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/DefaultEngineReuseService.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/DefaultEngineReuseService.scala index 9436e2e706..cbc7c06eb7 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/DefaultEngineReuseService.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/DefaultEngineReuseService.scala @@ -179,10 +179,10 @@ class DefaultEngineReuseService extends AbstractEngineService with EngineReuseSe .filter(engine => { if (engine.getNodeResource.getUsedResource != null) { // 引擎资源只有满足需要的资源才复用 - engine.getNodeResource.getUsedResource >= resource.getMaxResource + engine.getNodeResource.getUsedResource.notLess(resource.getMaxResource) } else { // 引擎正在启动中，比较锁住的资源，最终是否复用沿用之前复用逻辑 - engine.getNodeResource.getLockedResource >= resource.getMaxResource + engine.getNodeResource.getLockedResource.notLess(resource.getMaxResource) } }) } @@ -201,8 +201,8 @@ class DefaultEngineReuseService extends AbstractEngineService with EngineReuseSe if (engineReuseRequest.getTimeOut <= 0) { AMConfiguration.ENGINE_REUSE_MAX_TIME.getValue.toLong } else engineReuseRequest.getTimeOut - val reuseLimit = - if (engineReuseRequest.getReuseCount <= 0) AMConfiguration.ENGINE_REUSE_COUNT_LIMIT.getValue + val reuseLimit: Int = + if (engineReuseRequest.getReuseCount <= 0) AMConfiguration.ENGINE_REUSE_COUNT_LIMIT else engineReuseRequest.getReuseCount def selectEngineToReuse: Boolean = { diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/DefaultEngineStopService.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/DefaultEngineStopService.scala index dc039786f4..3c6d7eaffd 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/DefaultEngineStopService.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/DefaultEngineStopService.scala @@ -153,8 +153,8 @@ class DefaultEngineStopService extends AbstractEngineService with EngineStopServ } val engineTypeLabel = - node.getLabels.asScala.find(_.isInstanceOf[EngineTypeLabel]).getOrElse(null) - val engineTypeStr = engineTypeLabel.asInstanceOf[EngineTypeLabel] getEngineType + node.getLabels.asScala.find(_.isInstanceOf[EngineTypeLabel]).orNull + val engineTypeStr = engineTypeLabel.asInstanceOf[EngineTypeLabel].getEngineType val isAllowKill = AMConfiguration.isAllowKilledEngineType(engineTypeStr) if (isAllowKill == false) { @@ -165,7 +165,7 @@ class DefaultEngineStopService extends AbstractEngineService with EngineStopServ // calculate the resources that can be released if (node.getNodeResource.getUsedResource != null) { val realResource = node.getNodeResource.getUsedResource match { - case dy: DriverAndYarnResource => dy.loadInstanceResource + case dy: DriverAndYarnResource => dy.getLoadInstanceResource case _ => node.getNodeResource.getUsedResource } loadInstanceResourceTotal = @@ -180,8 +180,8 @@ class DefaultEngineStopService extends AbstractEngineService with EngineStopServ } } resultMap.put("killEngineNum", killEngineNum) - resultMap.put("memory", loadInstanceResourceTotal.memory) - resultMap.put("cores", loadInstanceResourceTotal.cores) + resultMap.put("memory", loadInstanceResourceTotal.getMemory) + resultMap.put("cores", loadInstanceResourceTotal.getCores) resultMap.put("batchKillEngineType", AMConfiguration.ALLOW_BATCH_KILL_ENGINE_TYPES.getValue) resultMap } diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/heartbeat/AMHeartbeatService.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/heartbeat/AMHeartbeatService.scala index c951e01052..7630757452 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/heartbeat/AMHeartbeatService.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/heartbeat/AMHeartbeatService.scala @@ -18,13 +18,13 @@ package org.apache.linkis.manager.am.service.heartbeat import org.apache.linkis.common.utils.{Logging, Utils} +import org.apache.linkis.manager.am.converter.MetricsConverter import org.apache.linkis.manager.am.service.HeartbeatService import org.apache.linkis.manager.common.conf.RMConfiguration import org.apache.linkis.manager.common.entity.metrics.AMNodeMetrics import org.apache.linkis.manager.common.monitor.ManagerMonitor import org.apache.linkis.manager.common.protocol.node.NodeHeartbeatMsg import org.apache.linkis.manager.persistence.{NodeManagerPersistence, NodeMetricManagerPersistence} -import org.apache.linkis.manager.service.common.metrics.MetricsConverter import org.apache.linkis.publicservice.common.lock.entity.CommonLock import org.apache.linkis.publicservice.common.lock.service.CommonLockService import org.apache.linkis.rpc.message.annotation.Receiver diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/monitor/NodeHeartbeatMonitor.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/monitor/NodeHeartbeatMonitor.scala index 3887b1a960..059cdbae58 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/monitor/NodeHeartbeatMonitor.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/monitor/NodeHeartbeatMonitor.scala @@ -21,6 +21,7 @@ import org.apache.linkis.common.ServiceInstance import org.apache.linkis.common.utils.{Logging, Utils} import org.apache.linkis.governance.common.conf.GovernanceCommonConf import org.apache.linkis.manager.am.conf.ManagerMonitorConf +import org.apache.linkis.manager.am.converter.MetricsConverter import org.apache.linkis.manager.am.service.em.EMUnregisterService import org.apache.linkis.manager.am.service.engine.EngineStopService import org.apache.linkis.manager.am.service.heartbeat.AMHeartbeatService @@ -34,7 +35,6 @@ import org.apache.linkis.manager.common.protocol.node.{NodeHeartbeatMsg, NodeHea import org.apache.linkis.manager.common.utils.ManagerUtils import org.apache.linkis.manager.persistence.{NodeManagerPersistence, NodeMetricManagerPersistence} import org.apache.linkis.manager.service.common.label.ManagerLabelService -import org.apache.linkis.manager.service.common.metrics.MetricsConverter import org.apache.linkis.rpc.Sender import org.apache.linkis.rpc.exception.NoInstanceExistsException diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/utils/AMUtils.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/utils/AMUtils.scala index 454c8211a5..428e5d23e9 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/utils/AMUtils.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/utils/AMUtils.scala @@ -23,10 +23,8 @@ import org.apache.linkis.manager.common.entity.node.{EMNode, EngineNode} import org.apache.linkis.manager.common.entity.resource.{ DriverAndYarnResource, Resource, - ResourceSerializer, ResourceType } -import org.apache.linkis.manager.common.serializer.NodeResourceSerializer import org.apache.linkis.manager.label.entity.engine.EngineTypeLabel import org.apache.linkis.server.BDPJettyServerHelper @@ -201,7 +199,7 @@ object AMUtils { } if (node.getNodeResource.getUsedResource != null) { val realResource = node.getNodeResource.getUsedResource match { - case dy: DriverAndYarnResource => dy.loadInstanceResource + case dy: DriverAndYarnResource => dy.getLoadInstanceResource case _ => node.getNodeResource.getUsedResource } AMEngineNodeVo.setUsedResource( diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/label/service/NodeLabelService.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/label/service/NodeLabelService.scala index f636f894e0..a4893a38bf 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/label/service/NodeLabelService.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/label/service/NodeLabelService.scala @@ -48,6 +48,11 @@ trait NodeLabelService { def updateLabelsToNode(instance: ServiceInstance, label: util.List[Label[_]]): Unit + def labelsFromInstanceToNewInstance( + oldServiceInstance: ServiceInstance, + newServiceInstance: ServiceInstance + ): Unit + /** * Remove the labels related by node instance * diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/label/service/impl/DefaultNodeLabelService.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/label/service/impl/DefaultNodeLabelService.scala index f143f91607..b9be35f7ca 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/label/service/impl/DefaultNodeLabelService.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/label/service/impl/DefaultNodeLabelService.scala @@ -19,14 +19,15 @@ package org.apache.linkis.manager.label.service.impl import org.apache.linkis.common.ServiceInstance import org.apache.linkis.common.utils.{Logging, Utils} +import org.apache.linkis.manager.am.conf.AMConfiguration +import org.apache.linkis.manager.am.converter.MetricsConverter import org.apache.linkis.manager.common.entity.node.{EngineNode, ScoreServiceInstance} import org.apache.linkis.manager.common.entity.persistence.PersistenceLabel import org.apache.linkis.manager.common.entity.resource.Resource import org.apache.linkis.manager.common.utils.ManagerUtils import org.apache.linkis.manager.label.LabelManagerUtils import org.apache.linkis.manager.label.builder.factory.LabelBuilderFactoryContext -import org.apache.linkis.manager.label.conf.LabelManagerConf -import org.apache.linkis.manager.label.entity.{Feature, Label} +import org.apache.linkis.manager.label.entity.{Feature, InheritableLabel, Label} import org.apache.linkis.manager.label.entity.engine.{ EngineInstanceLabel, EngineTypeLabel, @@ -42,7 +43,6 @@ import org.apache.linkis.manager.persistence.{ } import org.apache.linkis.manager.rm.service.LabelResourceService import org.apache.linkis.manager.rm.utils.{RMUtils, UserConfiguration} -import org.apache.linkis.manager.service.common.metrics.MetricsConverter import org.apache.linkis.server.toScalaBuffer import org.springframework.beans.factory.annotation.Autowired @@ -51,7 +51,7 @@ import org.springframework.transaction.annotation.Transactional import org.springframework.util.CollectionUtils import java.util -import java.util.List +import java.util.{ArrayList, List} import java.util.stream.Collectors import scala.collection.JavaConverters._ @@ -201,6 +201,42 @@ class DefaultNodeLabelService extends NodeLabelService with Logging { } } + override def labelsFromInstanceToNewInstance( + oldServiceInstance: ServiceInstance, + newServiceInstance: ServiceInstance + ): Unit = { + val labels = labelManagerPersistence.getLabelByServiceInstance(newServiceInstance) + val newKeyList = if (null != labels) { + labels.map(_.getLabelKey).asJava + } else { + new util.ArrayList[String]() + } + val nodeLabels = labelManagerPersistence.getLabelByServiceInstance(oldServiceInstance) + if (null == nodeLabels) { + return + } + val oldKeyList = nodeLabels.map(_.getLabelKey).asJava + oldKeyList.removeAll(newKeyList) + // Assign the old association to the newServiceInstance + if (!CollectionUtils.isEmpty(oldKeyList)) { + nodeLabels.foreach(nodeLabel => { + if (oldKeyList.contains(nodeLabel.getLabelKey)) { + val persistenceLabel = LabelManagerUtils.convertPersistenceLabel(nodeLabel) + val labelId = tryToAddLabel(persistenceLabel) + if (labelId > 0) { + val labelIds = new util.ArrayList[Integer] + labelIds.add(labelId) + labelManagerPersistence.addLabelToNode(newServiceInstance, labelIds) + } + } + + }) + } + // Delete an old association + val oldLabelId = nodeLabels.map(_.getId).asJava + labelManagerPersistence.removeNodeLabels(oldServiceInstance, oldLabelId) + } + /** * Remove the labels related by node instance * @@ -234,13 +270,13 @@ class DefaultNodeLabelService extends NodeLabelService with Logging { val removeLabels = if (isEngine) { labels } else { - labels.filter(label => !LabelManagerConf.LONG_LIVED_LABEL.contains(label.getLabelKey)) + labels.filter(label => !AMConfiguration.LONG_LIVED_LABEL.contains(label.getLabelKey)) } labelManagerPersistence.removeNodeLabels(instance, removeLabels.map(_.getId).asJava) // remove taskId label labels.foreach(label => { - if (LabelManagerConf.TMP_LIVED_LABEL.contains(label.getLabelKey)) { + if (AMConfiguration.TMP_LIVED_LABEL.contains(label.getLabelKey)) { labelManagerPersistence.removeLabel(label) } }) diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/message/RMMessageService.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/message/RMMessageService.scala index 811ae2cadf..1dcabe61b2 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/message/RMMessageService.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/message/RMMessageService.scala @@ -44,7 +44,7 @@ class RMMessageService extends Logging { Utils.tryCatch(resourceManager.resourceUsed(labels, resourceUsedProtocol.engineResource)) { case exception: Exception => val nodeLabels = new RMLabelContainer(labels) - val value: String = Option(nodeLabels.getCombinedUserCreatorEngineTypeLabel) + val value: String = Option(nodeLabels.getCombinedResourceLabel) .map(_.getStringValue) .getOrElse("") logger.warn( diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/restful/RMMonitorRest.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/restful/RMMonitorRest.scala index 24c03ab323..8b47daf87b 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/restful/RMMonitorRest.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/restful/RMMonitorRest.scala @@ -24,6 +24,7 @@ import org.apache.linkis.governance.common.protocol.conf.{ AcrossClusterResponse } import org.apache.linkis.manager.am.conf.{AMConfiguration, ManagerMonitorConf} +import org.apache.linkis.manager.am.converter.MetricsConverter import org.apache.linkis.manager.common.conf.RMConfiguration import org.apache.linkis.manager.common.entity.enumeration.NodeStatus import org.apache.linkis.manager.common.entity.node.EngineNode @@ -56,7 +57,6 @@ import org.apache.linkis.manager.rm.restful.vo.{UserCreatorEngineType, UserResou import org.apache.linkis.manager.rm.service.{LabelResourceService, ResourceManager} import org.apache.linkis.manager.rm.service.impl.UserResourceService import org.apache.linkis.manager.rm.utils.{RMUtils, UserConfiguration} -import org.apache.linkis.manager.service.common.metrics.MetricsConverter import org.apache.linkis.rpc.Sender import org.apache.linkis.server.{toScalaBuffer, BDPJettyServerHelper, Message} import org.apache.linkis.server.security.SecurityFilter @@ -393,16 +393,17 @@ class RMMonitorRest extends Logging { queueInfo.put("queuename", maxResource) queueInfo.put( "maxResources", - Map("memory" -> maxResource.queueMemory, "cores" -> maxResource.queueCores) + Map("memory" -> maxResource.getQueueName, "cores" -> maxResource.getQueueCores) ) queueInfo.put( "usedResources", - Map("memory" -> usedResource.queueMemory, "cores" -> usedResource.queueCores) + Map("memory" -> usedResource.getQueueMemory, "cores" -> usedResource.getQueueCores) ) - usedMemoryPercentage = usedResource.queueMemory - .asInstanceOf[Double] / maxResource.queueMemory.asInstanceOf[Double] - usedCPUPercentage = usedResource.queueCores.asInstanceOf[Double] / maxResource.queueCores - .asInstanceOf[Double] + usedMemoryPercentage = usedResource.getQueueMemory + .asInstanceOf[Double] / maxResource.getQueueMemory.asInstanceOf[Double] + usedCPUPercentage = + usedResource.getQueueCores.asInstanceOf[Double] / maxResource.getQueueCores + .asInstanceOf[Double] queueInfo.put( "usedPercentage", Map("memory" -> usedMemoryPercentage, "cores" -> usedCPUPercentage) @@ -431,10 +432,11 @@ class RMMonitorRest extends Logging { if (node.getNodeResource != null && node.getNodeResource.getUsedResource != null) { node.getNodeResource.getUsedResource match { case driverYarn: DriverAndYarnResource - if driverYarn.yarnResource.queueName.equals(yarnIdentifier.getQueueName) => - appIdToEngineNode.put(driverYarn.yarnResource.applicationId, node) - case yarn: YarnResource if yarn.queueName.equals(yarnIdentifier.getQueueName) => - appIdToEngineNode.put(yarn.applicationId, node) + if driverYarn.getYarnResource.getQueueName + .equals(yarnIdentifier.getQueueName) => + appIdToEngineNode.put(driverYarn.getYarnResource.getApplicationId, node) + case yarn: YarnResource if yarn.getQueueName.equals(yarnIdentifier.getQueueName) => + appIdToEngineNode.put(yarn.getApplicationId, node) case _ => } } @@ -454,40 +456,40 @@ class RMMonitorRest extends Logging { } val totalResource = busyResource.add(idleResource) - if (totalResource > Resource.getZeroResource(totalResource)) { + if (totalResource.moreThan(Resource.getZeroResource(totalResource))) { val userResource = new mutable.HashMap[String, Any]() userResource.put("username", userAppInfo._1) val queueResource = providedYarnResource.getMaxResource.asInstanceOf[YarnResource] if (usedMemoryPercentage > usedCPUPercentage) { userResource.put( "busyPercentage", - busyResource.queueMemory.asInstanceOf[Double] / queueResource.queueMemory + busyResource.getQueueMemory.asInstanceOf[Double] / queueResource.getQueueMemory .asInstanceOf[Double] ) userResource.put( "idlePercentage", - idleResource.queueMemory.asInstanceOf[Double] / queueResource.queueMemory + idleResource.getQueueMemory.asInstanceOf[Double] / queueResource.getQueueMemory .asInstanceOf[Double] ) userResource.put( "totalPercentage", - totalResource.queueMemory.asInstanceOf[Double] / queueResource.queueMemory + totalResource.getQueueMemory.asInstanceOf[Double] / queueResource.getQueueMemory .asInstanceOf[Double] ) } else { userResource.put( "busyPercentage", - busyResource.queueCores.asInstanceOf[Double] / queueResource.queueCores + busyResource.getQueueCores.asInstanceOf[Double] / queueResource.getQueueCores .asInstanceOf[Double] ) userResource.put( "idlePercentage", - idleResource.queueCores.asInstanceOf[Double] / queueResource.queueCores + idleResource.getQueueCores.asInstanceOf[Double] / queueResource.getQueueCores .asInstanceOf[Double] ) userResource.put( "totalPercentage", - totalResource.queueCores.asInstanceOf[Double] / queueResource.queueCores + totalResource.getQueueCores.asInstanceOf[Double] / queueResource.getQueueCores .asInstanceOf[Double] ) } @@ -627,21 +629,21 @@ class RMMonitorRest extends Logging { val usedResource = engineResource.getUsedResource.asInstanceOf[LoadInstanceResource] val lockedResource = engineResource.getLockedResource.asInstanceOf[LoadInstanceResource] val maxResource = engineResource.getMaxResource.asInstanceOf[LoadInstanceResource] - val usedMemory = usedResource.memory - val usedCores = usedResource.cores - val usedInstances = usedResource.instances + val usedMemory = usedResource.getMemory + val usedCores = usedResource.getCores + val usedInstances = usedResource.getInstances totalUsedMemory += usedMemory totalUsedCores += usedCores totalUsedInstances += usedInstances - val lockedMemory = lockedResource.memory - val lockedCores = lockedResource.cores - val lockedInstances = lockedResource.instances + val lockedMemory = lockedResource.getMemory + val lockedCores = lockedResource.getCores + val lockedInstances = lockedResource.getInstances totalLockedMemory += lockedMemory totalLockedCores += lockedCores totalLockedInstances += lockedInstances - val maxMemory = maxResource.memory - val maxCores = maxResource.cores - val maxInstances = maxResource.instances + val maxMemory = maxResource.getMemory + val maxCores = maxResource.getCores + val maxInstances = maxResource.getInstances totalMaxMemory += maxMemory totalMaxCores += maxCores totalMaxInstances += maxInstances @@ -699,14 +701,16 @@ class RMMonitorRest extends Logging { val nodeResource = CommonNodeResource.initNodeResource(ResourceType.LoadInstance) engineTypeResourceMap.put(engineType, nodeResource) } - val resource = engineTypeResourceMap.get(engineType).get - resource.setUsedResource(node.getNodeResource.getUsedResource + resource.getUsedResource) + val resource = engineTypeResourceMap(engineType) + resource.setUsedResource(node.getNodeResource.getUsedResource.add(resource.getUsedResource)) // combined label val combinedLabel = combinedLabelBuilder.build("", Lists.newArrayList(userCreatorLabel, engineTypeLabel)); var labelResource = labelResourceService.getLabelResource(combinedLabel) if (labelResource == null) { - resource.setLeftResource(node.getNodeResource.getMaxResource - resource.getUsedResource) + resource.setLeftResource( + node.getNodeResource.getMaxResource.minus(resource.getUsedResource) + ) } else { labelResource = ResourceUtils.convertTo(labelResource, ResourceType.LoadInstance) resource.setUsedResource(labelResource.getUsedResource) @@ -716,7 +720,7 @@ class RMMonitorRest extends Logging { } resource.getLeftResource match { case dResource: DriverAndYarnResource => - resource.setLeftResource(dResource.loadInstanceResource) + resource.setLeftResource(dResource.getLoadInstanceResource) case _ => } } @@ -759,7 +763,7 @@ class RMMonitorRest extends Logging { applicationList .get("usedResource") .asInstanceOf[Resource] - }) + node.getNodeResource.getUsedResource + }).add(node.getNodeResource.getUsedResource) ) applicationList.put( "maxResource", @@ -769,7 +773,7 @@ class RMMonitorRest extends Logging { applicationList .get("maxResource") .asInstanceOf[Resource] - }) + node.getNodeResource.getMaxResource + }).add(node.getNodeResource.getMaxResource) ) applicationList.put( "minResource", @@ -779,7 +783,7 @@ class RMMonitorRest extends Logging { applicationList .get("minResource") .asInstanceOf[Resource] - }) + node.getNodeResource.getMinResource + }).add(node.getNodeResource.getMinResource) ) applicationList.put( "lockedResource", @@ -789,7 +793,7 @@ class RMMonitorRest extends Logging { applicationList .get("lockedResource") .asInstanceOf[Resource] - }) + node.getNodeResource.getLockedResource + }).add(node.getNodeResource.getLockedResource) ) val engineInstance = new mutable.HashMap[String, Any] engineInstance.put("creator", userCreatorLabel.getCreator) diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/service/RequestResourceService.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/service/RequestResourceService.scala index 1e3f8c1c37..d67864d2c9 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/service/RequestResourceService.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/service/RequestResourceService.scala @@ -71,7 +71,7 @@ abstract class RequestResourceService(labelResourceService: LabelResourceService if (ecmResource != null) { val labelAvailableResource = ecmResource.getLeftResource canCreateECRes.setEcmResource(RMUtils.serializeResource(labelAvailableResource)) - if (labelAvailableResource < requestResource) { + if (!labelAvailableResource.notLess(requestResource)) { logger.info( s"user want to use resource[${requestResource}] > em ${emInstanceLabel.getInstance()} available resource[${labelAvailableResource}]" ) @@ -85,19 +85,19 @@ abstract class RequestResourceService(labelResourceService: LabelResourceService } } // get CombinedLabel Resource Usage - labelContainer.setCurrentLabel(labelContainer.getCombinedUserCreatorEngineTypeLabel) + labelContainer.setCurrentLabel(labelContainer.getCombinedResourceLabel) val labelResource = getCombinedLabelResourceUsage(labelContainer, resource) labelResourceService.setLabelResource( labelContainer.getCurrentLabel, labelResource, - labelContainer.getCombinedUserCreatorEngineTypeLabel.getStringValue + labelContainer.getCombinedResourceLabel.getStringValue ) if (labelResource != null) { val labelAvailableResource = labelResource.getLeftResource canCreateECRes.setLabelResource(RMUtils.serializeResource(labelAvailableResource)) val labelMaxResource = labelResource.getMaxResource - if (labelAvailableResource < requestResource) { + if (!labelAvailableResource.notLess(requestResource)) { logger.info( s"Failed check: ${labelContainer.getUserCreatorLabel.getUser} want to use label [${labelContainer.getCurrentLabel}] resource[${requestResource}] > " + s"label available resource[${labelAvailableResource}]" @@ -136,7 +136,9 @@ abstract class RequestResourceService(labelResourceService: LabelResourceService labelResource.setMaxResource(configuredResource) labelResource.setMinResource(Resource.initResource(labelResource.getResourceType)) labelResource.setLeftResource( - labelResource.getMaxResource - labelResource.getUsedResource - labelResource.getLockedResource + labelResource.getMaxResource + .minus(labelResource.getUsedResource) + .minus(labelResource.getLockedResource) ) logger.debug( s"${labelContainer.getCurrentLabel} ecmResource: Max: ${labelResource.getMaxResource} \t " + @@ -161,9 +163,7 @@ abstract class RequestResourceService(labelResourceService: LabelResourceService case _ => } // check combined label resource - if ( - !labelContainer.getCombinedUserCreatorEngineTypeLabel.equals(labelContainer.getCurrentLabel) - ) { + if (!labelContainer.getCombinedResourceLabel.equals(labelContainer.getCurrentLabel)) { throw new RMErrorException( RESOURCE_LATER_ERROR.getErrorCode, RESOURCE_LATER_ERROR.getErrorDesc + labelContainer.getCurrentLabel @@ -212,13 +212,13 @@ abstract class RequestResourceService(labelResourceService: LabelResourceService labelResourceService.setLabelResource( labelContainer.getCurrentLabel, labelResource, - labelContainer.getCombinedUserCreatorEngineTypeLabel.getStringValue + labelContainer.getCombinedResourceLabel.getStringValue ) logger.debug(s"Label [${labelContainer.getCurrentLabel}] has resource + [${labelResource}]") if (labelResource != null) { val labelAvailableResource = labelResource.getLeftResource val labelMaxResource = labelResource.getMaxResource - if (labelAvailableResource < requestResource) { + if (!labelAvailableResource.notLess(requestResource)) { logger.info( s"Failed check: ${labelContainer.getUserCreatorLabel.getUser} want to use label [${labelContainer.getCurrentLabel}] resource[${requestResource}] > " + s"label available resource[${labelAvailableResource}]" @@ -247,7 +247,7 @@ abstract class RequestResourceService(labelResourceService: LabelResourceService logger.debug(s"emInstanceLabel resource info ${labelResource}") if (labelResource != null) { val labelAvailableResource = labelResource.getLeftResource - if (labelAvailableResource < requestResource) { + if (!labelAvailableResource.notLess(requestResource)) { logger.info( s"user want to use resource[${requestResource}] > em ${emInstanceLabel.getInstance()} available resource[${labelAvailableResource}]" ) @@ -281,35 +281,35 @@ abstract class RequestResourceService(labelResourceService: LabelResourceService val loadRequestResource = requestResource match { case li: LoadInstanceResource => li case driverAndYarnResource: DriverAndYarnResource => - driverAndYarnResource.loadInstanceResource + driverAndYarnResource.getLoadInstanceResource case _ => null } loadRequestResource match { case li: LoadInstanceResource => val loadInstanceAvailable = availableResource.asInstanceOf[LoadInstanceResource] val loadInstanceMax = maxResource.asInstanceOf[LoadInstanceResource] - if (li.cores > loadInstanceAvailable.cores) { + if (li.getCores > loadInstanceAvailable.getCores) { ( RMErrorCode.ECM_CPU_INSUFFICIENT.getErrorCode, RMErrorCode.ECM_CPU_INSUFFICIENT.getErrorDesc + RMUtils.getResourceInfoMsg( RMConstant.CPU, RMConstant.CPU_UNIT, - li.cores, - loadInstanceAvailable.cores, - loadInstanceMax.cores + li.getCores, + loadInstanceAvailable.getCores, + loadInstanceMax.getCores ) ) - } else if (li.memory > loadInstanceAvailable.memory) { + } else if (li.getMemory > loadInstanceAvailable.getMemory) { ( RMErrorCode.ECM_MEMORY_INSUFFICIENT.getErrorCode, RMErrorCode.ECM_MEMORY_INSUFFICIENT.getErrorDesc + RMUtils.getResourceInfoMsg( RMConstant.MEMORY, RMConstant.MEMORY_UNIT_BYTE, - li.memory, - loadInstanceAvailable.memory, - loadInstanceMax.memory + li.getMemory, + loadInstanceAvailable.getMemory, + loadInstanceMax.getMemory ) ) } else { @@ -319,9 +319,9 @@ abstract class RequestResourceService(labelResourceService: LabelResourceService RMUtils.getResourceInfoMsg( RMConstant.APP_INSTANCE, RMConstant.INSTANCE_UNIT, - li.instances, - loadInstanceAvailable.instances, - loadInstanceMax.instances + li.getInstances, + loadInstanceAvailable.getInstances, + loadInstanceMax.getInstances ) ) } @@ -348,9 +348,9 @@ abstract class RequestResourceService(labelResourceService: LabelResourceService RMUtils.getResourceInfoMsg( RMConstant.MEMORY, RMConstant.MEMORY_UNIT_BYTE, - m.memory, - avail.memory, - max.memory + m.getMemory, + avail.getMemory, + max.getMemory ) ) case i: InstanceResource => @@ -362,9 +362,9 @@ abstract class RequestResourceService(labelResourceService: LabelResourceService RMUtils.getResourceInfoMsg( RMConstant.APP_INSTANCE, RMConstant.INSTANCE_UNIT, - i.instances, - avail.instances, - max.instances + i.getInstances, + avail.getInstances, + max.getInstances ) ) case c: CPUResource => @@ -376,25 +376,25 @@ abstract class RequestResourceService(labelResourceService: LabelResourceService RMUtils.getResourceInfoMsg( RMConstant.CPU, RMConstant.CPU_UNIT, - c.cores, - avail.cores, - max.cores + c.getCores, + avail.getCores, + max.getCores ) ) case l: LoadResource => val loadAvailable = availableResource.asInstanceOf[LoadResource] val avail = availableResource.asInstanceOf[LoadResource] val max = maxResource.asInstanceOf[LoadResource] - if (l.cores > loadAvailable.cores) { + if (l.getCores > loadAvailable.getCores) { ( RMErrorCode.DRIVER_CPU_INSUFFICIENT.getErrorCode, RMErrorCode.DRIVER_CPU_INSUFFICIENT.getErrorDesc + RMUtils.getResourceInfoMsg( RMConstant.CPU, RMConstant.CPU_UNIT, - l.cores, - avail.cores, - max.cores + l.getCores, + avail.getCores, + max.getCores ) ) } else { @@ -404,9 +404,9 @@ abstract class RequestResourceService(labelResourceService: LabelResourceService RMUtils.getResourceInfoMsg( RMConstant.MEMORY, RMConstant.MEMORY_UNIT_BYTE, - l.memory, - avail.memory, - max.memory + l.getMemory, + avail.getMemory, + max.getMemory ) ) } @@ -414,28 +414,28 @@ abstract class RequestResourceService(labelResourceService: LabelResourceService val loadInstanceAvailable = availableResource.asInstanceOf[LoadInstanceResource] val avail = availableResource.asInstanceOf[LoadInstanceResource] val max = maxResource.asInstanceOf[LoadInstanceResource] - if (li.cores > loadInstanceAvailable.cores) { + if (li.getCores > loadInstanceAvailable.getCores) { ( RMErrorCode.DRIVER_CPU_INSUFFICIENT.getErrorCode, RMErrorCode.DRIVER_CPU_INSUFFICIENT.getErrorDesc + RMUtils.getResourceInfoMsg( RMConstant.CPU, RMConstant.CPU_UNIT, - li.cores, - avail.cores, - max.cores + li.getCores, + avail.getCores, + max.getCores ) ) - } else if (li.memory > loadInstanceAvailable.memory) { + } else if (li.getMemory > loadInstanceAvailable.getMemory) { ( RMErrorCode.DRIVER_MEMORY_INSUFFICIENT.getErrorCode, RMErrorCode.DRIVER_MEMORY_INSUFFICIENT.getErrorDesc + RMUtils.getResourceInfoMsg( RMConstant.MEMORY, RMConstant.MEMORY_UNIT_BYTE, - li.memory, - avail.memory, - max.memory + li.getMemory, + avail.getMemory, + max.getMemory ) ) } else { @@ -445,9 +445,9 @@ abstract class RequestResourceService(labelResourceService: LabelResourceService RMUtils.getResourceInfoMsg( RMConstant.APP_INSTANCE, RMConstant.INSTANCE_UNIT, - li.instances, - avail.instances, - max.instances + li.getInstances, + avail.getInstances, + max.getInstances ) ) } @@ -455,28 +455,28 @@ abstract class RequestResourceService(labelResourceService: LabelResourceService val yarnAvailable = availableResource.asInstanceOf[YarnResource] val avail = availableResource.asInstanceOf[YarnResource] val max = maxResource.asInstanceOf[YarnResource] - if (yarn.queueCores > yarnAvailable.queueCores) { + if (yarn.getQueueCores > yarnAvailable.getQueueCores) { ( RMErrorCode.QUEUE_CPU_INSUFFICIENT.getErrorCode, RMErrorCode.QUEUE_CPU_INSUFFICIENT.getErrorDesc + RMUtils.getResourceInfoMsg( RMConstant.CPU, RMConstant.CPU_UNIT, - yarn.queueCores, - avail.queueCores, - max.queueCores + yarn.getQueueCores, + avail.getQueueCores, + max.getQueueCores ) ) - } else if (yarn.queueMemory > yarnAvailable.queueMemory) { + } else if (yarn.getQueueMemory > yarnAvailable.getQueueMemory) { ( RMErrorCode.QUEUE_MEMORY_INSUFFICIENT.getErrorCode, RMErrorCode.QUEUE_MEMORY_INSUFFICIENT.getErrorDesc + RMUtils.getResourceInfoMsg( RMConstant.MEMORY, RMConstant.MEMORY_UNIT_BYTE, - yarn.queueMemory, - avail.queueMemory, - max.queueMemory + yarn.getQueueMemory, + avail.getQueueMemory, + max.getQueueMemory ) ) } else { @@ -486,9 +486,9 @@ abstract class RequestResourceService(labelResourceService: LabelResourceService RMUtils.getResourceInfoMsg( RMConstant.APP_INSTANCE, RMConstant.INSTANCE_UNIT, - yarn.queueInstances, - avail.queueInstances, - max.queueInstances + yarn.getQueueInstances, + avail.getQueueInstances, + max.getQueueInstances ) ) } @@ -496,19 +496,23 @@ abstract class RequestResourceService(labelResourceService: LabelResourceService val dyAvailable = availableResource.asInstanceOf[DriverAndYarnResource] val dyMax = maxResource.asInstanceOf[DriverAndYarnResource] if ( - dy.loadInstanceResource.memory > dyAvailable.loadInstanceResource.memory || - dy.loadInstanceResource.cores > dyAvailable.loadInstanceResource.cores || - dy.loadInstanceResource.instances > dyAvailable.loadInstanceResource.instances + dy.getLoadInstanceResource.getMemory > dyAvailable.getLoadInstanceResource.getMemory || + dy.getLoadInstanceResource.getCores > dyAvailable.getLoadInstanceResource.getCores || + dy.getLoadInstanceResource.getInstances > dyAvailable.getLoadInstanceResource.getInstances ) { val detail = generateNotEnoughMessage( - dy.loadInstanceResource, - dyAvailable.loadInstanceResource, - dyMax.loadInstanceResource + dy.getLoadInstanceResource, + dyAvailable.getLoadInstanceResource, + dyMax.getLoadInstanceResource ) (detail._1, { detail._2 }) } else { val detail = - generateNotEnoughMessage(dy.yarnResource, dyAvailable.yarnResource, dyMax.yarnResource) + generateNotEnoughMessage( + dy.getYarnResource, + dyAvailable.getYarnResource, + dyMax.getYarnResource + ) (detail._1, { detail._2 }) } case s: SpecialResource => diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/service/impl/DefaultResourceManager.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/service/impl/DefaultResourceManager.scala index 457bae9bed..d0a32e4900 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/service/impl/DefaultResourceManager.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/service/impl/DefaultResourceManager.scala @@ -160,21 +160,21 @@ class DefaultResourceManager extends ResourceManager with Logging with Initializ registerResourceFlag = false logger.warn(s"${serviceInstance} has been registered, resource is ${emResource}.") val leftResource = emResource.getLeftResource - if (leftResource != null && Resource.getZeroResource(leftResource) > leftResource) { + if (leftResource != null && Resource.getZeroResource(leftResource).moreThan(leftResource)) { logger.warn( s"${serviceInstance} has been registered, but left Resource <0 need to register resource." ) registerResourceFlag = true } - val usedResource = emResource.getLockedResource + emResource.getUsedResource - if (usedResource > emResource.getMaxResource) { + val usedResource = emResource.getLockedResource.add(emResource.getUsedResource) + if (usedResource.moreThan(emResource.getMaxResource)) { logger.warn( s"${serviceInstance} has been registered, but usedResource > MaxResource need to register resource." ) registerResourceFlag = true } - if (!(resource.getMaxResource == emResource.getMaxResource)) { + if (!(resource.getMaxResource.equalsTo(emResource.getMaxResource))) { logger.warn( s"${serviceInstance} has been registered, but inconsistent newly registered resources need to register resource." ) @@ -277,7 +277,7 @@ class DefaultResourceManager extends ResourceManager with Logging with Initializ val resourceLabels = labelContainer.getResourceLabels.asScala val persistenceLocks = new ArrayBuffer[PersistenceLock]() val emInstanceLabel = labelContainer.getEMInstanceLabel - val userCreatorEngineTypeLabel = labelContainer.getCombinedUserCreatorEngineTypeLabel + val userCreatorEngineTypeLabel = labelContainer.getCombinedResourceLabel Utils.tryFinally { // check ecm resource if not enough return @@ -323,14 +323,16 @@ class DefaultResourceManager extends ResourceManager with Logging with Initializ labelContainer.setCurrentLabel(label) val labelResource = labelResourceService.getLabelResource(label) if (labelResource != null) { - labelResource.setLeftResource(labelResource.getLeftResource - resource.getLockedResource) + labelResource.setLeftResource( + labelResource.getLeftResource.minus(resource.getLockedResource) + ) labelResource.setLockedResource( - labelResource.getLockedResource + resource.getLockedResource + labelResource.getLockedResource.add(resource.getLockedResource) ) labelResourceService.setLabelResource( label, labelResource, - labelContainer.getCombinedUserCreatorEngineTypeLabel.getStringValue + labelContainer.getCombinedResourceLabel.getStringValue ) logger.info(s"ResourceChanged:${label.getStringValue} --> ${labelResource}") resourceCheck(label, labelResource) @@ -363,7 +365,7 @@ class DefaultResourceManager extends ResourceManager with Logging with Initializ labelResourceService.setEngineConnLabelResource( engineInstanceLabel, resource, - labelContainer.getCombinedUserCreatorEngineTypeLabel.getStringValue + labelContainer.getCombinedResourceLabel.getStringValue ) // record engine locked resource labelContainer.getLabels.add(engineInstanceLabel) @@ -435,8 +437,8 @@ class DefaultResourceManager extends ResourceManager with Logging with Initializ ) } if ( - lockedResource == null || lockedResource.getLockedResource <= Resource.initResource( - lockedResource.getResourceType + lockedResource == null || !lockedResource.getLockedResource.moreThan( + Resource.initResource(lockedResource.getResourceType) ) ) { throw new RMErrorException( @@ -448,7 +450,7 @@ class DefaultResourceManager extends ResourceManager with Logging with Initializ s"resourceUsed ready:${labelContainer.getEMInstanceLabel.getServiceInstance}, used resource ${lockedResource.getLockedResource}" ) val addedResource = - Resource.initResource(lockedResource.getResourceType) + lockedResource.getLockedResource + Resource.initResource(lockedResource.getResourceType).add(lockedResource.getLockedResource) val engineInstanceLabel: EngineInstanceLabel = labelContainer.getEngineInstanceLabel Utils.tryCatch { @@ -458,7 +460,7 @@ class DefaultResourceManager extends ResourceManager with Logging with Initializ labelResourceService.setLabelResource( engineInstanceLabel, lockedResource, - labelContainer.getCombinedUserCreatorEngineTypeLabel.getStringValue + labelContainer.getCombinedResourceLabel.getStringValue ) resourceLogService.success( ChangeType.ENGINE_INIT, @@ -482,15 +484,15 @@ class DefaultResourceManager extends ResourceManager with Logging with Initializ labelContainer.setCurrentLabel(label) val labelResource = labelResourceService.getLabelResource(label) if (labelResource != null) { - labelResource.setLockedResource(labelResource.getLockedResource - addedResource) + labelResource.setLockedResource(labelResource.getLockedResource.minus(addedResource)) if (null == labelResource.getUsedResource) { labelResource.setUsedResource(Resource.initResource(labelResource.getResourceType)) } - labelResource.setUsedResource(labelResource.getUsedResource + addedResource) + labelResource.setUsedResource(labelResource.getUsedResource.add(addedResource)) labelResourceService.setLabelResource( label, labelResource, - labelContainer.getCombinedUserCreatorEngineTypeLabel.getStringValue + labelContainer.getCombinedResourceLabel.getStringValue ) labelResourceSet.add( new LabelResourceMapping(label, addedResource, ResourceOperationType.USED) @@ -500,11 +502,7 @@ class DefaultResourceManager extends ResourceManager with Logging with Initializ } { resourceLockService.unLock(persistenceLock) } - if ( - label.getClass.isAssignableFrom( - labelContainer.getCombinedUserCreatorEngineTypeLabel.getClass - ) - ) { + if (label.getClass.isAssignableFrom(labelContainer.getCombinedResourceLabel.getClass)) { resourceLogService.recordUserResourceAction( labelContainer, persistenceResource.getTicketId, @@ -527,9 +525,9 @@ class DefaultResourceManager extends ResourceManager with Logging with Initializ if (labelResource != null && label != null) { val resourceInit = Resource.initResource(labelResource.getResourceType) if ( - labelResource.getLockedResource < resourceInit || - labelResource.getUsedResource < resourceInit || - labelResource.getLeftResource < resourceInit + !labelResource.getLockedResource.notLess(resourceInit) || + !labelResource.getUsedResource.notLess(resourceInit) || + !labelResource.getLeftResource.notLess(resourceInit) ) { logger.info( s"found error resource! resource label:${label.getStringValue}, resource:${labelResource}, please check!" @@ -546,20 +544,20 @@ class DefaultResourceManager extends ResourceManager with Logging with Initializ case driverAndYarnResource: DriverAndYarnResource => nodeResource.getUsedResource match { case resource: DriverAndYarnResource => - val newYarnResource = resource.yarnResource + val newYarnResource = resource.getYarnResource val applicationId: String = if (null != newYarnResource) { - newYarnResource.applicationId + newYarnResource.getApplicationId } else { null } - val oriYarnResource = driverAndYarnResource.yarnResource + val oriYarnResource = driverAndYarnResource.getYarnResource val tmpUsedResource = new DriverAndYarnResource( - driverAndYarnResource.loadInstanceResource, + driverAndYarnResource.getLoadInstanceResource, new YarnResource( - oriYarnResource.queueMemory, - oriYarnResource.queueCores, - oriYarnResource.queueInstances, - oriYarnResource.queueName, + oriYarnResource.getQueueMemory, + oriYarnResource.getQueueCores, + oriYarnResource.getQueueInstances, + oriYarnResource.getQueueName, applicationId ) ) @@ -570,11 +568,11 @@ class DefaultResourceManager extends ResourceManager with Logging with Initializ nodeResource.getUsedResource match { case resource: YarnResource => val tmpYarnResource = new YarnResource( - yarnResource.queueMemory, - yarnResource.queueCores, - yarnResource.queueInstances, - yarnResource.queueName, - resource.applicationId + yarnResource.getQueueMemory, + yarnResource.getQueueCores, + yarnResource.getQueueInstances, + yarnResource.getQueueName, + resource.getApplicationId ) lockedResource.setUsedResource(tmpYarnResource) case _ => @@ -593,16 +591,18 @@ class DefaultResourceManager extends ResourceManager with Logging with Initializ val resource = labelResourceService.getLabelResource(labelResourceMapping.getLabel()) labelResourceMapping.getResourceOperationType match { case LOCK => - resource.setLeftResource(resource.getLeftResource + labelResourceMapping.getResource()) + resource.setLeftResource( + resource.getLeftResource.add(labelResourceMapping.getResource()) + ) resource.setLockedResource( - resource.getLockedResource - labelResourceMapping.getResource() + resource.getLockedResource.minus(labelResourceMapping.getResource()) ) case USED => resource.setLockedResource( - resource.getLeftResource + labelResourceMapping.getResource() + resource.getLeftResource.add(labelResourceMapping.getResource()) ) resource.setUsedResource( - resource.getLockedResource - labelResourceMapping.getResource() + resource.getLockedResource.minus(labelResourceMapping.getResource()) ) case _ => } @@ -695,31 +695,36 @@ class DefaultResourceManager extends ResourceManager with Logging with Initializ if (labelResource != null) { if (label.isInstanceOf[EMInstanceLabel]) timeCheck(labelResource, usedResource) if ( - null != usedResource.getUsedResource && usedResource.getUsedResource > Resource - .initResource(usedResource.getResourceType) + null != usedResource.getUsedResource && usedResource.getUsedResource.moreThan( + Resource + .initResource(usedResource.getResourceType) + ) ) { labelResource.setUsedResource( - labelResource.getUsedResource - usedResource.getUsedResource + labelResource.getUsedResource.minus(usedResource.getUsedResource) ) labelResource.setLeftResource( - labelResource.getLeftResource + usedResource.getUsedResource + labelResource.getLeftResource.add(usedResource.getUsedResource) ) } if ( - null != usedResource.getLockedResource && usedResource.getLockedResource != Resource - .initResource(usedResource.getResourceType) + null != usedResource.getLockedResource && usedResource.getLockedResource + .moreThan( + Resource + .initResource(usedResource.getResourceType) + ) ) { labelResource.setLockedResource( - labelResource.getLockedResource - usedResource.getLockedResource + labelResource.getLockedResource.minus(usedResource.getLockedResource) ) labelResource.setLeftResource( - labelResource.getLeftResource + usedResource.getLockedResource + labelResource.getLeftResource.add(usedResource.getLockedResource) ) } labelResourceService.setLabelResource( label, labelResource, - labelContainer.getCombinedUserCreatorEngineTypeLabel.getStringValue + labelContainer.getCombinedResourceLabel.getStringValue ) resourceCheck(label, labelResource) } @@ -739,11 +744,7 @@ class DefaultResourceManager extends ResourceManager with Logging with Initializ heartbeatMsgMetrics = oldMetrics.getHeartBeatMsg } } - if ( - label.getClass.isAssignableFrom( - labelContainer.getCombinedUserCreatorEngineTypeLabel.getClass - ) - ) { + if (label.getClass.isAssignableFrom(labelContainer.getCombinedResourceLabel.getClass)) { resourceLogService.recordUserResourceAction( labelContainer, persistenceResource.getTicketId, @@ -878,8 +879,8 @@ class DefaultResourceManager extends ResourceManager with Logging with Initializ if ( usedResource != null && usedResource.getLockedResource != null - && usedResource.getLockedResource > Resource.getZeroResource( - usedResource.getLockedResource + && usedResource.getLockedResource.moreThan( + Resource.getZeroResource(usedResource.getLockedResource) ) ) { val dbEngineInstanceLabel = labelManagerPersistence.getLabel(persistenceEngineLabel.getId) @@ -929,7 +930,7 @@ class DefaultResourceManager extends ResourceManager with Logging with Initializ // check resource with lock val requestResourceService = getRequestResourceService(resource.getResourceType) if ( - labelContainer.getEMInstanceLabel == null || labelContainer.getCombinedUserCreatorEngineTypeLabel == null + labelContainer.getEMInstanceLabel == null || labelContainer.getCombinedResourceLabel == null ) { val canCreateECRes = new CanCreateECRes canCreateECRes.setCanCreateEC(false) diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/service/impl/DriverAndYarnReqResourceService.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/service/impl/DriverAndYarnReqResourceService.scala index 1039fec795..50bf8b8743 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/service/impl/DriverAndYarnReqResourceService.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/service/impl/DriverAndYarnReqResourceService.scala @@ -17,9 +17,8 @@ package org.apache.linkis.manager.rm.service.impl -import org.apache.linkis.manager.am.conf.AMConfiguration import org.apache.linkis.manager.am.vo.CanCreateECRes -import org.apache.linkis.manager.common.constant.{AMConstant, RMConstant} +import org.apache.linkis.manager.common.constant.RMConstant import org.apache.linkis.manager.common.entity.resource._ import org.apache.linkis.manager.common.entity.resource.ResourceType.DriverAndYarn import org.apache.linkis.manager.common.exception.RMWarnException @@ -29,13 +28,11 @@ import org.apache.linkis.manager.rm.exception.RMErrorCode import org.apache.linkis.manager.rm.external.service.ExternalResourceService import org.apache.linkis.manager.rm.external.yarn.YarnResourceIdentifier import org.apache.linkis.manager.rm.service.{LabelResourceService, RequestResourceService} -import org.apache.linkis.manager.rm.utils.{AcrossClusterRulesJudgeUtils, RMUtils} import org.apache.linkis.manager.rm.utils.AcrossClusterRulesJudgeUtils.{ originClusterResourceCheck, targetClusterResourceCheck } - -import org.apache.commons.lang3.StringUtils +import org.apache.linkis.manager.rm.utils.RMUtils class DriverAndYarnReqResourceService( labelResourceService: LabelResourceService, @@ -55,20 +52,20 @@ class DriverAndYarnReqResourceService( } val requestedDriverAndYarnResource = resource.getMaxResource.asInstanceOf[DriverAndYarnResource] - val requestedYarnResource = requestedDriverAndYarnResource.yarnResource - val yarnIdentifier = new YarnResourceIdentifier(requestedYarnResource.queueName) + val requestedYarnResource = requestedDriverAndYarnResource.getYarnResource + val yarnIdentifier = new YarnResourceIdentifier(requestedYarnResource.getQueueName) val providedYarnResource = externalResourceService.getResource(ResourceType.Yarn, labelContainer, yarnIdentifier) val (maxCapacity, usedCapacity) = (providedYarnResource.getMaxResource, providedYarnResource.getUsedResource) logger.debug( - s"This queue: ${requestedYarnResource.queueName} used resource:$usedCapacity and max resource: $maxCapacity" + s"This queue: ${requestedYarnResource.getQueueName} used resource:$usedCapacity and max resource: $maxCapacity" ) - val queueLeftResource = maxCapacity - usedCapacity + val queueLeftResource = maxCapacity.minus(usedCapacity) logger.info( - s"queue: ${requestedYarnResource.queueName} left $queueLeftResource, this request requires: $requestedYarnResource" + s"queue: ${requestedYarnResource.getQueueName} left $queueLeftResource, this request requires: $requestedYarnResource" ) - if (queueLeftResource < requestedYarnResource) { + if (!queueLeftResource.notLess(requestedYarnResource)) { logger.info( s"user: ${labelContainer.getUserCreatorLabel.getUser} request queue resource $requestedYarnResource > left resource $queueLeftResource" ) @@ -92,24 +89,24 @@ class DriverAndYarnReqResourceService( } val requestedDriverAndYarnResource = resource.getMaxResource.asInstanceOf[DriverAndYarnResource] - val requestedYarnResource = requestedDriverAndYarnResource.yarnResource - val yarnIdentifier = new YarnResourceIdentifier(requestedYarnResource.queueName) + val requestedYarnResource = requestedDriverAndYarnResource.getYarnResource + val yarnIdentifier = new YarnResourceIdentifier(requestedYarnResource.getQueueName) val providedYarnResource = externalResourceService.getResource(ResourceType.Yarn, labelContainer, yarnIdentifier) val (maxCapacity, usedCapacity) = (providedYarnResource.getMaxResource, providedYarnResource.getUsedResource) logger.debug( - s"This queue: ${requestedYarnResource.queueName} used resource:$usedCapacity and max resource: $maxCapacity" + s"This queue: ${requestedYarnResource.getQueueName} used resource:$usedCapacity and max resource: $maxCapacity" ) - val queueLeftResource = maxCapacity - usedCapacity + val queueLeftResource = maxCapacity.minus(usedCapacity) logger.info( - s"queue: ${requestedYarnResource.queueName} left $queueLeftResource, this request requires: $requestedYarnResource" + s"queue: ${requestedYarnResource.getQueueName} left $queueLeftResource, this request requires: $requestedYarnResource" ) if (engineCreateRequest.getProperties != null) { // judge if is cross cluster task and origin cluster priority first originClusterResourceCheck(engineCreateRequest, maxCapacity, usedCapacity) } - if (queueLeftResource < requestedYarnResource) { + if (!queueLeftResource.notLess(requestedYarnResource)) { logger.info( s"user: ${labelContainer.getUserCreatorLabel.getUser} request queue resource $requestedYarnResource > left resource $queueLeftResource" ) @@ -139,30 +136,30 @@ class DriverAndYarnReqResourceService( case yarn: YarnResource => val yarnAvailable = availableResource.asInstanceOf[YarnResource] val maxYarn = maxResource.asInstanceOf[YarnResource] - if (yarn.queueCores > yarnAvailable.queueCores) { + if (yarn.getQueueCores > yarnAvailable.getQueueCores) { ( RMErrorCode.CLUSTER_QUEUE_CPU_INSUFFICIENT.getErrorCode, RMErrorCode.CLUSTER_QUEUE_CPU_INSUFFICIENT.getErrorDesc + RMUtils.getResourceInfoMsg( RMConstant.CPU, RMConstant.CPU_UNIT, - yarn.queueCores, - yarnAvailable.queueCores, - maxYarn.queueCores, - yarn.queueName + yarn.getQueueCores, + yarnAvailable.getQueueCores, + maxYarn.getQueueCores, + yarn.getQueueName ) ) - } else if (yarn.queueMemory > yarnAvailable.queueMemory) { + } else if (yarn.getQueueMemory > yarnAvailable.getQueueMemory) { ( RMErrorCode.CLUSTER_QUEUE_MEMORY_INSUFFICIENT.getErrorCode, RMErrorCode.CLUSTER_QUEUE_MEMORY_INSUFFICIENT.getErrorDesc + RMUtils.getResourceInfoMsg( RMConstant.MEMORY, RMConstant.MEMORY_UNIT_BYTE, - yarn.queueMemory, - yarnAvailable.queueMemory, - maxYarn.queueMemory, - yarn.queueName + yarn.getQueueMemory, + yarnAvailable.getQueueMemory, + maxYarn.getQueueMemory, + yarn.getQueueName ) ) } else { @@ -172,10 +169,10 @@ class DriverAndYarnReqResourceService( RMUtils.getResourceInfoMsg( RMConstant.APP_INSTANCE, RMConstant.INSTANCE_UNIT, - yarn.queueInstances, - yarnAvailable.queueInstances, - maxYarn.queueInstances, - yarn.queueName + yarn.getQueueInstances, + yarnAvailable.getQueueInstances, + maxYarn.getQueueInstances, + yarn.getQueueName ) ) } diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/service/impl/ResourceLogService.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/service/impl/ResourceLogService.scala index 2d67edb9a7..0631e485e8 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/service/impl/ResourceLogService.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/service/impl/ResourceLogService.scala @@ -148,7 +148,7 @@ class ResourceLogService extends Logging { metrics: String = null ): Unit = if (RMUtils.RM_RESOURCE_ACTION_RECORD.getValue) Utils.tryAndWarn { val userCreatorEngineType: CombinedLabel = - labelContainer.getCombinedUserCreatorEngineTypeLabel + labelContainer.getCombinedResourceLabel val engineInstanceLabel: EngineInstanceLabel = labelContainer.getEngineInstanceLabel val eMInstanceLabel = labelContainer.getEMInstanceLabel if (null == userCreatorEngineType) return diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/service/impl/UserResourceService.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/service/impl/UserResourceService.scala index 86d9c1e884..e4a89d4bf1 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/service/impl/UserResourceService.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/service/impl/UserResourceService.scala @@ -85,7 +85,9 @@ class UserResourceService { labelResource.setMaxResource(configuredResource) labelResource.setMinResource(Resource.initResource(labelResource.getResourceType)) labelResource.setLeftResource( - labelResource.getMaxResource - labelResource.getUsedResource - labelResource.getLockedResource + labelResource.getMaxResource + .minus(labelResource.getUsedResource) + .minus(labelResource.getLockedResource) ) val idList = new util.ArrayList[Integer]() idList.add(resourceId) diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/utils/AcrossClusterRulesJudgeUtils.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/utils/AcrossClusterRulesJudgeUtils.scala index 65154f9ad2..535300cfbe 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/utils/AcrossClusterRulesJudgeUtils.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/utils/AcrossClusterRulesJudgeUtils.scala @@ -49,10 +49,10 @@ object AcrossClusterRulesJudgeUtils extends Logging { leftResource != null && usedResource != null && maxResource != null && clusterMaxCapacity != null && clusterUsedCapacity != null ) { - val clusterUsedCPUPercentage = clusterUsedCapacity.queueCores - .asInstanceOf[Double] / clusterMaxCapacity.queueCores.asInstanceOf[Double] - val clusterUsedMemoryPercentage = clusterUsedCapacity.queueMemory - .asInstanceOf[Double] / clusterMaxCapacity.queueMemory.asInstanceOf[Double] + val clusterUsedCPUPercentage = clusterUsedCapacity.getQueueCores + .asInstanceOf[Double] / clusterMaxCapacity.getQueueCores.asInstanceOf[Double] + val clusterUsedMemoryPercentage = clusterUsedCapacity.getQueueMemory + .asInstanceOf[Double] / clusterMaxCapacity.getQueueMemory.asInstanceOf[Double] if ( clusterUsedCPUPercentage >= clusterCPUPercentageThreshold || clusterUsedMemoryPercentage >= clusterMemoryPercentageThreshold @@ -64,13 +64,13 @@ object AcrossClusterRulesJudgeUtils extends Logging { ) } - val leftQueueMemory = leftResource.queueMemory / Math.pow(1024, 3).toLong - if (leftResource.queueCores >= CPUThreshold && leftQueueMemory >= MemoryThreshold) { + val leftQueueMemory = leftResource.getQueueMemory / Math.pow(1024, 3).toLong + if (leftResource.getQueueCores >= CPUThreshold && leftQueueMemory >= MemoryThreshold) { val usedCPUPercentage = - usedResource.queueCores.asInstanceOf[Double] / maxResource.queueCores + usedResource.getQueueCores.asInstanceOf[Double] / maxResource.getQueueCores .asInstanceOf[Double] - val usedMemoryPercentage = usedResource.queueMemory - .asInstanceOf[Double] / maxResource.queueMemory.asInstanceOf[Double] + val usedMemoryPercentage = usedResource.getQueueMemory + .asInstanceOf[Double] / maxResource.getQueueMemory.asInstanceOf[Double] logger.info( "cross cluster test in target rule check" + s"usedCPUPercentage: $usedCPUPercentage, CPUPercentageThreshold: $CPUPercentageThreshold" + @@ -89,7 +89,7 @@ object AcrossClusterRulesJudgeUtils extends Logging { } else { throw new RMWarnException( RMErrorCode.ACROSS_CLUSTER_RULE_FAILED.getErrorCode, - s"leftResource.queueCores: ${leftResource.queueCores}, CPUThreshold: $CPUThreshold," + + s"leftResource.queueCores: ${leftResource.getQueueCores}, CPUThreshold: $CPUThreshold," + s"leftQueueMemory: $leftQueueMemory, MemoryThreshold: $MemoryThreshold" ) } @@ -105,10 +105,10 @@ object AcrossClusterRulesJudgeUtils extends Logging { if (usedResource != null && maxResource != null) { val usedCPUPercentage = - usedResource.queueCores.asInstanceOf[Double] / maxResource.queueCores + usedResource.getQueueCores.asInstanceOf[Double] / maxResource.getQueueCores .asInstanceOf[Double] - val usedMemoryPercentage = usedResource.queueMemory - .asInstanceOf[Double] / maxResource.queueMemory.asInstanceOf[Double] + val usedMemoryPercentage = usedResource.getQueueMemory + .asInstanceOf[Double] / maxResource.getQueueMemory.asInstanceOf[Double] logger.info( "cross cluster test in origin rule check" + s"usedCPUPercentage: $usedCPUPercentage, CPUPercentageThreshold: $CPUPercentageThreshold" + @@ -186,7 +186,7 @@ object AcrossClusterRulesJudgeUtils extends Logging { AMConfiguration.PRIORITY_CLUSTER_TARGET ) ) { - val leftResource = maxCapacity - usedCapacity + val leftResource = maxCapacity.minus(usedCapacity) // get target cluster resource threshold val targetCPUThreshold = engineCreateRequest.getProperties.get(AMConfiguration.TARGET_CPU_THRESHOLD) diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/utils/RMUtils.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/utils/RMUtils.scala index 41618649fd..17783a98db 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/utils/RMUtils.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/utils/RMUtils.scala @@ -141,13 +141,13 @@ object RMUtils extends Logging { if (userResource.getLeftResource != null && userResource.getMaxResource != null) { if (userResource.getResourceType.equals(ResourceType.DriverAndYarn)) { val leftDriverResource = - userResource.getLeftResource.asInstanceOf[DriverAndYarnResource].loadInstanceResource + userResource.getLeftResource.asInstanceOf[DriverAndYarnResource].getLoadInstanceResource val leftYarnResource = - userResource.getLeftResource.asInstanceOf[DriverAndYarnResource].yarnResource + userResource.getLeftResource.asInstanceOf[DriverAndYarnResource].getYarnResource val maxDriverResource = - userResource.getMaxResource.asInstanceOf[DriverAndYarnResource].loadInstanceResource + userResource.getMaxResource.asInstanceOf[DriverAndYarnResource].getLoadInstanceResource val maxYarnResource = - userResource.getMaxResource.asInstanceOf[DriverAndYarnResource].yarnResource + userResource.getMaxResource.asInstanceOf[DriverAndYarnResource].getYarnResource userResourceVo.setLoadResourceStatus( ResourceStatus.measure(leftDriverResource, maxDriverResource) ) @@ -327,13 +327,13 @@ object RMUtils extends Logging { def getYarnResourceMap(resource: Resource): util.HashMap[String, Any] = { val resourceMap = new util.HashMap[String, Any] - val yarnResource = resource.asInstanceOf[DriverAndYarnResource].yarnResource + val yarnResource = resource.asInstanceOf[DriverAndYarnResource].getYarnResource resourceMap.put( "queueMemory", - ByteTimeUtils.negativeByteStringAsGb(yarnResource.queueMemory + "b") + "G" + ByteTimeUtils.negativeByteStringAsGb(yarnResource.getQueueMemory + "b") + "G" ) - resourceMap.put("queueCpu", yarnResource.queueCores) - resourceMap.put("instance", yarnResource.queueInstances) + resourceMap.put("queueCpu", yarnResource.getQueueCores) + resourceMap.put("instance", yarnResource.getQueueInstances) resourceMap } @@ -341,16 +341,16 @@ object RMUtils extends Logging { val resourceMap = new util.HashMap[String, Any] var loadInstanceResource = new LoadInstanceResource(0, 0, 0) if (engineType.contains("spark")) { - loadInstanceResource = resource.asInstanceOf[DriverAndYarnResource].loadInstanceResource + loadInstanceResource = resource.asInstanceOf[DriverAndYarnResource].getLoadInstanceResource } else { loadInstanceResource = resource.asInstanceOf[LoadInstanceResource] } resourceMap.put( "memory", - ByteTimeUtils.negativeByteStringAsGb(loadInstanceResource.memory + "b") + "G" + ByteTimeUtils.negativeByteStringAsGb(loadInstanceResource.getMemory + "b") + "G" ) - resourceMap.put("core", loadInstanceResource.cores) + resourceMap.put("core", loadInstanceResource.getCores) resourceMap } diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/utils/UserConfiguration.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/utils/UserConfiguration.scala index c141a809c1..017b354c5d 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/utils/UserConfiguration.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/utils/UserConfiguration.scala @@ -163,7 +163,7 @@ object UserConfiguration extends Logging { USER_AVAILABLE_YARN_QUEUE_NAME.getValue(userConfiguration) ) ) - case ResourceType.Special => new SpecialResource(new java.util.HashMap[String, AnyVal]()) + case ResourceType.Special => new SpecialResource(new java.util.HashMap[String, AnyRef]()) case _ => throw new RMWarnException( NOT_RESOURCE_RESULT_TYPE.getErrorCode, diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/service/common/metrics/MetricsConverter.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/service/common/metrics/MetricsConverter.scala deleted file mode 100644 index 3da3a3cc52..0000000000 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/service/common/metrics/MetricsConverter.scala +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.manager.service.common.metrics - -import org.apache.linkis.common.ServiceInstance -import org.apache.linkis.manager.common.entity.enumeration.{NodeHealthy, NodeStatus} -import org.apache.linkis.manager.common.entity.metrics._ -import org.apache.linkis.manager.common.entity.node.AMNode - -trait MetricsConverter { - - def parseTaskInfo(nodeMetrics: NodeMetrics): NodeTaskInfo - - def parseHealthyInfo(nodeMetrics: NodeMetrics): NodeHealthyInfo - - def parseOverLoadInfo(nodeMetrics: NodeMetrics): NodeOverLoadInfo - - def parseStatus(nodeMetrics: NodeMetrics): NodeStatus - - def convertTaskInfo(nodeTaskInfo: NodeTaskInfo): String - - def convertHealthyInfo(nodeHealthyInfo: NodeHealthyInfo): String - - def convertOverLoadInfo(nodeOverLoadInfo: NodeOverLoadInfo): String - - def convertStatus(nodeStatus: NodeStatus): Int - - def fillMetricsToNode(amNode: AMNode, metrics: NodeMetrics): AMNode - - def getInitMetric(serviceInstance: ServiceInstance): NodeMetrics = { - val nodeMetrics: AMNodeMetrics = new AMNodeMetrics - - nodeMetrics.setStatus(NodeStatus.Starting.ordinal) - - val nodeHealthyInfo: NodeHealthyInfo = new NodeHealthyInfo - nodeHealthyInfo.setNodeHealthy(NodeHealthy.Healthy) - nodeMetrics.setHealthy(convertHealthyInfo(nodeHealthyInfo)) - nodeMetrics.setServiceInstance(serviceInstance) - nodeMetrics - } - -} diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/service/common/pointer/EngineNodePointer.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/service/common/pointer/EngineNodePointer.scala deleted file mode 100644 index 7fa34b8363..0000000000 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/service/common/pointer/EngineNodePointer.scala +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.manager.service.common.pointer - -import org.apache.linkis.manager.common.protocol.{RequestEngineLock, RequestEngineUnlock} -import org.apache.linkis.manager.common.protocol.engine.{ - EngineOperateRequest, - EngineOperateResponse -} - -trait EngineNodePointer extends NodePointer { - - def lockEngine(requestEngineLock: RequestEngineLock): Option[String] - - def releaseLock(requestEngineUnlock: RequestEngineUnlock): Unit - - def executeOperation(engineOperateRequest: EngineOperateRequest): EngineOperateResponse - -} diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/test/scala/org/apache/linkis/manager/label/conf/LabelManagerConfTest.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/test/scala/org/apache/linkis/manager/label/conf/LabelManagerConfTest.scala index 836278336b..a902466ae6 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/test/scala/org/apache/linkis/manager/label/conf/LabelManagerConfTest.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/test/scala/org/apache/linkis/manager/label/conf/LabelManagerConfTest.scala @@ -17,17 +17,19 @@ package org.apache.linkis.manager.label.conf +import org.apache.linkis.manager.am.conf.AMConfiguration + import org.junit.jupiter.api.Test class LabelManagerConfTest { @Test def testRandomFiltering(): Unit = { var label = "tenant" - assert(LabelManagerConf.LONG_LIVED_LABEL.contains(label)) + assert(AMConfiguration.LONG_LIVED_LABEL.contains(label)) label = "yarnCluster" - assert(LabelManagerConf.LONG_LIVED_LABEL.contains(label)) + assert(AMConfiguration.LONG_LIVED_LABEL.contains(label)) label = "test" - assert(!LabelManagerConf.LONG_LIVED_LABEL.contains(label)) + assert(!AMConfiguration.LONG_LIVED_LABEL.contains(label)) } } diff --git a/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/conf/LabelCommonConfig.java b/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/conf/LabelCommonConfig.java index f4d089041a..3304b45100 100644 --- a/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/conf/LabelCommonConfig.java +++ b/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/conf/LabelCommonConfig.java @@ -42,6 +42,9 @@ public class LabelCommonConfig { public static final CommonVars PYTHON_ENGINE_VERSION = CommonVars.apply("wds.linkis.python.engine.version", "python2"); + public static final CommonVars REPL_ENGINE_VERSION = + CommonVars.apply("linkis.repl.engine.version", "1"); + public static final CommonVars FILE_ENGINE_VERSION = CommonVars.apply("wds.linkis.file.engine.version", "1.0"); @@ -72,9 +75,15 @@ public class LabelCommonConfig { public static final CommonVars NEBULA_ENGINE_VERSION = CommonVars.apply("wds.linkis.nebula.engine.version", "3.0.0"); + public static final CommonVars DORIS_ENGINE_VERSION = + CommonVars.apply("linkis.doris.engine.version", "1.2.6"); + public static final CommonVars PRESTO_ENGINE_VERSION = CommonVars.apply("wds.linkis.presto.engine.version", "0.234"); + public static final CommonVars HBASE_ENGINE_VERSION = + CommonVars.apply("wds.linkis.hbase.engine.version", "2.5.3"); + public static final CommonVars OPENLOOKENG_ENGINE_VERSION = CommonVars.apply("wds.linkis.openlookeng.engine.version", "1.5.0"); diff --git a/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/constant/LabelKeyConstant.java b/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/constant/LabelKeyConstant.java index c5e4ad06d1..4856961abf 100644 --- a/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/constant/LabelKeyConstant.java +++ b/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/constant/LabelKeyConstant.java @@ -64,6 +64,8 @@ public class LabelKeyConstant { public static final String FIXED_EC_KEY = "fixedEngineConn"; + public static final String ENGINGE_CONN_RUNTIME_MODE_KEY = "engingeConnRuntimeMode"; + public static final String TEMPLATE_CONF_KEY = "ec.conf.templateId"; public static final String TEMPLATE_CONF_NAME_KEY = "ec.resource.name"; diff --git a/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/constant/LabelValueConstant.java b/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/constant/LabelValueConstant.java index cc62921c81..35c0d06e2e 100644 --- a/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/constant/LabelValueConstant.java +++ b/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/constant/LabelValueConstant.java @@ -20,4 +20,6 @@ public class LabelValueConstant { public static final String OFFLINE_VALUE = "offline"; + + public static final String YARN_CLUSTER_VALUE = "yarnCluster"; } diff --git a/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/entity/cluster/ClusterLabel.java b/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/entity/cluster/ClusterLabel.java index 5e1a3d5c23..b07c77f4d8 100644 --- a/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/entity/cluster/ClusterLabel.java +++ b/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/entity/cluster/ClusterLabel.java @@ -26,7 +26,7 @@ import java.util.HashMap; -import static org.apache.linkis.manager.label.errorcode.LabelCommonErrorCodeSummary.CLUSTER_LABEL_ERROR_CODE; +import static org.apache.linkis.manager.label.errorcode.LabelCommonErrorCodeSummary.CLUSTER_LABEL_VALUE_ERROR; public class ClusterLabel extends GenericLabel implements EMNodeLabel, EngineNodeLabel, UserModifiable { @@ -70,12 +70,26 @@ public String getClusterType() { return null; } + @Override + public boolean equals(Object other) { + if (other instanceof ClusterLabel) { + if (null != getClusterName() && null != getClusterType()) { + return getClusterName().equals(((ClusterLabel) other).getClusterName()) + && getClusterType().equals(((ClusterLabel) other).getClusterType()); + } else { + return false; + } + } else { + return false; + } + } + @Override public void valueCheck(String stringValue) throws LabelErrorException { if (!StringUtils.isEmpty(stringValue)) { if (stringValue.split(SerializableLabel.VALUE_SEPARATOR).length != 2) { throw new LabelErrorException( - CLUSTER_LABEL_ERROR_CODE.getErrorCode(), CLUSTER_LABEL_ERROR_CODE.getErrorDesc()); + CLUSTER_LABEL_VALUE_ERROR.getErrorCode(), CLUSTER_LABEL_VALUE_ERROR.getErrorDesc()); } } } diff --git a/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/entity/engine/EngineType.scala b/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/entity/engine/EngineType.scala index eb441ba476..e155016542 100644 --- a/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/entity/engine/EngineType.scala +++ b/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/entity/engine/EngineType.scala @@ -5,16 +5,16 @@ * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ - + package org.apache.linkis.manager.label.entity.engine import org.apache.linkis.common.utils.Logging @@ -29,8 +29,14 @@ object EngineType extends Enumeration with Logging { val HIVE = Value("hive") + val HBASE = Value("hbase") + val PYTHON = Value("python") + val REPL = Value("repl") + + val DORIS = Value("doris") + val SHELL = Value("shell") val JDBC = Value("jdbc") @@ -63,7 +69,7 @@ object EngineType extends Enumeration with Logging { val SEATUNNEL = Value("seatunnel") - val HBASE = Value("hbase") + val IMPALA = Value("impala") val JOBSERVER = Value("jobserver") @@ -86,6 +92,7 @@ object EngineType extends Enumeration with Logging { case _ if null == str || "".equals(str) => null case _ if SPARK.toString.equalsIgnoreCase(str) => SPARK case _ if HIVE.toString.equalsIgnoreCase(str) => HIVE + case _ if HBASE.toString.equalsIgnoreCase(str) => HBASE case _ if PYTHON.toString.equalsIgnoreCase(str) => PYTHON case _ if SHELL.toString.equalsIgnoreCase(str) => SHELL case _ if JDBC.toString.equalsIgnoreCase(str) => JDBC @@ -94,6 +101,8 @@ object EngineType extends Enumeration with Logging { case _ if PIPELINE.toString.equalsIgnoreCase(str) => PIPELINE case _ if PRESTO.toString.equalsIgnoreCase(str) => PRESTO case _ if NEBULA.toString.equalsIgnoreCase(str) => NEBULA + case _ if REPL.toString.equalsIgnoreCase(str) => REPL + case _ if DORIS.toString.equalsIgnoreCase(str) => DORIS case _ if FLINK.toString.equalsIgnoreCase(str) => FLINK case _ if APPCONN.toString.equals(str) => APPCONN case _ if SQOOP.toString.equalsIgnoreCase(str) => SQOOP @@ -102,7 +111,6 @@ object EngineType extends Enumeration with Logging { case _ if TRINO.toString.equalsIgnoreCase(str) => TRINO case _ if ELASTICSEARCH.toString.equalsIgnoreCase(str) => ELASTICSEARCH case _ if SEATUNNEL.toString.equalsIgnoreCase(str) => SEATUNNEL - case _ if HBASE.toString.equalsIgnoreCase(str) => HBASE case _ => null } diff --git a/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/entity/engine/EngineTypeLabel.java b/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/entity/engine/EngineTypeLabel.java index 09492b146a..53145503e2 100644 --- a/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/entity/engine/EngineTypeLabel.java +++ b/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/entity/engine/EngineTypeLabel.java @@ -17,6 +17,7 @@ package org.apache.linkis.manager.label.entity.engine; +import org.apache.linkis.common.utils.JsonUtils; import org.apache.linkis.manager.label.constant.LabelKeyConstant; import org.apache.linkis.manager.label.entity.EMNodeLabel; import org.apache.linkis.manager.label.entity.EngineNodeLabel; @@ -28,6 +29,8 @@ import java.util.HashMap; +import com.fasterxml.jackson.core.JsonProcessingException; + public class EngineTypeLabel extends GenericLabel implements EngineNodeLabel, EMNodeLabel { public EngineTypeLabel() { @@ -74,4 +77,31 @@ public void setVersion(String version) { public Boolean isEmpty() { return StringUtils.isBlank(getEngineType()) || StringUtils.isBlank(getVersion()); } + + @Override + public void setStringValue(String stringValue) { + if (StringUtils.isNotBlank(stringValue)) { + try { + HashMap valueMap = + JsonUtils.jackson().readValue(stringValue, HashMap.class); + setEngineType(valueMap.get("engineType")); + setVersion(valueMap.get("version")); + } catch (JsonProcessingException e) { + String version; + String engineType = stringValue.split("-")[0]; + + if (engineType.equals("*")) { + version = stringValue.replaceFirst("[" + engineType + "]-", ""); + } else { + version = stringValue.replaceFirst(engineType + "-", ""); + } + + setEngineType(engineType); + setVersion(version); + } + } else { + setEngineType("*"); + setVersion("*"); + } + } } diff --git a/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/entity/engine/EngingeConnRuntimeModeLabel.java b/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/entity/engine/EngingeConnRuntimeModeLabel.java new file mode 100644 index 0000000000..7460f5589d --- /dev/null +++ b/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/entity/engine/EngingeConnRuntimeModeLabel.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.manager.label.entity.engine; + +import org.apache.linkis.manager.label.constant.LabelKeyConstant; +import org.apache.linkis.manager.label.entity.*; +import org.apache.linkis.manager.label.entity.annon.ValueSerialNum; +import org.apache.linkis.manager.label.exception.LabelErrorException; + +import org.apache.commons.lang3.StringUtils; + +import java.util.HashMap; + +import static org.apache.linkis.manager.label.errorcode.LabelCommonErrorCodeSummary.CHECK_LABEL_VALUE_EMPTY; +import static org.apache.linkis.manager.label.errorcode.LabelCommonErrorCodeSummary.LABEL_ERROR_CODE; + +public class EngingeConnRuntimeModeLabel extends GenericLabel + implements EngineNodeLabel, UserModifiable { + + public EngingeConnRuntimeModeLabel() { + setLabelKey(LabelKeyConstant.ENGINGE_CONN_RUNTIME_MODE_KEY); + } + + @ValueSerialNum(0) + public void setModeValue(String modeValue) { + if (getValue() == null) { + setValue(new HashMap<>()); + } + getValue().put("modeValue", modeValue); + } + + public String getModeValue() { + if (getValue() == null) { + return null; + } + return getValue().get("modeValue"); + } + + @Override + public Feature getFeature() { + return Feature.CORE; + } + + @Override + public void valueCheck(String stringValue) throws LabelErrorException { + if (!StringUtils.isBlank(stringValue)) { + if (stringValue.split(SerializableLabel.VALUE_SEPARATOR).length != 1) { + throw new LabelErrorException( + LABEL_ERROR_CODE.getErrorCode(), LABEL_ERROR_CODE.getErrorDesc()); + } + } else { + throw new LabelErrorException( + CHECK_LABEL_VALUE_EMPTY.getErrorCode(), CHECK_LABEL_VALUE_EMPTY.getErrorDesc()); + } + } +} diff --git a/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/entity/engine/RunType.scala b/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/entity/engine/RunType.scala index 1bebe7d347..4732bd56c2 100644 --- a/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/entity/engine/RunType.scala +++ b/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/entity/engine/RunType.scala @@ -24,6 +24,8 @@ object RunType extends Enumeration { val HIVE = Value("hql") val SCALA = Value("scala") val PYTHON = Value("python") + val REPL = Value("repl") + val DORIS = Value("doris") val JAVA = Value("java") val PYSPARK = Value("py") val R = Value("r") @@ -45,11 +47,12 @@ object RunType extends Enumeration { val TRINO_SQL = Value("tsql") val JSON = Value("json") - val SEATUNNEL_FLINK_SQL = Value("sfsql") + val SEATUNNEL_ZETA = Value("szeta") val SEATUNNEL_FLINK = Value("sflink") val SEATUNNEL_SPARK = Value("sspark") val DATA_CALC = Value("data_calc") // spark datacalc (ETL) + val IMPALA_SQL = Value("isql") } diff --git a/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/errorcode/LabelCommonErrorCodeSummary.java b/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/errorcode/LabelCommonErrorCodeSummary.java index 7ce6c6546d..322eb5af86 100644 --- a/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/errorcode/LabelCommonErrorCodeSummary.java +++ b/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/errorcode/LabelCommonErrorCodeSummary.java @@ -21,9 +21,6 @@ public enum LabelCommonErrorCodeSummary implements LinkisErrorCode { UPDATE_LABEL_FAILED(25001, "Update label realtion failed(更新标签属性失败)"), - CLUSTER_LABEL_ERROR_CODE( - 25002, - "The value of the label is set incorrectly, the setting value is: ClusterType-ClusterName "), LABEL_ERROR_CODE( 25002, "The value of the label is set incorrectly, only one value can be set, and the separator symbol '-' cannot be used(标签的值设置错误,只能设置一个值，不能使用分割符符号 '-') "), @@ -34,7 +31,9 @@ public enum LabelCommonErrorCodeSummary implements LinkisErrorCode { CHECK_LABEL_REMOVE_REQUEST( 25007, "ServiceInstance in request is null, please check label remove request(请求中的 ServiceInstance 为空，请检查标签删除请求)"), - CHECK_LABEL_VALUE_EMPTY(25008, "Label value cannot be empty"); + CHECK_LABEL_VALUE_EMPTY(25008, "Label value cannot be empty"), + CLUSTER_LABEL_VALUE_ERROR( + 25009, "The value of cluster label should have a length of 2 and be separated by '-' "); /** (errorCode)错误码 */ private final int errorCode; diff --git a/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/exception/LabelErrorException.java b/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/exception/LabelErrorException.java index 33dfe42a9c..5e9e3bca3b 100644 --- a/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/exception/LabelErrorException.java +++ b/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/exception/LabelErrorException.java @@ -17,10 +17,11 @@ package org.apache.linkis.manager.label.exception; -import org.apache.linkis.common.exception.ErrorException; +import org.apache.linkis.common.exception.ExceptionLevel; +import org.apache.linkis.common.exception.LinkisRuntimeException; /** Define Exception */ -public class LabelErrorException extends ErrorException { +public class LabelErrorException extends LinkisRuntimeException { public LabelErrorException(int errCode, String desc) { super(errCode, desc); } @@ -29,4 +30,9 @@ public LabelErrorException(int errCode, String desc, Throwable t) { super(errCode, desc); this.initCause(t); } + + @Override + public ExceptionLevel getLevel() { + return ExceptionLevel.ERROR; + } } diff --git a/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/utils/EngineTypeLabelCreator.java b/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/utils/EngineTypeLabelCreator.java index e90f282aaf..88cc9139ec 100644 --- a/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/utils/EngineTypeLabelCreator.java +++ b/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/utils/EngineTypeLabelCreator.java @@ -50,6 +50,8 @@ private static void init() { EngineType.HIVE().toString(), LabelCommonConfig.HIVE_ENGINE_VERSION.getValue()); defaultVersion.put( EngineType.PYTHON().toString(), LabelCommonConfig.PYTHON_ENGINE_VERSION.getValue()); + defaultVersion.put( + EngineType.REPL().toString(), LabelCommonConfig.REPL_ENGINE_VERSION.getValue()); defaultVersion.put( EngineType.IO_ENGINE_FILE().toString(), LabelCommonConfig.FILE_ENGINE_VERSION.getValue()); @@ -69,8 +71,13 @@ private static void init() { EngineType.FLINK().toString(), LabelCommonConfig.FLINK_ENGINE_VERSION.getValue()); defaultVersion.put( EngineType.PRESTO().toString(), LabelCommonConfig.PRESTO_ENGINE_VERSION.getValue()); + + defaultVersion.put( + EngineType.HBASE().toString(), LabelCommonConfig.HBASE_ENGINE_VERSION.getValue()); defaultVersion.put( EngineType.NEBULA().toString(), LabelCommonConfig.NEBULA_ENGINE_VERSION.getValue()); + defaultVersion.put( + EngineType.DORIS().toString(), LabelCommonConfig.DORIS_ENGINE_VERSION.getValue()); defaultVersion.put( EngineType.SQOOP().toString(), LabelCommonConfig.SQOOP_ENGINE_VERSION.getValue()); defaultVersion.put( diff --git a/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/utils/LabelUtils.java b/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/utils/LabelUtils.java index 52c41e5f69..8fd8f07131 100644 --- a/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/utils/LabelUtils.java +++ b/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/utils/LabelUtils.java @@ -213,7 +213,8 @@ public static T fromJson(String json, Class tClass, Class... parameter + Class.class + ", ...: " + Class.class - + ")"); + + ")", + e); return null; } } diff --git a/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/scala/org/apache/linkis/manager/label/utils/LabelUtil.scala b/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/scala/org/apache/linkis/manager/label/utils/LabelUtil.scala index 557104ff32..b221424088 100644 --- a/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/scala/org/apache/linkis/manager/label/utils/LabelUtil.scala +++ b/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/scala/org/apache/linkis/manager/label/utils/LabelUtil.scala @@ -17,11 +17,13 @@ package org.apache.linkis.manager.label.utils +import org.apache.linkis.manager.label.constant.LabelValueConstant import org.apache.linkis.manager.label.entity.{Label, TenantLabel} import org.apache.linkis.manager.label.entity.engine.{ CodeLanguageLabel, EngineConnModeLabel, EngineTypeLabel, + EngingeConnRuntimeModeLabel, UserCreatorLabel } import org.apache.linkis.manager.label.entity.entrance.{ @@ -96,6 +98,10 @@ object LabelUtil { getLabelFromList[TenantLabel](labels) } + def getEngingeConnRuntimeModeLabel(labels: util.List[Label[_]]): EngingeConnRuntimeModeLabel = { + getLabelFromList[EngingeConnRuntimeModeLabel](labels) + } + def getEngineConnModeLabel(labels: util.List[Label[_]]): EngineConnModeLabel = { getLabelFromList[EngineConnModeLabel](labels) } @@ -146,6 +152,15 @@ object LabelUtil { null.asInstanceOf[A] } + def isYarnClusterMode(labels: util.List[Label[_]]): Boolean = { + val label = LabelUtil.getEngingeConnRuntimeModeLabel(labels) + val isYarnClusterMode: Boolean = { + if (null != label && label.getModeValue.equals(LabelValueConstant.YARN_CLUSTER_VALUE)) true + else false + } + isYarnClusterMode + } + def getFromLabelStr(labelStr: String, key: String): String = { // hadoop-IDE,hive-2.3.3 or hadoop-IDE or hive-2.3.3 if (StringUtils.isNotBlank(labelStr)) { diff --git a/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/conf/RMConfiguration.java b/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/conf/RMConfiguration.java new file mode 100644 index 0000000000..a28736d630 --- /dev/null +++ b/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/conf/RMConfiguration.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.manager.common.conf; + +import org.apache.linkis.common.conf.ByteType; +import org.apache.linkis.common.conf.CommonVars; +import org.apache.linkis.common.conf.TimeType; + +public class RMConfiguration { + + public static final CommonVars RM_WAIT_EVENT_TIME_OUT = + CommonVars.apply("wds.linkis.rm.wait.event.time.out", 1000 * 60 * 12L); + + public static final CommonVars LOCK_RELEASE_TIMEOUT = + CommonVars.apply("wds.linkis.manager.rm.lock.release.timeout", new TimeType("5m")); + + public static final CommonVars LOCK_RELEASE_CHECK_INTERVAL = + CommonVars.apply("wds.linkis.manager.rm.lock.release.check.interval", new TimeType("5m")); + + // Resource parameter(资源参数) + public static final CommonVars USER_AVAILABLE_CPU = + CommonVars.apply("wds.linkis.rm.client.core.max", 10); + public static final CommonVars USER_AVAILABLE_MEMORY = + CommonVars.apply("wds.linkis.rm.client.memory.max", new ByteType("20g")); + public static final CommonVars USER_AVAILABLE_INSTANCE = + CommonVars.apply("wds.linkis.rm.instance", 10); + + public static final CommonVars USER_AVAILABLE_YARN_INSTANCE_CPU = + CommonVars.apply("wds.linkis.rm.yarnqueue.cores.max", 150); + + public static final CommonVars USER_AVAILABLE_YARN_INSTANCE_MEMORY = + CommonVars.apply("wds.linkis.rm.yarnqueue.memory.max", new ByteType("450g")); + + public static final CommonVars USER_AVAILABLE_YARN_INSTANCE = + CommonVars.apply("wds.linkis.rm.yarnqueue.instance.max", 30); + public static final CommonVars USER_AVAILABLE_YARN_QUEUE_NAME = + CommonVars.apply("wds.linkis.rm.yarnqueue", "default"); + public static final CommonVars USER_AVAILABLE_CLUSTER_NAME = + CommonVars.apply("wds.linkis.rm.cluster", "default"); + + public static final CommonVars USER_AVAILABLE_KUBERNETES_INSTANCE_CPU = + CommonVars.apply("wds.linkis.rm.kubernetes.cores.max", 150000); + public static final CommonVars USER_AVAILABLE_KUBERNETES_INSTANCE_MEMORY = + CommonVars.apply("wds.linkis.rm.kubernetes.memory.max", new ByteType("450g")); + public static final CommonVars USER_AVAILABLE_KUBERNETES_INSTANCE_NAMESPACE = + CommonVars.apply("wds.linkis.rm.kubernetes.namespace", "default"); + + public static final CommonVars RM_ENGINE_SCAN_INTERVAL = + CommonVars.apply("wds.linkis.rm.engine.scan.interval", 120000L); + + public static final CommonVars DEFAULT_YARN_CLUSTER_NAME = + CommonVars.apply("wds.linkis.rm.default.yarn.cluster.name", "default"); + public static final CommonVars DEFAULT_YARN_TYPE = + CommonVars.apply("wds.linkis.rm.default.yarn.cluster.type", "Yarn"); + public static final CommonVars DEFAULT_KUBERNETES_CLUSTER_NAME = + CommonVars.apply("wds.linkis.rm.default.kubernetes.cluster.name", "default"); + public static final CommonVars DEFAULT_KUBERNETES_TYPE = + CommonVars.apply("wds.linkis.rm.default.kubernetes.cluster.type", "K8S"); + public static final CommonVars EXTERNAL_RETRY_NUM = + CommonVars.apply("wds.linkis.rm.external.retry.num", 3); +} diff --git a/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/constant/AMConstant.java b/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/constant/AMConstant.java index 4adf6510e4..0128adfbb8 100644 --- a/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/constant/AMConstant.java +++ b/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/constant/AMConstant.java @@ -29,6 +29,8 @@ public class AMConstant { public static final String PROCESS_MARK = "process"; + public static final String CLUSTER_PROCESS_MARK = "cluster_process"; + public static final String THREAD_MARK = "thread"; public static final String START_REASON = "start_reason"; diff --git a/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/constant/RMConstant.java b/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/constant/RMConstant.java index 635bd5af06..209cf3e601 100644 --- a/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/constant/RMConstant.java +++ b/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/constant/RMConstant.java @@ -27,6 +27,9 @@ public class RMConstant { public static final String CPU_UNIT = "cores"; + public static final String KUBERNETES_CPU_UNIT = "milli cores"; + public static final String KUBERNETES_NAMESPACE = "namespace"; + public static final String KUBERNETES_NAMESPACE_UNIT = ""; public static final String MEMORY_UNIT_BYTE = "bytes"; public static final String INSTANCE_UNIT = ""; diff --git a/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/entity/node/AMEMNode.java b/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/entity/node/AMEMNode.java index f2c2848c22..3be0c7d6bf 100644 --- a/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/entity/node/AMEMNode.java +++ b/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/entity/node/AMEMNode.java @@ -30,7 +30,7 @@ public class AMEMNode implements EMNode, ScoreServiceInstance { - private List + + + eureka + + true + + discovery + eureka + + + + + org.springframework.cloud + spring-cloud-starter-netflix-eureka-client + + + javax.ws.rs + jsr311-api + + + org.springframework.boot + spring-boot-autoconfigure + + + org.springframework.boot + spring-boot-starter-aop + + + org.springframework.cloud + spring-cloud-starter + + + org.springframework.cloud + spring-cloud-commons + + + org.springframework.cloud + spring-cloud-context + + + org.springframework.boot + spring-boot-starter + + + org.springframework.boot + spring-boot-starter-cache + + + com.fasterxml.jackson.core + jackson-core + + + com.fasterxml.jackson.core + jackson-databind + + + io.github.x-stream + mxparser + + + + + + + nacos + + + discovery + nacos + + + + + com.alibaba.cloud + spring-cloud-starter-alibaba-nacos-discovery + ${spring-cloud-alibaba.version} + + + org.springframework.boot + * + + + org.springframework.cloud + spring-cloud-commons + + + org.springframework.cloud + spring-cloud-context + + + org.springframework.boot + spring-boot-starter + + + com.fasterxml.jackson.core + jackson-core + + + com.fasterxml.jackson.core + jackson-databind + + + com.google.code.findbugs + jsr305 + + + org.yaml + snakeyaml + + + io.prometheus + simpleclient + + + com.google.guava + guava + + + + + + + diff --git a/linkis-commons/linkis-module/src/main/scala/org/apache/linkis/server/conf/ServerConfiguration.scala b/linkis-commons/linkis-module/src/main/scala/org/apache/linkis/server/conf/ServerConfiguration.scala index 4043e86eea..ed6c680648 100644 --- a/linkis-commons/linkis-module/src/main/scala/org/apache/linkis/server/conf/ServerConfiguration.scala +++ b/linkis-commons/linkis-module/src/main/scala/org/apache/linkis/server/conf/ServerConfiguration.scala @@ -207,4 +207,7 @@ object ServerConfiguration extends Logging { val LINKIS_SERVER_SESSION_PROXY_TICKETID_KEY = CommonVars("wds.linkis.session.proxy.user.ticket.key", "linkis_user_session_proxy_ticket_id_v1") + val LINKIS_SERVER_ENTRANCE_HEADER_KEY = + CommonVars("linkis.server.entrance.header.key", "jobInstanceKey") + } diff --git a/linkis-commons/linkis-module/src/main/scala/org/apache/linkis/server/ticket/RedisClient.scala b/linkis-commons/linkis-module/src/main/scala/org/apache/linkis/server/ticket/RedisClient.scala index cdd1493178..73699f38ef 100644 --- a/linkis-commons/linkis-module/src/main/scala/org/apache/linkis/server/ticket/RedisClient.scala +++ b/linkis-commons/linkis-module/src/main/scala/org/apache/linkis/server/ticket/RedisClient.scala @@ -98,8 +98,8 @@ object RedisClient { maxIdle: Int, minIdle: Int, maxWaitMillis: Long - ): GenericObjectPoolConfig[Nothing] = { - val poolConfig = new GenericObjectPoolConfig + ): GenericObjectPoolConfig[Jedis] = { + val poolConfig = new GenericObjectPoolConfig[Jedis]() poolConfig.setMaxTotal(maxTotal) poolConfig.setMaxIdle(maxIdle) poolConfig.setMinIdle(minIdle) diff --git a/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/protocol/job/JobReqProcotol.scala b/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/protocol/job/JobReqProcotol.scala index 2d730a5cf8..829a967aab 100644 --- a/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/protocol/job/JobReqProcotol.scala +++ b/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/protocol/job/JobReqProcotol.scala @@ -52,3 +52,10 @@ class RequestOneJob extends JobReq { } case class RequestAllJob(instance: String) extends JobReq + +case class RequestFailoverJob( + reqMap: util.Map[String, java.lang.Long], + statusList: util.List[String], + startTimestamp: Long, + limit: Int = 10 +) extends JobReq diff --git a/linkis-computation-governance/linkis-entrance/src/main/assembly/distribution.xml b/linkis-computation-governance/linkis-entrance/src/main/assembly/distribution.xml index 306cb71fdb..313506fd08 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/assembly/distribution.xml +++ b/linkis-computation-governance/linkis-entrance/src/main/assembly/distribution.xml @@ -245,6 +245,9 @@ org.slf4j:jul-to-slf4j:jar org.slf4j:slf4j-api:jar org.springframework.boot:spring-boot:jar + org.springframework.boot:spring-boot-actuator:jar + org.springframework.boot:spring-boot-actuator-autoconfigure:jar + org.springframework.boot:spring-boot-autoconfigure:jar org.springframework.boot:spring-boot-starter:jar org.springframework.boot:spring-boot-starter-actuator:jar org.springframework.boot:spring-boot-starter-aop:jar diff --git a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/conf/EntranceSpringConfiguration.java b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/conf/EntranceSpringConfiguration.java index ff4f28f407..f5f61604f0 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/conf/EntranceSpringConfiguration.java +++ b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/conf/EntranceSpringConfiguration.java @@ -190,9 +190,7 @@ public SchedulerContext schedulerContext( @Bean @ConditionalOnMissingBean public ExecutorManager executorManager(GroupFactory groupFactory) { - EngineConnManagerBuilder engineConnManagerBuilder = EngineConnManagerBuilder$.MODULE$.builder(); - engineConnManagerBuilder.setPolicy(Policy.Process); - return new EntranceExecutorManagerImpl(groupFactory, engineConnManagerBuilder.build()); + return new EntranceExecutorManagerImpl(groupFactory); } @Bean diff --git a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/constant/ServiceNameConsts.java b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/constant/ServiceNameConsts.java index cb37279c11..bee17b8ed4 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/constant/ServiceNameConsts.java +++ b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/constant/ServiceNameConsts.java @@ -26,4 +26,6 @@ private ServiceNameConsts() {} public static final String ENTRANCE_SERVER = "entranceServer"; public static final String ENTRANCE_INTERCEPTOR = "entranceInterceptors"; + + public static final String ENTRANCE_FAILOVER_SERVER = "entranceFailoverServer"; } diff --git a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/errorcode/EntranceErrorCodeSummary.java b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/errorcode/EntranceErrorCodeSummary.java index f92083df0c..51a522d3d2 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/errorcode/EntranceErrorCodeSummary.java +++ b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/errorcode/EntranceErrorCodeSummary.java @@ -76,7 +76,11 @@ public enum EntranceErrorCodeSummary implements LinkisErrorCode { SHELL_BLACKLISTED_CODE(50081, "Shell code contains blacklisted code(shell中包含黑名单代码)"), JOB_HISTORY_FAILED_ID(50081, ""), - LOGPATH_NOT_NULL(20301, "The logPath cannot be empty(日志路径不能为空)"); + LOGPATH_NOT_NULL(20301, "The logPath cannot be empty(日志路径不能为空)"), + + FAILOVER_RUNNING_TO_CANCELLED( + 30001, + "Job {0} failover, status changed from Running to Cancelled (任务故障转移，状态从Running变更为Cancelled)"); /** (errorCode)错误码 */ private final int errorCode; diff --git a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/persistence/QueryPersistenceManager.java b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/persistence/QueryPersistenceManager.java index da187cddcb..b912b58ebb 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/persistence/QueryPersistenceManager.java +++ b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/persistence/QueryPersistenceManager.java @@ -191,7 +191,7 @@ private void updateJobStatus(Job job) { createPersistenceEngine().updateIfNeeded(jobRequest); } catch (ErrorException e) { entranceContext.getOrCreateLogManager().onLogUpdate(job, e.getMessage()); - logger.error("update job status failed, reason: ", e); + throw e; } } diff --git a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/server/DefaultEntranceServer.java b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/server/DefaultEntranceServer.java index fac85f32c1..7c38d27947 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/server/DefaultEntranceServer.java +++ b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/server/DefaultEntranceServer.java @@ -20,13 +20,17 @@ import org.apache.linkis.common.ServiceInstance; import org.apache.linkis.entrance.EntranceContext; import org.apache.linkis.entrance.EntranceServer; +import org.apache.linkis.entrance.conf.EntranceConfiguration; import org.apache.linkis.entrance.conf.EntranceConfiguration$; import org.apache.linkis.entrance.constant.ServiceNameConsts; import org.apache.linkis.entrance.execute.EntranceJob; +import org.apache.linkis.entrance.job.EntranceExecutionJob; import org.apache.linkis.entrance.log.LogReader; import org.apache.linkis.governance.common.protocol.conf.EntranceInstanceConfRequest; import org.apache.linkis.rpc.Sender; +import org.apache.commons.io.IOUtils; + import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.event.ContextClosedEvent; import org.springframework.context.event.EventListener; @@ -63,6 +67,7 @@ public void init() { private void cleanUpEntranceDirtyData() { if ((Boolean) EntranceConfiguration$.MODULE$.ENABLE_ENTRANCE_DIRTY_DATA_CLEAR().getValue()) { + logger.info("start to clean up entrance dirty data."); Sender sender = Sender.getSender( EntranceConfiguration$.MODULE$.JOBHISTORY_SPRING_APPLICATION_NAME().getValue()); @@ -93,13 +98,19 @@ private void shutdownEntrance(ContextClosedEvent event) { if (shutdownFlag) { logger.warn("event has been handled"); } else { + if (EntranceConfiguration.ENTRANCE_SHUTDOWN_FAILOVER_CONSUME_QUEUE_ENABLED()) { + logger.warn("Entrance exit to update and clean all ConsumeQueue task instances"); + updateAllNotExecutionTaskInstances(false); + } + logger.warn("Entrance exit to stop all job"); - EntranceJob[] allUndoneJobs = getAllUndoneTask(null, null); - if (null != allUndoneJobs) { - for (EntranceJob job : allUndoneJobs) { + EntranceJob[] allUndoneTask = getAllUndoneTask(null, null); + if (null != allUndoneTask) { + for (EntranceJob job : allUndoneTask) { job.onFailure( "Your job will be marked as canceled because the Entrance service restarted(因为Entrance服务重启，您的任务将被标记为取消)", null); + IOUtils.closeQuietly(((EntranceExecutionJob) job).getLogWriter().get()); } } } diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/conf/EntranceConfiguration.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/conf/EntranceConfiguration.scala index a23bac28a5..a4082f7e77 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/conf/EntranceConfiguration.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/conf/EntranceConfiguration.scala @@ -229,12 +229,58 @@ object EntranceConfiguration { CommonVars("linkis.entrance.creator.job.concurrency.limit.conf.cache.time", 30L) val ENTRANCE_TASK_TIMEOUT = - CommonVars("wds.linkis.entrance.task.timeout", new TimeType("48h")) + CommonVars("linkis.entrance.task.timeout", new TimeType("48h")) val ENTRANCE_TASK_TIMEOUT_SCAN = - CommonVars("wds.linkis.entrance.task.timeout.scan", new TimeType("12h")) + CommonVars("linkis.entrance.task.timeout.scan", new TimeType("12h")) val ENABLE_HDFS_JVM_USER = CommonVars[Boolean]("linkis.entrance.enable.hdfs.jvm.user", true).getValue + val ENTRANCE_FAILOVER_ENABLED = CommonVars("linkis.entrance.failover.enable", false).getValue + + val ENTRANCE_FAILOVER_SCAN_INIT_TIME = + CommonVars("linkis.entrance.failover.scan.init.time", 3 * 1000).getValue + + val ENTRANCE_FAILOVER_SCAN_INTERVAL = + CommonVars("linkis.entrance.failover.scan.interval", 30 * 1000).getValue + + val ENTRANCE_FAILOVER_DATA_NUM_LIMIT = + CommonVars("linkis.entrance.failover.data.num.limit", 10).getValue + + val ENTRANCE_FAILOVER_DATA_INTERVAL_TIME = + CommonVars("linkis.entrance.failover.data.interval.time", new TimeType("1d").toLong).getValue + + // if true, the waitForRetry job in runningJobs can be failover + val ENTRANCE_FAILOVER_RETRY_JOB_ENABLED = + CommonVars("linkis.entrance.failover.retry.job.enable", false) + + val ENTRANCE_UPDATE_BATCH_SIZE = CommonVars("linkis.entrance.update.batch.size", 100) + + // if true, the job in ConsumeQueue can be failover + val ENTRANCE_SHUTDOWN_FAILOVER_CONSUME_QUEUE_ENABLED = + CommonVars("linkis.entrance.shutdown.failover.consume.queue.enable", true).getValue + + val ENTRANCE_GROUP_SCAN_ENABLED = CommonVars("linkis.entrance.group.scan.enable", false) + + val ENTRANCE_GROUP_SCAN_INIT_TIME = CommonVars("linkis.entrance.group.scan.init.time", 3 * 1000) + + val ENTRANCE_GROUP_SCAN_INTERVAL = CommonVars("linkis.entrance.group.scan.interval", 60 * 1000) + + val ENTRANCE_FAILOVER_RETAIN_METRIC_ENGINE_CONN_ENABLED = + CommonVars("linkis.entrance.failover.retain.metric.engine.conn.enable", false) + + val ENTRANCE_FAILOVER_RETAIN_METRIC_YARN_RESOURCE_ENABLED = + CommonVars("linkis.entrance.failover.retain.metric.yarn.resource.enable", false) + + // if true, job whose status is running will be set to Cancelled + val ENTRANCE_FAILOVER_RUNNING_KILL_ENABLED = + CommonVars("linkis.entrance.failover.running.kill.enable", false) + + val LINKIS_ENTRANCE_SKIP_ORCHESTRATOR = + CommonVars("linkis.entrance.skip.orchestrator", false).getValue + + val ENABLE_HDFS_RES_DIR_PRIVATE = + CommonVars[Boolean]("linkis.entrance.enable.hdfs.res.dir.private", false).getValue + } diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/execute/EntranceExecutorManager.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/execute/EntranceExecutorManager.scala index 1560ffb947..e5248b056e 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/execute/EntranceExecutorManager.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/execute/EntranceExecutorManager.scala @@ -19,15 +19,16 @@ package org.apache.linkis.entrance.execute import org.apache.linkis.common.exception.WarnException import org.apache.linkis.common.utils.{Logging, Utils} +import org.apache.linkis.entrance.conf.EntranceConfiguration import org.apache.linkis.entrance.errorcode.EntranceErrorCodeSummary._ import org.apache.linkis.entrance.exception.EntranceErrorException +import org.apache.linkis.entrance.execute.simple.{SimpleEntranceExecutor, SimpleExecuteBusContext} import org.apache.linkis.governance.common.entity.job.JobRequest import org.apache.linkis.scheduler.executer.{Executor, ExecutorManager} import org.apache.linkis.scheduler.queue.{GroupFactory, Job, SchedulerEvent} import java.util.Date import java.util.concurrent.atomic.AtomicLong - import scala.concurrent.duration.Duration abstract class EntranceExecutorManager(groupFactory: GroupFactory) @@ -87,7 +88,14 @@ abstract class EntranceExecutorManager(groupFactory: GroupFactory) job.getJobRequest match { case jobReq: JobRequest => val entranceEntranceExecutor = - new DefaultEntranceExecutor(jobReq.getId) + if (EntranceConfiguration.LINKIS_ENTRANCE_SKIP_ORCHESTRATOR) { + new SimpleEntranceExecutor( + jobReq.getId, + SimpleExecuteBusContext.getOrchestratorListenerBusContext() + ) + } else { + new DefaultEntranceExecutor(jobReq.getId) + } jobReq.setUpdatedTime(new Date(System.currentTimeMillis())) entranceEntranceExecutor case _ => diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/execute/impl/EntranceExecutorManagerImpl.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/execute/impl/EntranceExecutorManagerImpl.scala index 89fcaa7b22..57de9cc0cb 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/execute/impl/EntranceExecutorManagerImpl.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/execute/impl/EntranceExecutorManagerImpl.scala @@ -22,7 +22,7 @@ import org.apache.linkis.orchestrator.ecm.EngineConnManager import org.apache.linkis.scheduler.listener.ExecutorListener import org.apache.linkis.scheduler.queue.GroupFactory -class EntranceExecutorManagerImpl(groupFactory: GroupFactory, engineConnManager: EngineConnManager) +class EntranceExecutorManagerImpl(groupFactory: GroupFactory) extends EntranceExecutorManager(groupFactory) { override def setExecutorListener(engineListener: ExecutorListener): Unit = {} diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/execute/simple/SimpleASyncListener.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/execute/simple/SimpleASyncListener.scala new file mode 100644 index 0000000000..bc52fbd800 --- /dev/null +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/execute/simple/SimpleASyncListener.scala @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.entrance.execute.simple + +import org.apache.linkis.common.listener.Event +import org.apache.linkis.common.utils.Logging +import org.apache.linkis.entrance.EntranceServer +import org.apache.linkis.entrance.conf.EntranceConfiguration +import org.apache.linkis.orchestrator.listener.OrchestratorAsyncEvent +import org.apache.linkis.orchestrator.listener.task.{ + TaskLogEvent, + TaskLogListener, + TaskProgressListener, + TaskRunningInfoEvent +} + +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.stereotype.Component + +import javax.annotation.PostConstruct + +@Component +class SimpleASyncListener extends TaskLogListener with TaskProgressListener with Logging { + + @Autowired private var entranceServer: EntranceServer = _ + + @PostConstruct + def init(): Unit = { + if (EntranceConfiguration.LINKIS_ENTRANCE_SKIP_ORCHESTRATOR) { + SimpleExecuteBusContext + .getOrchestratorListenerBusContext() + .getOrchestratorAsyncListenerBus + .addListener(this) + } + } + + override def onLogUpdate(taskLogEvent: TaskLogEvent): Unit = {} + + override def onProgressOn(taskProgressEvent: TaskRunningInfoEvent): Unit = {} + + override def onEvent(event: OrchestratorAsyncEvent): Unit = {} + + override def onEventError(event: Event, t: Throwable): Unit = {} +} diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/execute/simple/SimpleEntranceExecutor.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/execute/simple/SimpleEntranceExecutor.scala new file mode 100644 index 0000000000..d9e18081d2 --- /dev/null +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/execute/simple/SimpleEntranceExecutor.scala @@ -0,0 +1,155 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.entrance.execute.simple + +import org.apache.linkis.common.utils.{Logging, Utils} +import org.apache.linkis.entrance.exception.{EntranceErrorCode, EntranceErrorException} +import org.apache.linkis.entrance.execute.{EngineExecuteAsyncReturn, EntranceExecutor} +import org.apache.linkis.entrance.job.EntranceExecuteRequest +import org.apache.linkis.governance.common.utils.LoggerUtils +import org.apache.linkis.manager.label.entity.Label +import org.apache.linkis.manager.label.entity.engine.CodeLanguageLabel +import org.apache.linkis.manager.label.utils.LabelUtil +import org.apache.linkis.orchestrator.code.plans.ast.CodeJob +import org.apache.linkis.orchestrator.code.plans.logical.CodeLogicalUnitTaskDesc +import org.apache.linkis.orchestrator.computation.entity.ComputationJobReq +import org.apache.linkis.orchestrator.computation.physical.CodeLogicalUnitExecTask +import org.apache.linkis.orchestrator.converter.ASTContextImpl +import org.apache.linkis.orchestrator.execution.{ + AsyncTaskResponse, + FailedTaskResponse, + SucceedTaskResponse +} +import org.apache.linkis.orchestrator.listener.OrchestratorListenerBusContext +import org.apache.linkis.orchestrator.plans.physical.{ExecTask, PhysicalContextImpl} +import org.apache.linkis.orchestrator.plans.unit.CodeLogicalUnit +import org.apache.linkis.scheduler.executer._ + +import java.util + +class SimpleEntranceExecutor( + id: Long, + orchestratorListenerBusContext: OrchestratorListenerBusContext +) extends EntranceExecutor(id) + with SingleTaskOperateSupport + with Logging { + + private var codeUnitExecTask: CodeLogicalUnitExecTask = null + + override protected def callExecute(request: ExecuteRequest): ExecuteResponse = { + val entranceExecuteRequest: EntranceExecuteRequest = request match { + case request: EntranceExecuteRequest => + request + case _ => + throw new EntranceErrorException( + EntranceErrorCode.EXECUTE_REQUEST_INVALID.getErrCode, + s"Invalid entranceExecuteRequest : ${request.code}" + ) + } + // 1. create JobReq + val computationJobReq = requestToComputationJobReq(entranceExecuteRequest) + // 2. create code job + val codeJob = new CodeJob(null, null) + val astContext = ASTContextImpl.newBuilder().setJobReq(computationJobReq).build() + codeJob.setAstContext(astContext) + codeJob.setCodeLogicalUnit(computationJobReq.getCodeLogicalUnit) + codeJob.setParams(computationJobReq.getParams) + codeJob.setName(computationJobReq.getName + "_Job") + codeJob.setSubmitUser(computationJobReq.getSubmitUser) + codeJob.setExecuteUser(computationJobReq.getExecuteUser) + codeJob.setLabels(computationJobReq.getLabels) + codeJob.setPriority(computationJobReq.getPriority) + codeUnitExecTask = new CodeLogicalUnitExecTask(Array[ExecTask](), Array[ExecTask]()) + // set job id, can find by getEntranceContext.getOrCreateScheduler().get(execId).map(_.asInstanceOf[Job]) + codeUnitExecTask.setId(entranceExecuteRequest.getJob.getId) + // 3.set code unit + codeUnitExecTask.setCodeLogicalUnit(computationJobReq.getCodeLogicalUnit) + codeUnitExecTask.setTaskDesc(CodeLogicalUnitTaskDesc(codeJob)) + // 4. set context + val context = new PhysicalContextImpl(codeUnitExecTask, Array.empty) + context.setSyncBus(orchestratorListenerBusContext.getOrchestratorSyncListenerBus) + context.setAsyncBus(orchestratorListenerBusContext.getOrchestratorAsyncListenerBus) + // 5. execute + val response = codeUnitExecTask.execute() + response match { + case async: AsyncTaskResponse => + new EngineExecuteAsyncReturn(request, null) + case succeed: SucceedTaskResponse => + logger.info(s"Succeed to execute ExecTask(${getId})") + SuccessExecuteResponse() + case failedTaskResponse: FailedTaskResponse => + logger.info(s"Failed to execute ExecTask(${getId})") + ErrorExecuteResponse(failedTaskResponse.getErrorMsg, failedTaskResponse.getCause) + case _ => + logger.warn(s"ExecTask(${getId}) need to retry") + ErrorExecuteResponse("unknown response: " + response, null) + } + } + + def requestToComputationJobReq( + entranceExecuteRequest: EntranceExecuteRequest + ): ComputationJobReq = { + val jobReqBuilder = ComputationJobReq.newBuilder() + jobReqBuilder.setId(entranceExecuteRequest.jobId()) + jobReqBuilder.setSubmitUser(entranceExecuteRequest.submitUser()) + jobReqBuilder.setExecuteUser(entranceExecuteRequest.executeUser()) + val codeTypeLabel: Label[_] = LabelUtil.getCodeTypeLabel(entranceExecuteRequest.getLabels) + if (null == codeTypeLabel) { + throw new EntranceErrorException( + EntranceErrorCode.EXECUTE_REQUEST_INVALID.getErrCode, + s"code Type Label is needed" + ) + } + val codes = new util.ArrayList[String]() + codes.add(entranceExecuteRequest.code()) + val codeLogicalUnit = + new CodeLogicalUnit(codes, codeTypeLabel.asInstanceOf[CodeLanguageLabel]) + jobReqBuilder.setCodeLogicalUnit(codeLogicalUnit) + jobReqBuilder.setLabels(entranceExecuteRequest.getLabels) + jobReqBuilder.setExecuteUser(entranceExecuteRequest.executeUser()) + jobReqBuilder.setParams(entranceExecuteRequest.properties()) + jobReqBuilder.build().asInstanceOf[ComputationJobReq] + } + + override def kill(): Boolean = { + LoggerUtils.setJobIdMDC(getId.toString) + logger.info("Entrance start to kill job {} invoke Orchestrator ", this.getId) + Utils.tryAndWarn { + if (null != codeUnitExecTask) { + codeUnitExecTask.kill() + } + } + LoggerUtils.removeJobIdMDC() + true + } + + override def pause(): Boolean = { + true + } + + override def resume(): Boolean = { + true + } + + override def close(): Unit = { + getEngineExecuteAsyncReturn.foreach { e => + e.notifyError(s"$toString has already been completed with state $state.") + } + } + +} diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/execute/simple/SimpleSyncListener.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/execute/simple/SimpleSyncListener.scala new file mode 100644 index 0000000000..46107ff701 --- /dev/null +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/execute/simple/SimpleSyncListener.scala @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.entrance.execute.simple + +import org.apache.linkis.common.listener.Event +import org.apache.linkis.common.utils.Logging +import org.apache.linkis.entrance.EntranceServer +import org.apache.linkis.entrance.conf.EntranceConfiguration +import org.apache.linkis.orchestrator.listener.OrchestratorSyncEvent +import org.apache.linkis.orchestrator.listener.task.{ + TaskErrorResponseEvent, + TaskResultSetEvent, + TaskResultSetListener, + TaskResultSetSizeEvent, + TaskStatusEvent, + TaskStatusListener +} + +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.stereotype.Component + +import javax.annotation.PostConstruct + +/** + * 1.TaskLogListener: job.getLogListener.foreach(_.onLogUpdate(job, logEvent.log)) + * + * 2.TaskProgressListener: entranceJob.getProgressListener.foreach( _.onProgressUpdate(entranceJob, + * progressInfoEvent.progress, entranceJob.getProgressInfo) + * + * 3.TaskResultSetListener entranceContext.getOrCreatePersistenceManager().onResultSizeCreated(j, + * taskResultSize.resultSize) .getOrCreatePersistenceManager() .onResultSetCreated( + * entranceExecuteRequest.getJob, AliasOutputExecuteResponse(firstResultSet.alias, + * firstResultSet.result) ) + * + * 4. TaskStatusListener getEngineExecuteAsyncReturn.foreach { jobReturn => jobReturn.notifyStatus( + * ResponseTaskStatus(entranceExecuteRequest.getJob.getId, ExecutionNodeStatus.Succeed) ) } val msg + * = failedResponse.getErrorCode + ", " + failedResponse.getErrorMsg + * getEngineExecuteAsyncReturn.foreach { jobReturn => jobReturn.notifyError(msg, + * failedResponse.getCause) jobReturn.notifyStatus( + * ResponseTaskStatus(entranceExecuteRequest.getJob.getId, ExecutionNodeStatus.Failed) ) } + */ +@Component +class SimpleSyncListener extends TaskStatusListener with TaskResultSetListener with Logging { + + @Autowired private var entranceServer: EntranceServer = _ + + @PostConstruct + def init(): Unit = { + if (EntranceConfiguration.LINKIS_ENTRANCE_SKIP_ORCHESTRATOR) { + SimpleExecuteBusContext + .getOrchestratorListenerBusContext() + .getOrchestratorSyncListenerBus + .addListener(this) + } + } + + override def onStatusUpdate(taskStatusEvent: TaskStatusEvent): Unit = {} + + override def onTaskErrorResponseEvent(taskErrorResponseEvent: TaskErrorResponseEvent): Unit = {} + + override def onResultSetCreate(taskResultSetEvent: TaskResultSetEvent): Unit = {} + + override def onResultSizeCreated(taskResultSetSizeEvent: TaskResultSetSizeEvent): Unit = {} + + override def onSyncEvent(event: OrchestratorSyncEvent): Unit = {} + + override def onEventError(event: Event, t: Throwable): Unit = {} +} diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/interceptor/OnceJobInterceptor.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/interceptor/OnceJobInterceptor.scala index 1291a8566c..9b05789800 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/interceptor/OnceJobInterceptor.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/interceptor/OnceJobInterceptor.scala @@ -94,8 +94,8 @@ class OnceJobInterceptor extends EntranceInterceptor { s"/tmp/${task.getExecuteUser}/${task.getId}" protected def getJobContent(task: JobRequest): util.Map[String, AnyRef] = { - // TODO Wait for optimizing since the class `JobRequest` is waiting for optimizing . val jobContent = new util.HashMap[String, AnyRef] + jobContent.putAll(TaskUtils.getStartupMap(task.getParams)) jobContent.put(TaskConstant.CODE, task.getExecutionCode) task.getLabels.foreach { case label: CodeLanguageLabel => diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/interceptor/impl/CommentInterceptor.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/interceptor/impl/CommentInterceptor.scala index 69b96cfe24..34bd6ead01 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/interceptor/impl/CommentInterceptor.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/interceptor/impl/CommentInterceptor.scala @@ -103,22 +103,7 @@ object ScalaCommentHelper extends CommentHelper { private val scalaCommentPattern: String = "(?ms)([\"'](?:|[^'])*['\"])|//.*?$|/\\*.*?\\*/" override def dealComment(code: String): String = code -// private val logger: Logger = LoggerFactory.getLogger(getClass) -// -// override def dealComment(code: String): String = { -// try { -// val p = Pattern.compile(scalaCommentPattern) -// val sql = p.matcher(code).replaceAll("$1") -// sql -// } catch { -// case e: Exception => -// logger.warn("scala comment failed") -// code -// case t: Throwable => -// logger.warn("scala comment failed") -// code -// } -// } + } object CommentMain { diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/interceptor/impl/Explain.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/interceptor/impl/Explain.scala index 413f94b347..3d9645581c 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/interceptor/impl/Explain.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/interceptor/impl/Explain.scala @@ -284,8 +284,8 @@ object SQLExplain extends Explain { } var overLimit: Boolean = false var code = cmd.trim - if (code.toLowerCase(Locale.getDefault).contains("limit")) { - code = code.substring(code.toLowerCase(Locale.getDefault).lastIndexOf("limit")).trim + if (code.toLowerCase(Locale.getDefault).contains(LIMIT)) { + code = code.substring(code.toLowerCase((Locale.getDefault)).lastIndexOf(LIMIT)).trim } val hasLimit = code.toLowerCase().matches("limit\\s+\\d+\\s*;?") if (hasLimit) { @@ -310,13 +310,14 @@ object SQLExplain extends Explain { * String */ def repairSelectOverLimit(cmd: String): String = { - var code = cmd.trim + val code = cmd.trim var preCode = "" var tailCode = "" - var limitNum = SQL_DEFAULT_LIMIT.getValue - if (code.toLowerCase(Locale.getDefault).contains("limit")) { - preCode = code.substring(0, code.toLowerCase(Locale.getDefault).lastIndexOf("limit")).trim - tailCode = code.substring(code.toLowerCase(Locale.getDefault).lastIndexOf("limit")).trim + val limitNum = SQL_DEFAULT_LIMIT.getValue + val lowerCaseCode = code.toLowerCase(Locale.getDefault) + if (lowerCaseCode.contains(LIMIT)) { + preCode = code.substring(0, lowerCaseCode.lastIndexOf(LIMIT)).trim + tailCode = code.substring(lowerCaseCode.lastIndexOf(LIMIT)).trim } if (isUpperSelect(cmd)) preCode + " LIMIT " + limitNum else preCode + " limit " + limitNum } @@ -395,24 +396,27 @@ object PythonExplain extends Explain { }) code.split(System.lineSeparator()) foreach { code => - if (IMPORT_SYS_MOUDLE.findAllIn(code).nonEmpty || FROM_SYS_IMPORT.findAllIn(code).nonEmpty) + if (IMPORT_SYS_MOUDLE.findAllIn(code).nonEmpty || FROM_SYS_IMPORT.findAllIn(code).nonEmpty) { throw PythonCodeCheckException(20070, "can not use sys module") - else if (IMPORT_OS_MOUDLE.findAllIn(code).nonEmpty || FROM_OS_IMPORT.findAllIn(code).nonEmpty) + } else if ( + IMPORT_OS_MOUDLE.findAllIn(code).nonEmpty || FROM_OS_IMPORT.findAllIn(code).nonEmpty + ) { throw PythonCodeCheckException(20071, "can not use os moudle") - else if ( + } else if ( IMPORT_PROCESS_MODULE.findAllIn(code).nonEmpty || FROM_MULTIPROCESS_IMPORT .findAllIn(code) .nonEmpty - ) + ) { throw PythonCodeCheckException(20072, "can not use process module") - else if ( + } else if ( IMPORT_SUBPORCESS_MODULE.findAllIn(code).nonEmpty || FROM_SUBPROCESS_IMPORT .findAllIn(code) .nonEmpty - ) + ) { throw PythonCodeCheckException(20072, "can not use subprocess module") - else if (SC_STOP.findAllIn(code).nonEmpty) + } else if (SC_STOP.findAllIn(code).nonEmpty) { throw PythonCodeCheckException(20073, "You can not stop SparkContext, It's dangerous") + } } true } diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/interceptor/impl/UserCreatorIPCheckUtils.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/interceptor/impl/UserCreatorIPCheckUtils.scala index 573c134493..653e9ad78b 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/interceptor/impl/UserCreatorIPCheckUtils.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/interceptor/impl/UserCreatorIPCheckUtils.scala @@ -67,7 +67,7 @@ object UserCreatorIPCheckUtils extends Logging { def checkUserIp(jobRequest: JobRequest, logAppender: lang.StringBuilder): JobRequest = { // Get IP address - val jobIp = jobRequest.getSource.get(TaskConstant.REQUEST_IP) + val jobIp = jobRequest.getSource.getOrDefault(TaskConstant.REQUEST_IP, "") logger.debug(s"start to checkTenantLabel $jobIp") if (StringUtils.isNotBlank(jobIp)) { jobRequest match { diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/parser/CommonEntranceParser.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/parser/CommonEntranceParser.scala index 60164ca58b..afc18bdc19 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/parser/CommonEntranceParser.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/parser/CommonEntranceParser.scala @@ -17,19 +17,24 @@ package org.apache.linkis.entrance.parser +import org.apache.linkis.common.conf.Configuration import org.apache.linkis.common.utils.Logging import org.apache.linkis.entrance.conf.EntranceConfiguration import org.apache.linkis.entrance.errorcode.EntranceErrorCodeSummary._ import org.apache.linkis.entrance.exception.{EntranceErrorCode, EntranceIllegalParamException} import org.apache.linkis.entrance.persistence.PersistenceManager import org.apache.linkis.entrance.timeout.JobTimeoutManager +import org.apache.linkis.governance.common.conf.GovernanceCommonConf import org.apache.linkis.governance.common.entity.job.JobRequest +import org.apache.linkis.manager.common.conf.RMConfiguration import org.apache.linkis.manager.label.builder.factory.{ LabelBuilderFactory, LabelBuilderFactoryContext } +import org.apache.linkis.manager.label.conf.LabelCommonConfig import org.apache.linkis.manager.label.constant.LabelKeyConstant import org.apache.linkis.manager.label.entity.Label +import org.apache.linkis.manager.label.entity.cluster.ClusterLabel import org.apache.linkis.manager.label.entity.engine.{CodeLanguageLabel, UserCreatorLabel} import org.apache.linkis.manager.label.utils.EngineTypeLabelCreator import org.apache.linkis.protocol.constants.TaskConstant @@ -117,6 +122,7 @@ class CommonEntranceParser(val persistenceManager: PersistenceManager) checkEngineTypeLabel(labels) generateAndVerifyCodeLanguageLabel(runType, labels) generateAndVerifyUserCreatorLabel(executeUser, labels) + generateAndVerifyClusterLabel(labels) jobRequest.setLabels(new util.ArrayList[Label[_]](labels.values())) jobRequest.setSource(source) @@ -131,7 +137,8 @@ class CommonEntranceParser(val persistenceManager: PersistenceManager) private def checkEngineTypeLabel(labels: util.Map[String, Label[_]]): Unit = { val engineTypeLabel = labels.getOrDefault(LabelKeyConstant.ENGINE_TYPE_KEY, null) if (null == engineTypeLabel) { - val msg = s"You need to specify engineTypeLabel in labels, such as spark-2.4.3" + val msg = s"You need to specify engineTypeLabel in labels," + + s"such as spark-${LabelCommonConfig.SPARK_ENGINE_VERSION.getValue}" throw new EntranceIllegalParamException( EntranceErrorCode.LABEL_PARAMS_INVALID.getErrCode, EntranceErrorCode.LABEL_PARAMS_INVALID.getDesc + msg @@ -187,6 +194,22 @@ class CommonEntranceParser(val persistenceManager: PersistenceManager) } } + private def generateAndVerifyClusterLabel(labels: util.Map[String, Label[_]]): Unit = { + if (!Configuration.IS_MULTIPLE_YARN_CLUSTER.getValue.asInstanceOf[Boolean]) { + return + } + var clusterLabel = labels + .getOrDefault(LabelKeyConstant.YARN_CLUSTER_KEY, null) + .asInstanceOf[ClusterLabel] + if (clusterLabel == null) { + clusterLabel = + LabelBuilderFactoryContext.getLabelBuilderFactory.createLabel(classOf[ClusterLabel]) + clusterLabel.setClusterName(RMConfiguration.DEFAULT_YARN_CLUSTER_NAME.getValue) + clusterLabel.setClusterType(RMConfiguration.DEFAULT_YARN_TYPE.getValue) + labels.put(clusterLabel.getLabelKey, clusterLabel) + } + } + private def parseToOldTask(params: util.Map[String, AnyRef]): JobRequest = { val jobReq = new JobRequest diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/EntranceSchedulerContext.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/EntranceSchedulerContext.scala index d5de2cc2da..1638b0fb1c 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/EntranceSchedulerContext.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/EntranceSchedulerContext.scala @@ -28,6 +28,11 @@ class EntranceSchedulerContext extends SchedulerContext { private var consumerManager: ConsumerManager = _ private var executorManager: ExecutorManager = _ + private var offlineFlag: Boolean = false + + def setOfflineFlag(offlineFlag: Boolean): Unit = this.offlineFlag = offlineFlag + def getOfflineFlag: Boolean = this.offlineFlag + def this( groupFactory: GroupFactory, consumerManager: ConsumerManager, diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/utils/JobHistoryHelper.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/utils/JobHistoryHelper.scala index 365a7b2d10..376a369ff1 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/utils/JobHistoryHelper.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/utils/JobHistoryHelper.scala @@ -69,6 +69,11 @@ object JobHistoryHelper extends Logging { else task.getStatus } + def getProgressByTaskID(taskID: Long): String = { + val task = getTaskByTaskID(taskID) + if (task == null) "0" else task.getProgress + } + def getRequestIpAddr(req: HttpServletRequest): String = { val addrList = List( Option(req.getHeader("x-forwarded-for")).getOrElse("").split(",")(0), @@ -123,6 +128,143 @@ object JobHistoryHelper extends Logging { sender.ask(jobReqBatchUpdate) } + /** + * Get all consume queue task and batch update instances(获取所有消费队列中的任务进行批量更新) + * + * @param taskIdList + * @param retryWhenUpdateFail + */ + def updateAllConsumeQueueTask( + taskIdList: util.List[Long], + retryWhenUpdateFail: Boolean = false + ): Unit = { + + if (taskIdList.isEmpty) return + + val updateTaskIds = new util.ArrayList[Long]() + + if ( + EntranceConfiguration.ENTRANCE_UPDATE_BATCH_SIZE.getValue > 0 && + taskIdList.size() > EntranceConfiguration.ENTRANCE_UPDATE_BATCH_SIZE.getValue + ) { + for (i <- 0 until EntranceConfiguration.ENTRANCE_UPDATE_BATCH_SIZE.getValue) { + updateTaskIds.add(taskIdList.get(i)) + } + } else { + updateTaskIds.addAll(taskIdList) + } + val list = new util.ArrayList[Long]() + list.addAll(taskIdList) + try { + val successTaskIds = updateBatchInstancesEmpty(updateTaskIds) + if (retryWhenUpdateFail) { + list.removeAll(successTaskIds) + } else { + list.removeAll(updateTaskIds) + } + } catch { + case e: Exception => + logger.warn("update batch instances failed, wait for retry", e) + Thread.sleep(1000) + } + updateAllConsumeQueueTask(list, retryWhenUpdateFail) + + } + + /** + * Batch update instances(批量更新instances字段) + * + * @param taskIdList + * @return + */ + def updateBatchInstancesEmpty(taskIdList: util.List[Long]): util.List[Long] = { + val jobReqList = new util.ArrayList[JobRequest]() + taskIdList.asScala.foreach(taskID => { + val jobRequest = new JobRequest + jobRequest.setId(taskID) + jobRequest.setInstances("") + jobReqList.add(jobRequest) + }) + val jobReqBatchUpdate = JobReqBatchUpdate(jobReqList) + Utils.tryCatch { + val response = sender.ask(jobReqBatchUpdate) + response match { + case resp: util.List[JobRespProtocol] => + // todo filter success data, rpc have bug + // resp.asScala + // .filter(r => + // r.getStatus == SUCCESS_FLAG && r.getData.containsKey(JobRequestConstants.JOB_ID) + // ) + // .map(_.getData.get(JobRequestConstants.JOB_ID).asInstanceOf[java.lang.Long]) + // .toList + + taskIdList + case _ => + throw JobHistoryFailedException( + "update batch instances from jobhistory not a correct List type" + ) + } + } { + case errorException: ErrorException => throw errorException + case e: Exception => + val e1 = + JobHistoryFailedException( + s"update batch instances ${taskIdList.asScala.mkString(",")} error" + ) + e1.initCause(e) + throw e + } + } + + /** + * query wait for failover task(获取待故障转移的任务) + * + * @param reqMap + * @param statusList + * @param startTimestamp + * @param limit + * @return + */ + def queryWaitForFailoverTask( + reqMap: util.Map[String, java.lang.Long], + statusList: util.List[String], + startTimestamp: Long, + limit: Int + ): util.List[JobRequest] = { + val requestFailoverJob = RequestFailoverJob(reqMap, statusList, startTimestamp, limit) + val tasks = Utils.tryCatch { + val response = sender.ask(requestFailoverJob) + response match { + case responsePersist: JobRespProtocol => + val status = responsePersist.getStatus + if (status != SUCCESS_FLAG) { + logger.error(s"query from jobHistory status failed, status is $status") + throw JobHistoryFailedException("query from jobHistory status failed") + } + val data = responsePersist.getData + data.get(JobRequestConstants.JOB_HISTORY_LIST) match { + case tasks: List[JobRequest] => + tasks.asJava + case _ => + throw JobHistoryFailedException( + s"query from jobhistory not a correct List type, instances ${reqMap.keySet()}" + ) + } + case _ => + logger.error("get query response incorrectly") + throw JobHistoryFailedException("get query response incorrectly") + } + } { + case errorException: ErrorException => throw errorException + case e: Exception => + val e1 = + JobHistoryFailedException(s"query failover task error, instances ${reqMap.keySet()} ") + e1.initCause(e) + throw e + } + tasks + } + private def getTaskByTaskID(taskID: Long): JobRequest = { val jobRequest = new JobRequest jobRequest.setId(taskID) @@ -176,15 +318,15 @@ object JobHistoryHelper extends Logging { val ecResourceMap = if (resourceInfo == null) new util.HashMap[String, ResourceWithStatus] else resourceInfo if (resourceMap != null) { - resourceMap.asInstanceOf[util.HashMap[String, ResourceWithStatus]].putAll(ecResourceMap) + resourceMap.asInstanceOf[util.Map[String, ResourceWithStatus]].putAll(ecResourceMap) } else { metricsMap.put(TaskConstant.JOB_YARNRESOURCE, ecResourceMap) } - var engineInstanceMap: util.HashMap[String, AnyRef] = null + var engineInstanceMap: util.Map[String, AnyRef] = null if (metricsMap.containsKey(TaskConstant.JOB_ENGINECONN_MAP)) { engineInstanceMap = metricsMap .get(TaskConstant.JOB_ENGINECONN_MAP) - .asInstanceOf[util.HashMap[String, AnyRef]] + .asInstanceOf[util.Map[String, AnyRef]] } else { engineInstanceMap = new util.HashMap[String, AnyRef]() metricsMap.put(TaskConstant.JOB_ENGINECONN_MAP, engineInstanceMap) @@ -194,7 +336,7 @@ object JobHistoryHelper extends Logging { val ticketId = infoMap.get(TaskConstant.TICKET_ID).asInstanceOf[String] val engineExtraInfoMap = engineInstanceMap .getOrDefault(ticketId, new util.HashMap[String, AnyRef]) - .asInstanceOf[util.HashMap[String, AnyRef]] + .asInstanceOf[util.Map[String, AnyRef]] engineExtraInfoMap.putAll(infoMap) engineInstanceMap.put(ticketId, engineExtraInfoMap) } else { diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/pom.xml b/linkis-computation-governance/linkis-manager/linkis-application-manager/pom.xml index 86b661a963..4b807cf150 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/pom.xml +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/pom.xml @@ -107,11 +107,30 @@ ${gson.version} provided - - org.apache.linkis - linkis-computation-governance-common - ${project.version} + io.fabric8 + kubernetes-client + ${kubernetes-client.version} + + + io.fabric8 + kubernetes-model-common + + + io.fabric8 + kubernetes-model-core + + + + + io.fabric8 + kubernetes-model-common + ${kubernetes-client.version} + + + io.fabric8 + kubernetes-model-core + ${kubernetes-client.version} diff --git a/linkis-engineconn-plugins/elasticsearch/pom.xml b/linkis-engineconn-plugins/elasticsearch/pom.xml deleted file mode 100644 index 6b8cc7d1a0..0000000000 --- a/linkis-engineconn-plugins/elasticsearch/pom.xml +++ /dev/null @@ -1,140 +0,0 @@ - - - - 4.0.0 - - org.apache.linkis - linkis - ${revision} - ../../pom.xml - - - linkis-engineplugin-elasticsearch - - - 7.6.2 - - - - - org.apache.linkis - linkis-engineconn-plugin-core - ${project.version} - - - - org.apache.linkis - linkis-computation-engineconn - ${project.version} - - - - org.apache.linkis - linkis-storage - ${project.version} - provided - - - - org.apache.linkis - linkis-rpc - ${project.version} - provided - - - - org.apache.linkis - linkis-common - ${project.version} - provided - - - - - org.elasticsearch.client - elasticsearch-rest-client - ${elasticsearch.version} - - - org.elasticsearch.client - elasticsearch-rest-client-sniffer - ${elasticsearch.version} - - - com.fasterxml.jackson.core - jackson-core - - - - - com.fasterxml.jackson.dataformat - jackson-dataformat-yaml - - - com.fasterxml.jackson.dataformat - jackson-dataformat-cbor - - - com.fasterxml.jackson.dataformat - jackson-dataformat-smile - - - com.fasterxml.jackson.dataformat - jackson-dataformat-csv - - - - - - - net.alchim31.maven - scala-maven-plugin - - - - org.apache.maven.plugins - maven-assembly-plugin - false - - false - out - false - false - - src/main/assembly/distribution.xml - - - - - make-assembly - - single - - package - - - src/main/assembly/distribution.xml - - - - - - - - - diff --git a/linkis-engineconn-plugins/elasticsearch/src/main/assembly/distribution.xml b/linkis-engineconn-plugins/elasticsearch/src/main/assembly/distribution.xml deleted file mode 100644 index 9511b709ae..0000000000 --- a/linkis-engineconn-plugins/elasticsearch/src/main/assembly/distribution.xml +++ /dev/null @@ -1,71 +0,0 @@ - - - - - linkis-engineplugin-elasticsearch - - dir - zip - - true - elasticsearch - - - - - - /dist/${elasticsearch.version}/lib - true - true - false - false - true - - - - - - - - ${basedir}/src/main/resources - - linkis-engineconn.properties - log4j2.xml - - 0777 - dist/${elasticsearch.version}/conf - unix - - - - ${basedir}/target - - *.jar - - - *doc.jar - - 0777 - plugin/${elasticsearch.version} - - - - - - diff --git a/linkis-engineconn-plugins/elasticsearch/src/main/java/org/apache/linkis/engineplugin/elasticsearch/errorcode/EasticsearchErrorCodeSummary.java b/linkis-engineconn-plugins/elasticsearch/src/main/java/org/apache/linkis/engineplugin/elasticsearch/errorcode/EasticsearchErrorCodeSummary.java deleted file mode 100644 index 74da50e75e..0000000000 --- a/linkis-engineconn-plugins/elasticsearch/src/main/java/org/apache/linkis/engineplugin/elasticsearch/errorcode/EasticsearchErrorCodeSummary.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.engineplugin.elasticsearch.errorcode; - -import org.apache.linkis.common.errorcode.LinkisErrorCode; - -public enum EasticsearchErrorCodeSummary implements LinkisErrorCode { - CLUSTER_IS_BLANK(70112, "The elasticsearch cluster is empty(es集群为空)!"), - RESPONSE_FAIL_IS_EMPTY( - 70113, - "EsEngineExecutor convert response fail, response content is empty(EsEngineExecutor response解析失败，response 内容为空)."); - - /** (errorCode)错误码 */ - private final int errorCode; - /** (errorDesc)错误描述 */ - private final String errorDesc; - - EasticsearchErrorCodeSummary(int errorCode, String errorDesc) { - this.errorCode = errorCode; - this.errorDesc = errorDesc; - } - - @Override - public int getErrorCode() { - return errorCode; - } - - @Override - public String getErrorDesc() { - return errorDesc; - } -} diff --git a/linkis-engineconn-plugins/elasticsearch/src/main/resources/linkis-engineconn.properties b/linkis-engineconn-plugins/elasticsearch/src/main/resources/linkis-engineconn.properties deleted file mode 100644 index 81d279da25..0000000000 --- a/linkis-engineconn-plugins/elasticsearch/src/main/resources/linkis-engineconn.properties +++ /dev/null @@ -1,23 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -wds.linkis.server.version=v1 -#wds.linkis.engineconn.debug.enable=true -wds.linkis.engineconn.plugin.default.class=org.apache.linkis.engineplugin.elasticsearch.ElasticSearchEngineConnPlugin -wds.linkis.engineconn.support.parallelism=true - -# ElasticSearch -linkis.es.cluster=127.0.0.1:9200 - diff --git a/linkis-engineconn-plugins/elasticsearch/src/main/resources/log4j2.xml b/linkis-engineconn-plugins/elasticsearch/src/main/resources/log4j2.xml deleted file mode 100644 index a121eeca81..0000000000 --- a/linkis-engineconn-plugins/elasticsearch/src/main/resources/log4j2.xml +++ /dev/null @@ -1,93 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/linkis-engineconn-plugins/elasticsearch/src/main/scala/org/apache/linkis/engineplugin/elasticsearch/ElasticSearchEngineConnPlugin.scala b/linkis-engineconn-plugins/elasticsearch/src/main/scala/org/apache/linkis/engineplugin/elasticsearch/ElasticSearchEngineConnPlugin.scala deleted file mode 100644 index 67e2d39cee..0000000000 --- a/linkis-engineconn-plugins/elasticsearch/src/main/scala/org/apache/linkis/engineplugin/elasticsearch/ElasticSearchEngineConnPlugin.scala +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.engineplugin.elasticsearch - -import org.apache.linkis.engineplugin.elasticsearch.builder.ElasticSearchProcessEngineConnLaunchBuilder -import org.apache.linkis.engineplugin.elasticsearch.conf.ElasticSearchConfiguration -import org.apache.linkis.engineplugin.elasticsearch.factory.ElasticSearchEngineConnFactory -import org.apache.linkis.manager.engineplugin.common.EngineConnPlugin -import org.apache.linkis.manager.engineplugin.common.creation.EngineConnFactory -import org.apache.linkis.manager.engineplugin.common.launch.EngineConnLaunchBuilder -import org.apache.linkis.manager.engineplugin.common.resource.{ - EngineResourceFactory, - GenericEngineResourceFactory -} -import org.apache.linkis.manager.label.entity.Label -import org.apache.linkis.manager.label.entity.engine.{EngineType, EngineTypeLabel} - -import java.util - -class ElasticSearchEngineConnPlugin extends EngineConnPlugin { - - private var engineResourceFactory: EngineResourceFactory = _ - - private var engineFactory: EngineConnFactory = _ - - private val defaultLabels: util.List[Label[_]] = new util.ArrayList[Label[_]]() - - private val resourceLocker = new Array[Byte](0) - - private val engineFactoryLocker = new Array[Byte](0) - - override def init(params: util.Map[String, AnyRef]): Unit = { - val typeLabel = new EngineTypeLabel() - typeLabel.setEngineType(EngineType.ELASTICSEARCH.toString) - typeLabel.setVersion(ElasticSearchConfiguration.DEFAULT_VERSION.getValue) - this.defaultLabels.add(typeLabel) - } - - override def getEngineResourceFactory: EngineResourceFactory = { - if (null == engineResourceFactory) resourceLocker.synchronized { - if (null == engineResourceFactory) engineResourceFactory = new GenericEngineResourceFactory - } - engineResourceFactory - } - - override def getEngineConnLaunchBuilder: EngineConnLaunchBuilder = { - new ElasticSearchProcessEngineConnLaunchBuilder() - } - - override def getEngineConnFactory: EngineConnFactory = { - if (null == engineFactory) engineFactoryLocker.synchronized { - if (null == engineFactory) engineFactory = new ElasticSearchEngineConnFactory - } - engineFactory - } - - override def getDefaultLabels: util.List[Label[_]] = defaultLabels - -} diff --git a/linkis-engineconn-plugins/elasticsearch/src/main/scala/org/apache/linkis/engineplugin/elasticsearch/conf/ElasticSearchConfiguration.scala b/linkis-engineconn-plugins/elasticsearch/src/main/scala/org/apache/linkis/engineplugin/elasticsearch/conf/ElasticSearchConfiguration.scala deleted file mode 100644 index d2909a0638..0000000000 --- a/linkis-engineconn-plugins/elasticsearch/src/main/scala/org/apache/linkis/engineplugin/elasticsearch/conf/ElasticSearchConfiguration.scala +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.engineplugin.elasticsearch.conf - -import org.apache.linkis.common.conf.{ByteType, CommonVars} - -object ElasticSearchConfiguration { - - // es client - val ES_CLUSTER = CommonVars("linkis.es.cluster", "127.0.0.1:9200") - val ES_DATASOURCE_NAME = CommonVars("linkis.es.datasource", "default_datasource") - val ES_AUTH_CACHE = CommonVars("linkis.es.auth.cache", false) - val ES_USERNAME = CommonVars("linkis.es.username", "") - val ES_PASSWORD = CommonVars("linkis.es.password", "") - val ES_SNIFFER_ENABLE = CommonVars("linkis.es.sniffer.enable", false) - val ES_HTTP_METHOD = CommonVars("linkis.es.http.method", "GET") - val ES_HTTP_ENDPOINT = CommonVars("linkis.es.http.endpoint", "/_search") - val ES_HTTP_SQL_ENDPOINT = CommonVars("linkis.es.sql.endpoint", "/_sql") - val ES_SQL_FORMAT = CommonVars("linkis.es.sql.format", "{\"query\": \"%s\"}") - val ES_HTTP_HEADER_PREFIX = "linkis.es.headers." - - // entrance resource - val ENTRANCE_MAX_JOB_INSTANCE = CommonVars("linkis.es.max.job.instance", 100) - val ENTRANCE_PROTECTED_JOB_INSTANCE = CommonVars("linkis.es.protected.job.instance", 20) - val ENGINE_DEFAULT_LIMIT = CommonVars("linkis.es.default.limit", 5000) - - // resultSet - val ENGINE_RESULT_SET_MAX_CACHE = CommonVars("linkis.resultSet.cache.max", new ByteType("512k")) - - val ENGINE_CONCURRENT_LIMIT = CommonVars[Int]("linkis.engineconn.concurrent.limit", 100) - - val DEFAULT_VERSION = CommonVars[String]("linkis.engineconn.io.version", "7.6.2") - -} diff --git a/linkis-engineconn-plugins/elasticsearch/src/main/scala/org/apache/linkis/engineplugin/elasticsearch/conf/ElasticSearchEngineConsoleConf.scala b/linkis-engineconn-plugins/elasticsearch/src/main/scala/org/apache/linkis/engineplugin/elasticsearch/conf/ElasticSearchEngineConsoleConf.scala deleted file mode 100644 index b197f2047d..0000000000 --- a/linkis-engineconn-plugins/elasticsearch/src/main/scala/org/apache/linkis/engineplugin/elasticsearch/conf/ElasticSearchEngineConsoleConf.scala +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.engineplugin.elasticsearch.conf - -import org.apache.linkis.common.conf.Configuration -import org.apache.linkis.governance.common.protocol.conf.{ - RequestQueryEngineConfig, - ResponseQueryConfig -} -import org.apache.linkis.manager.label.entity.Label -import org.apache.linkis.manager.label.entity.engine.{EngineTypeLabel, UserCreatorLabel} -import org.apache.linkis.protocol.CacheableProtocol -import org.apache.linkis.rpc.RPCMapCache - -import java.util - -object ElasticSearchEngineConsoleConf - extends RPCMapCache[Array[Label[_]], String, String]( - Configuration.CLOUD_CONSOLE_CONFIGURATION_SPRING_APPLICATION_NAME.getValue - ) { - - override protected def createRequest(labels: Array[Label[_]]): CacheableProtocol = { - val userCreatorLabel = - labels.find(_.isInstanceOf[UserCreatorLabel]).get.asInstanceOf[UserCreatorLabel] - val engineTypeLabel = - labels.find(_.isInstanceOf[EngineTypeLabel]).get.asInstanceOf[EngineTypeLabel] - RequestQueryEngineConfig(userCreatorLabel, engineTypeLabel) - } - - override protected def createMap(any: Any): util.Map[String, String] = any match { - case response: ResponseQueryConfig => response.getKeyAndValue - } - -} diff --git a/linkis-engineconn-plugins/elasticsearch/src/main/scala/org/apache/linkis/engineplugin/elasticsearch/exception/EsConvertResponseException.scala b/linkis-engineconn-plugins/elasticsearch/src/main/scala/org/apache/linkis/engineplugin/elasticsearch/exception/EsConvertResponseException.scala deleted file mode 100644 index 998a2d81d1..0000000000 --- a/linkis-engineconn-plugins/elasticsearch/src/main/scala/org/apache/linkis/engineplugin/elasticsearch/exception/EsConvertResponseException.scala +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.engineplugin.elasticsearch.exception - -import org.apache.linkis.common.exception.ErrorException -import org.apache.linkis.engineplugin.elasticsearch.errorcode.EasticsearchErrorCodeSummary.RESPONSE_FAIL_IS_EMPTY - -case class EsConvertResponseException(errorMsg: String) - extends ErrorException(RESPONSE_FAIL_IS_EMPTY.getErrorCode, errorMsg) diff --git a/linkis-engineconn-plugins/elasticsearch/src/main/scala/org/apache/linkis/engineplugin/elasticsearch/exception/EsParamsIllegalException.scala b/linkis-engineconn-plugins/elasticsearch/src/main/scala/org/apache/linkis/engineplugin/elasticsearch/exception/EsParamsIllegalException.scala deleted file mode 100644 index f6c7d4131a..0000000000 --- a/linkis-engineconn-plugins/elasticsearch/src/main/scala/org/apache/linkis/engineplugin/elasticsearch/exception/EsParamsIllegalException.scala +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.engineplugin.elasticsearch.exception - -import org.apache.linkis.common.exception.ErrorException -import org.apache.linkis.engineplugin.elasticsearch.errorcode.EasticsearchErrorCodeSummary.CLUSTER_IS_BLANK - -case class EsParamsIllegalException(errorMsg: String) - extends ErrorException(CLUSTER_IS_BLANK.getErrorCode, errorMsg) diff --git a/linkis-engineconn-plugins/elasticsearch/src/main/scala/org/apache/linkis/engineplugin/elasticsearch/executor/ElasticSearchEngineConnExecutor.scala b/linkis-engineconn-plugins/elasticsearch/src/main/scala/org/apache/linkis/engineplugin/elasticsearch/executor/ElasticSearchEngineConnExecutor.scala deleted file mode 100644 index 198bd7089a..0000000000 --- a/linkis-engineconn-plugins/elasticsearch/src/main/scala/org/apache/linkis/engineplugin/elasticsearch/executor/ElasticSearchEngineConnExecutor.scala +++ /dev/null @@ -1,229 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.engineplugin.elasticsearch.executor - -import org.apache.linkis.common.utils.{Logging, OverloadUtils, Utils} -import org.apache.linkis.engineconn.common.conf.{EngineConnConf, EngineConnConstant} -import org.apache.linkis.engineconn.computation.executor.entity.EngineConnTask -import org.apache.linkis.engineconn.computation.executor.execute.{ - ConcurrentComputationExecutor, - EngineExecutionContext -} -import org.apache.linkis.engineconn.core.EngineConnObject -import org.apache.linkis.engineplugin.elasticsearch.conf.{ - ElasticSearchConfiguration, - ElasticSearchEngineConsoleConf -} -import org.apache.linkis.engineplugin.elasticsearch.executor.client.{ - ElasticSearchErrorResponse, - ElasticSearchExecutor, - ElasticSearchJsonResponse, - ElasticSearchTableResponse -} -import org.apache.linkis.engineplugin.elasticsearch.executor.client.ElasticSearchErrorResponse -import org.apache.linkis.governance.common.entity.ExecutionNodeStatus -import org.apache.linkis.manager.common.entity.resource.{ - CommonNodeResource, - LoadResource, - NodeResource -} -import org.apache.linkis.manager.engineplugin.common.conf.EngineConnPluginConf -import org.apache.linkis.manager.engineplugin.common.util.NodeResourceUtils -import org.apache.linkis.manager.label.entity.Label -import org.apache.linkis.protocol.engine.JobProgressInfo -import org.apache.linkis.rpc.Sender -import org.apache.linkis.scheduler.executer.{ - AliasOutputExecuteResponse, - ErrorExecuteResponse, - ExecuteResponse -} -import org.apache.linkis.storage.LineRecord -import org.apache.linkis.storage.resultset.ResultSetFactory -import org.apache.linkis.storage.resultset.table.TableMetaData - -import org.apache.commons.io.IOUtils - -import org.springframework.util.CollectionUtils - -import java.util -import java.util.concurrent.TimeUnit - -import scala.collection.JavaConverters._ - -import com.google.common.cache.{Cache, CacheBuilder, RemovalListener, RemovalNotification} - -class ElasticSearchEngineConnExecutor( - override val outputPrintLimit: Int, - val id: Int, - runType: String -) extends ConcurrentComputationExecutor(outputPrintLimit) - with Logging { - - private val executorLabels: util.List[Label[_]] = new util.ArrayList[Label[_]](2) - - private val elasticSearchExecutorCache: Cache[String, ElasticSearchExecutor] = CacheBuilder - .newBuilder() - .expireAfterAccess(EngineConnConf.ENGINE_TASK_EXPIRE_TIME.getValue, TimeUnit.MILLISECONDS) - .removalListener(new RemovalListener[String, ElasticSearchExecutor] { - - override def onRemoval( - notification: RemovalNotification[String, ElasticSearchExecutor] - ): Unit = { - notification.getValue.close - val task = getTaskById(notification.getKey) - if (!ExecutionNodeStatus.isCompleted(task.getStatus)) { - killTask(notification.getKey) - } - } - - }) - .maximumSize(EngineConnConstant.MAX_TASK_NUM) - .build() - - override def init(): Unit = { - super.init() - } - - override def execute(engineConnTask: EngineConnTask): ExecuteResponse = { - - val properties: util.Map[String, String] = buildRuntimeParams(engineConnTask) - logger.info(s"The elasticsearch properties is: $properties") - - val elasticSearchExecutor = ElasticSearchExecutor(runType, properties) - elasticSearchExecutor.open - elasticSearchExecutorCache.put(engineConnTask.getTaskId, elasticSearchExecutor) - super.execute(engineConnTask) - } - - override def executeLine( - engineExecutorContext: EngineExecutionContext, - code: String - ): ExecuteResponse = { - val taskId = engineExecutorContext.getJobId.get - val elasticSearchExecutor = elasticSearchExecutorCache.getIfPresent(taskId) - val elasticSearchResponse = elasticSearchExecutor.executeLine(code) - - elasticSearchResponse match { - case ElasticSearchTableResponse(columns, records) => - val metaData = new TableMetaData(columns) - val resultSetWriter = - engineExecutorContext.createResultSetWriter(ResultSetFactory.TABLE_TYPE) - resultSetWriter.addMetaData(metaData) - records.foreach(record => resultSetWriter.addRecord(record)) - val output = resultSetWriter.toString - Utils.tryQuietly { - IOUtils.closeQuietly(resultSetWriter) - } - AliasOutputExecuteResponse(null, output) - case ElasticSearchJsonResponse(content) => - val resultSetWriter = - engineExecutorContext.createResultSetWriter(ResultSetFactory.TEXT_TYPE) - resultSetWriter.addMetaData(null) - content.split("\\n").foreach(item => resultSetWriter.addRecord(new LineRecord(item))) - val output = resultSetWriter.toString - Utils.tryQuietly { - IOUtils.closeQuietly(resultSetWriter) - } - AliasOutputExecuteResponse(null, output) - case ElasticSearchErrorResponse(message, body, cause) => - ErrorExecuteResponse(message, cause) - } - } - - private def buildRuntimeParams(engineConnTask: EngineConnTask): util.Map[String, String] = { - - // parameters specified at runtime - var executorProperties = engineConnTask.getProperties.asInstanceOf[util.Map[String, String]] - if (executorProperties == null) { - executorProperties = new util.HashMap[String, String]() - } - - // global engine params by console - val globalConfig: util.Map[String, String] = - Utils.tryAndWarn(ElasticSearchEngineConsoleConf.getCacheMap(engineConnTask.getLables)) - - if (!executorProperties.isEmpty) { - globalConfig.putAll(executorProperties) - } - - globalConfig - } - - override def executeCompletely( - engineExecutorContext: EngineExecutionContext, - code: String, - completedLine: String - ): ExecuteResponse = null - - override def progress(taskID: String): Float = 0.0f - - override def getProgressInfo(taskID: String): Array[JobProgressInfo] = - Array.empty[JobProgressInfo] - - override def getExecutorLabels(): util.List[Label[_]] = executorLabels - - override def setExecutorLabels(labels: util.List[Label[_]]): Unit = { - if (!CollectionUtils.isEmpty(labels)) { - executorLabels.clear() - executorLabels.addAll(labels) - } - } - - override def supportCallBackLogs(): Boolean = false - - override def requestExpectedResource(expectedResource: NodeResource): NodeResource = null - - override def getCurrentNodeResource(): NodeResource = { - NodeResourceUtils.appendMemoryUnitIfMissing( - EngineConnObject.getEngineCreationContext.getOptions - ) - - val resource = new CommonNodeResource - val usedResource = new LoadResource(OverloadUtils.getProcessMaxMemory, 1) - resource.setUsedResource(usedResource) - resource - } - - override def getId(): String = Sender.getThisServiceInstance.getInstance + s"_$id" - - override def killTask(taskId: String): Unit = { - Utils.tryAndWarn { - val elasticSearchExecutor = elasticSearchExecutorCache.getIfPresent(taskId) - if (null != elasticSearchExecutor) { - elasticSearchExecutor.close - } - } - super.killTask(taskId) - } - - override def killAll(): Unit = { - elasticSearchExecutorCache - .asMap() - .values() - .asScala - .foreach(e => e.close) - } - - override def transformTaskStatus(task: EngineConnTask, newStatus: ExecutionNodeStatus): Unit = { - super.transformTaskStatus(task, newStatus) - if (ExecutionNodeStatus.isCompleted(newStatus)) { - elasticSearchExecutorCache.invalidate(task.getTaskId) - } - } - -} diff --git a/linkis-engineconn-plugins/elasticsearch/src/main/scala/org/apache/linkis/engineplugin/elasticsearch/executor/client/ElasticSearchResponse.scala b/linkis-engineconn-plugins/elasticsearch/src/main/scala/org/apache/linkis/engineplugin/elasticsearch/executor/client/ElasticSearchResponse.scala deleted file mode 100644 index b01b109554..0000000000 --- a/linkis-engineconn-plugins/elasticsearch/src/main/scala/org/apache/linkis/engineplugin/elasticsearch/executor/client/ElasticSearchResponse.scala +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.engineplugin.elasticsearch.executor.client - -import org.apache.linkis.storage.domain.Column -import org.apache.linkis.storage.resultset.table.TableRecord - -trait ElasticSearchResponse - -case class ElasticSearchTableResponse(columns: Array[Column], records: Array[TableRecord]) - extends ElasticSearchResponse - -case class ElasticSearchJsonResponse(value: String) extends ElasticSearchResponse - -case class ElasticSearchErrorResponse(message: String, body: String = null, cause: Throwable = null) - extends ElasticSearchResponse diff --git a/linkis-engineconn-plugins/elasticsearch/src/main/scala/org/apache/linkis/engineplugin/elasticsearch/executor/client/EsClient.scala b/linkis-engineconn-plugins/elasticsearch/src/main/scala/org/apache/linkis/engineplugin/elasticsearch/executor/client/EsClient.scala deleted file mode 100644 index 8b894b4664..0000000000 --- a/linkis-engineconn-plugins/elasticsearch/src/main/scala/org/apache/linkis/engineplugin/elasticsearch/executor/client/EsClient.scala +++ /dev/null @@ -1,131 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.engineplugin.elasticsearch.executor.client - -import org.apache.linkis.common.utils.Utils -import org.apache.linkis.engineplugin.elasticsearch.conf.ElasticSearchConfiguration._ -import org.apache.linkis.server.JMap - -import org.apache.commons.codec.binary.Base64 -import org.apache.commons.lang3.StringUtils -import org.apache.http.Header -import org.apache.http.auth.{AUTH, Credentials, UsernamePasswordCredentials} -import org.apache.http.message.BufferedHeader -import org.apache.http.util.{CharArrayBuffer, EncodingUtils} - -import java.nio.charset.StandardCharsets.UTF_8 -import java.util - -import scala.collection.JavaConverters._ - -import org.elasticsearch.client.{Cancellable, Request, RequestOptions, ResponseListener, RestClient} -import org.elasticsearch.client.sniff.Sniffer - -trait EsClientOperate { - - def execute( - code: String, - options: util.Map[String, String], - responseListener: ResponseListener - ): Cancellable - - def close(): Unit - -} - -abstract class EsClient(datasourceName: String, client: RestClient, sniffer: Sniffer) - extends EsClientOperate { - - def getDatasourceName: String = datasourceName - - def getRestClient: RestClient = client - - def getSniffer: Sniffer = sniffer - - override def close(): Unit = Utils.tryQuietly { - sniffer match { - case s: Sniffer => s.close() - case _ => - } - client match { - case c: RestClient => c.close() - case _ => - } - } - -} - -class EsClientImpl(datasourceName: String, client: RestClient, sniffer: Sniffer) - extends EsClient(datasourceName, client, sniffer) { - - override def execute( - code: String, - options: util.Map[String, String], - responseListener: ResponseListener - ): Cancellable = { - val request = createRequest(code, options) - client.performRequestAsync(request, responseListener) - } - - private def createRequest(code: String, options: util.Map[String, String]): Request = { - val endpoint = ES_HTTP_ENDPOINT.getValue(options) - val method = ES_HTTP_METHOD.getValue(options) - val request = new Request(method, endpoint) - request.setOptions(getRequestOptions(options)) - request.setJsonEntity(code) - request - } - - private def getRequestOptions(options: util.Map[String, String]): RequestOptions = { - val builder = RequestOptions.DEFAULT.toBuilder() - - val username = ES_USERNAME.getValue(options) - val password = ES_PASSWORD.getValue(options) - // username / password convert to base auth - if (StringUtils.isNotBlank(username) && StringUtils.isNotBlank(password)) { - val authHeader = - authenticate(new UsernamePasswordCredentials(username, password), UTF_8.name()) - builder.addHeader(authHeader.getName, authHeader.getValue) - } - - options.asScala - .filter(entry => - entry._1 != null && entry._2 != null && entry._1.startsWith(ES_HTTP_HEADER_PREFIX) - ) - .foreach(entry => builder.addHeader(entry._1, entry._2)) - - builder.build() - } - - private def authenticate(credentials: Credentials, charset: String): Header = { - val tmp = new StringBuilder - tmp.append(credentials.getUserPrincipal.getName) - tmp.append(":") - tmp.append( - if (credentials.getPassword == null) "null" - else credentials.getPassword - ) - val base64password = Base64.encodeBase64(EncodingUtils.getBytes(tmp.toString, charset), false) - val buffer = new CharArrayBuffer(32) - buffer.append(AUTH.WWW_AUTH_RESP) - buffer.append(": Basic ") - buffer.append(base64password, 0, base64password.length) - new BufferedHeader(buffer) - } - -} diff --git a/linkis-engineconn-plugins/elasticsearch/src/main/scala/org/apache/linkis/engineplugin/elasticsearch/executor/client/EsClientFactory.scala b/linkis-engineconn-plugins/elasticsearch/src/main/scala/org/apache/linkis/engineplugin/elasticsearch/executor/client/EsClientFactory.scala deleted file mode 100644 index 30cb6690b5..0000000000 --- a/linkis-engineconn-plugins/elasticsearch/src/main/scala/org/apache/linkis/engineplugin/elasticsearch/executor/client/EsClientFactory.scala +++ /dev/null @@ -1,185 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.engineplugin.elasticsearch.executor.client - -import org.apache.linkis.common.conf.CommonVars -import org.apache.linkis.engineplugin.elasticsearch.conf.ElasticSearchConfiguration._ -import org.apache.linkis.engineplugin.elasticsearch.errorcode.EasticsearchErrorCodeSummary.CLUSTER_IS_BLANK -import org.apache.linkis.engineplugin.elasticsearch.exception.EsParamsIllegalException - -import org.apache.commons.lang3.StringUtils -import org.apache.http.{Header, HttpHost} -import org.apache.http.auth.{AuthScope, UsernamePasswordCredentials} -import org.apache.http.client.CredentialsProvider -import org.apache.http.impl.client.BasicCredentialsProvider -import org.apache.http.impl.nio.client.HttpAsyncClientBuilder -import org.apache.http.message.BasicHeader - -import java.util -import java.util.Map - -import scala.collection.JavaConverters._ - -import org.elasticsearch.client.{RestClient, RestClientBuilder} -import org.elasticsearch.client.sniff.Sniffer - -object EsClientFactory { - - def getRestClient(options: util.Map[String, String]): EsClient = { - val key = getDatasourceName(options) - if (StringUtils.isBlank(key)) { - return defaultClient - } - - if (!ES_CLIENT_MAP.containsKey(key)) { - ES_CLIENT_MAP synchronized { - if (!ES_CLIENT_MAP.containsKey(key)) { - cacheClient(createRestClient(options)) - } - } - } - - ES_CLIENT_MAP.get(key) - } - - private val MAX_CACHE_CLIENT_SIZE = 20 - - private val ES_CLIENT_MAP: Map[String, EsClient] = new util.LinkedHashMap[String, EsClient]() { - - override def removeEldestEntry(eldest: Map.Entry[String, EsClient]): Boolean = - if (size > MAX_CACHE_CLIENT_SIZE) { - eldest.getValue.close() - true - } else { - false - } - - } - - private def getDatasourceName(options: util.Map[String, String]): String = { - options.getOrDefault(ES_DATASOURCE_NAME.key, "") - } - - private def cacheClient(client: EsClient) = { - ES_CLIENT_MAP.put(client.getDatasourceName, client) - } - - private def createRestClient(options: util.Map[String, String]): EsClient = { - val clusterStr = options.get(ES_CLUSTER.key) - if (StringUtils.isBlank(clusterStr)) { - throw EsParamsIllegalException(CLUSTER_IS_BLANK.getErrorDesc) - } - val cluster = getCluster(clusterStr) - if (cluster.isEmpty) { - throw EsParamsIllegalException(CLUSTER_IS_BLANK.getErrorDesc) - } - val username = options.get(ES_USERNAME.key) - val password = options.get(ES_PASSWORD.key) - - if (ES_AUTH_CACHE.getValue) { - setAuthScope(cluster, username, password) - } - - val httpHosts = cluster.map(item => new HttpHost(item._1, item._2)) - val builder = RestClient - .builder(httpHosts: _*) - .setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() { - override def customizeHttpClient( - httpAsyncClientBuilder: HttpAsyncClientBuilder - ): HttpAsyncClientBuilder = { - if (!ES_AUTH_CACHE.getValue) { - httpAsyncClientBuilder.disableAuthCaching - } - // httpClientBuilder.setDefaultRequestConfig(RequestConfig.DEFAULT) - // httpClientBuilder.setDefaultConnectionConfig(ConnectionConfig.DEFAULT) - httpAsyncClientBuilder.setDefaultCredentialsProvider(credentialsProvider) - } - }) - if (defaultHeaders != null) { - builder.setDefaultHeaders(defaultHeaders) - } - val client = builder.build - - val sniffer = if (ES_SNIFFER_ENABLE.getValue(options)) { - Sniffer.builder(client).build - } else null - - val datasourceName = getDatasourceName(options) - new EsClientImpl(datasourceName, client, sniffer) - } - - private val credentialsProvider: CredentialsProvider = new BasicCredentialsProvider() - - private val defaultClient = { - val cluster = ES_CLUSTER.getValue - if (StringUtils.isBlank(cluster)) { - null - } else { - val defaultOpts = new util.HashMap[String, String]() - defaultOpts.put(ES_CLUSTER.key, cluster) - defaultOpts.put(ES_DATASOURCE_NAME.key, ES_DATASOURCE_NAME.getValue) - defaultOpts.put(ES_USERNAME.key, ES_USERNAME.getValue) - defaultOpts.put(ES_PASSWORD.key, ES_PASSWORD.getValue) - val client = createRestClient(defaultOpts) - cacheClient(client) - client - } - } - - private val defaultHeaders: Array[Header] = CommonVars.properties - .entrySet() - .asScala - .filter(entry => - entry.getKey != null && entry.getValue != null && entry.getKey.toString - .startsWith(ES_HTTP_HEADER_PREFIX) - ) - .map(entry => new BasicHeader(entry.getKey.toString, entry.getValue.toString)) - .toArray[Header] - - // host1:port1,host2:port2 -> [(host1,port1),(host2,port2)] - private def getCluster(clusterStr: String): Array[(String, Int)] = - if (StringUtils.isNotBlank(clusterStr)) { - clusterStr - .split(",") - .map(value => { - val arr = value.replace("http://", "").split(":") - (arr(0).trim, arr(1).trim.toInt) - }) - } else Array() - - // set cluster auth - private def setAuthScope( - cluster: Array[(String, Int)], - username: String, - password: String - ): Unit = if ( - cluster != null && !cluster.isEmpty - && StringUtils.isNotBlank(username) - && StringUtils.isNotBlank(password) - ) { - cluster.foreach { - case (host, port) => - credentialsProvider.setCredentials( - new AuthScope(host, port, AuthScope.ANY_REALM, AuthScope.ANY_SCHEME), - new UsernamePasswordCredentials(username, password) - ) - case _ => - } - } - -} diff --git a/linkis-engineconn-plugins/elasticsearch/src/main/scala/org/apache/linkis/engineplugin/elasticsearch/executor/client/ResponseHandler.scala b/linkis-engineconn-plugins/elasticsearch/src/main/scala/org/apache/linkis/engineplugin/elasticsearch/executor/client/ResponseHandler.scala deleted file mode 100644 index d259ea7b69..0000000000 --- a/linkis-engineconn-plugins/elasticsearch/src/main/scala/org/apache/linkis/engineplugin/elasticsearch/executor/client/ResponseHandler.scala +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.engineplugin.elasticsearch.executor.client - -import org.apache.linkis.common.utils.Logging -import org.apache.linkis.engineplugin.elasticsearch.executor.client.impl.ResponseHandlerImpl -import org.apache.linkis.storage.domain._ - -import java.util.Locale - -import com.fasterxml.jackson.databind.{JsonNode, ObjectMapper} -import com.fasterxml.jackson.databind.node.JsonNodeType -import com.fasterxml.jackson.dataformat.cbor.CBORFactory -import com.fasterxml.jackson.dataformat.csv.CsvMapper -import com.fasterxml.jackson.dataformat.smile.SmileFactory -import com.fasterxml.jackson.dataformat.yaml.YAMLMapper -import org.elasticsearch.client.Response - -trait ResponseHandler extends Logging { - - def handle(response: Response): ElasticSearchResponse - -} - -object ResponseHandler { - - val RESPONSE_HANDLER = new ResponseHandlerImpl() - - def handle(response: Response): ElasticSearchResponse = - RESPONSE_HANDLER.handle(response) - - val jsonMapper = new ObjectMapper() - val yamlMapper = new YAMLMapper() - val cborMapper = new ObjectMapper(new CBORFactory()) - val smileMapper = new ObjectMapper(new SmileFactory()) - val csvMapper = new CsvMapper() - - def getNodeDataType(node: JsonNode): DataType = node.getNodeType match { - case JsonNodeType.ARRAY => ArrayType - case JsonNodeType.BINARY => BinaryType - case JsonNodeType.BOOLEAN => BooleanType - case JsonNodeType.NULL => NullType - case JsonNodeType.NUMBER => DecimalType - case JsonNodeType.OBJECT => StructType - case JsonNodeType.POJO => StructType - case JsonNodeType.STRING => StringType - case JsonNodeType.MISSING => StringType - case _ => StringType - } - - def getNodeTypeByEsType(estype: String): DataType = estype.toLowerCase(Locale.getDefault) match { - case "long" | "integer" | "short" | "byte" | "double" | "float" | "half_float" | - "scaled_float" => - DecimalType - case "text" | "keyword" => StringType - case "date" => DateType - case "binary" => BinaryType - case _ => StringType - } - - def getNodeValue(node: JsonNode): Any = node.getNodeType match { - case JsonNodeType.NUMBER => node.asDouble() - case JsonNodeType.NULL => null - case _ => node.toString().replaceAll("\n|\t", " ") - } - -} diff --git a/linkis-engineconn-plugins/elasticsearch/src/main/scala/org/apache/linkis/engineplugin/elasticsearch/executor/client/impl/ElasticSearchExecutorImpl.scala b/linkis-engineconn-plugins/elasticsearch/src/main/scala/org/apache/linkis/engineplugin/elasticsearch/executor/client/impl/ElasticSearchExecutorImpl.scala deleted file mode 100644 index 86d10660b6..0000000000 --- a/linkis-engineconn-plugins/elasticsearch/src/main/scala/org/apache/linkis/engineplugin/elasticsearch/executor/client/impl/ElasticSearchExecutorImpl.scala +++ /dev/null @@ -1,112 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.engineplugin.elasticsearch.executor.client.impl - -import org.apache.linkis.common.utils.Utils -import org.apache.linkis.engineplugin.elasticsearch.conf.ElasticSearchConfiguration -import org.apache.linkis.engineplugin.elasticsearch.exception.EsConvertResponseException -import org.apache.linkis.engineplugin.elasticsearch.executor.client.{ - ElasticSearchErrorResponse, - ElasticSearchExecutor, - ElasticSearchResponse, - EsClient, - EsClientFactory, - ResponseHandler -} -import org.apache.linkis.engineplugin.elasticsearch.executor.client.ResponseHandler -import org.apache.linkis.protocol.constants.TaskConstant -import org.apache.linkis.scheduler.executer.{ - AliasOutputExecuteResponse, - ErrorExecuteResponse, - ExecuteResponse, - SuccessExecuteResponse -} -import org.apache.linkis.server.JMap -import org.apache.linkis.storage.utils.StorageUtils - -import java.util -import java.util.Locale -import java.util.concurrent.CountDownLatch - -import org.elasticsearch.client.{Cancellable, Response, ResponseListener} - -class ElasticSearchExecutorImpl(runType: String, properties: util.Map[String, String]) - extends ElasticSearchExecutor { - - private var client: EsClient = _ - private var cancelable: Cancellable = _ - private var user: String = _ - - override def open: Unit = { - this.client = EsClientFactory.getRestClient(properties) - this.user = properties.getOrDefault(TaskConstant.UMUSER, StorageUtils.getJvmUser) - runType.trim.toLowerCase(Locale.getDefault) match { - case "essql" | "sql" => - properties.putIfAbsent( - ElasticSearchConfiguration.ES_HTTP_ENDPOINT.key, - ElasticSearchConfiguration.ES_HTTP_SQL_ENDPOINT.getValue(properties) - ) - case _ => - } - } - - override def executeLine(code: String): ElasticSearchResponse = { - val realCode = code.trim() - logger.info(s"es client begins to run $runType code:\n ${realCode.trim}") - val countDown = new CountDownLatch(1) - var executeResponse: ElasticSearchResponse = ElasticSearchErrorResponse("INCOMPLETE") - cancelable = client.execute( - realCode, - properties, - new ResponseListener { - override def onSuccess(response: Response): Unit = { - executeResponse = convertResponse(response) - countDown.countDown() - } - override def onFailure(exception: Exception): Unit = { - executeResponse = - ElasticSearchErrorResponse("EsEngineExecutor execute fail. ", null, exception) - countDown.countDown() - } - } - ) - countDown.await() - executeResponse - } - - // convert response to executeResponse - private def convertResponse(response: Response): ElasticSearchResponse = - Utils.tryCatch[ElasticSearchResponse] { - val statusCode = response.getStatusLine.getStatusCode - if (statusCode >= 200 && statusCode < 300) { - ResponseHandler.handle(response) - } else { - ElasticSearchErrorResponse( - "EsEngineExecutor convert response fail. response code: " + response.getStatusLine.getStatusCode - ) - } - } { case t: Throwable => - ElasticSearchErrorResponse("EsEngineExecutor convert response error.", null, t) - } - - override def close: Unit = cancelable match { - case c: Cancellable => c.cancel() - case _ => - } - -} diff --git a/linkis-engineconn-plugins/elasticsearch/src/main/scala/org/apache/linkis/engineplugin/elasticsearch/executor/client/impl/ResponseHandlerImpl.scala b/linkis-engineconn-plugins/elasticsearch/src/main/scala/org/apache/linkis/engineplugin/elasticsearch/executor/client/impl/ResponseHandlerImpl.scala deleted file mode 100644 index 19233a07d8..0000000000 --- a/linkis-engineconn-plugins/elasticsearch/src/main/scala/org/apache/linkis/engineplugin/elasticsearch/executor/client/impl/ResponseHandlerImpl.scala +++ /dev/null @@ -1,170 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.engineplugin.elasticsearch.executor.client.impl - -import org.apache.linkis.common.utils.Utils -import org.apache.linkis.engineplugin.elasticsearch.errorcode.EasticsearchErrorCodeSummary._ -import org.apache.linkis.engineplugin.elasticsearch.exception.EsConvertResponseException -import org.apache.linkis.engineplugin.elasticsearch.executor.client.{ - ElasticSearchJsonResponse, - ElasticSearchResponse, - ElasticSearchTableResponse, - ResponseHandler -} -import org.apache.linkis.engineplugin.elasticsearch.executor.client.ResponseHandler -import org.apache.linkis.engineplugin.elasticsearch.executor.client.ResponseHandler._ -import org.apache.linkis.storage.domain._ -import org.apache.linkis.storage.resultset.table.TableRecord - -import org.apache.http.entity.ContentType -import org.apache.http.util.EntityUtils - -import java.nio.charset.Charset - -import scala.collection.JavaConverters._ -import scala.collection.mutable.ArrayBuffer - -import com.fasterxml.jackson.databind.node.{ArrayNode, ObjectNode} -import org.elasticsearch.client.Response - -class ResponseHandlerImpl extends ResponseHandler { - - // scalastyle:off - override def handle(response: Response): ElasticSearchResponse = { - val contentType = ContentType.get(response.getEntity).getMimeType.toLowerCase - val charSet = ContentType.get(response.getEntity).getCharset match { - case c: Charset => c - case _ => Charset.forName("UTF-8") - } - val contentBytes = EntityUtils.toByteArray(response.getEntity) - - if (contentBytes == null || contentBytes.isEmpty) { - throw EsConvertResponseException(RESPONSE_FAIL_IS_EMPTY.getErrorDesc) - } - - val jsonNode = Utils.tryCatch { - contentType match { - case "application/yaml" => - yamlMapper.readTree(contentBytes) - case "application/cbor" => - cborMapper.readTree(contentBytes) - case "application/smile" => - smileMapper.readTree(contentBytes) - case _ => - jsonMapper.readTree(contentBytes) - } - } { - case t: Throwable => { - warn("deserialize response content error, {}", t) - null - } - } - - if (jsonNode == null) { - ElasticSearchJsonResponse(new String(contentBytes, charSet)) - } - - var isTable = false - val columns = ArrayBuffer[Column]() - val records = new ArrayBuffer[TableRecord] - - // es json runType response - jsonNode.at("/hits/hits") match { - case hits: ArrayNode => { - isTable = true - columns += Column("_index", StringType, "") - columns += Column("_type", StringType, "") - columns += Column("_id", StringType, "") - columns += Column("_score", DoubleType, "") - hits.asScala.foreach { - case obj: ObjectNode => { - val lineValues = new Array[Any](columns.length).toBuffer - obj - .fields() - .asScala - .foreach(entry => { - val key = entry.getKey - val value = entry.getValue - if ("_source".equals(key.trim)) { - value - .fields() - .asScala - .foreach(sourceEntry => { - val sourcekey = sourceEntry.getKey - val sourcevalue = sourceEntry.getValue - val index = columns.indexWhere(_.columnName.equals(sourcekey)) - if (index < 0) { - columns += Column(sourcekey, getNodeDataType(sourcevalue), "") - lineValues += getNodeValue(sourcevalue) - } else { - lineValues(index) = getNodeValue(sourcevalue) - } - }) - } else { - val index = columns.indexWhere(_.columnName.equals(key)) - if (index < 0) { - columns += Column(key, getNodeDataType(value), "") - lineValues += getNodeValue(value) - } else { - lineValues(index) = getNodeValue(value) - } - } - }) - records += new TableRecord(lineValues.toArray) - } - case _ => - } - } - case _ => - } - - // es sql runType response - jsonNode.at("/rows") match { - case rows: ArrayNode => { - isTable = true - jsonNode - .get("columns") - .asInstanceOf[ArrayNode] - .asScala - .foreach(node => { - val name = node.get("name").asText() - val estype = node.get("type").asText().trim - columns += Column(name, getNodeTypeByEsType(estype), "") - }) - rows.asScala.foreach { - case row: ArrayNode => { - val lineValues = new ArrayBuffer[Any]() - row.asScala.foreach(node => lineValues += getNodeValue(node)) - records += new TableRecord(lineValues.toArray) - } - case _ => - } - } - case _ => - } - - // write result - if (isTable) { - ElasticSearchTableResponse(columns.toArray, records.toArray) - } else { - ElasticSearchJsonResponse(new String(contentBytes, charSet)) - } - } - // scalastyle:on - -} diff --git a/linkis-engineconn-plugins/elasticsearch/src/main/scala/org/apache/linkis/engineplugin/elasticsearch/factory/ElasticSearchEngineConnFactory.scala b/linkis-engineconn-plugins/elasticsearch/src/main/scala/org/apache/linkis/engineplugin/elasticsearch/factory/ElasticSearchEngineConnFactory.scala deleted file mode 100644 index 636febbff9..0000000000 --- a/linkis-engineconn-plugins/elasticsearch/src/main/scala/org/apache/linkis/engineplugin/elasticsearch/factory/ElasticSearchEngineConnFactory.scala +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.engineplugin.elasticsearch.factory - -import org.apache.linkis.common.utils.Logging -import org.apache.linkis.engineconn.common.creation.EngineCreationContext -import org.apache.linkis.manager.engineplugin.common.creation.{ - ExecutorFactory, - MultiExecutorEngineConnFactory -} -import org.apache.linkis.manager.label.entity.engine.EngineType -import org.apache.linkis.manager.label.entity.engine.EngineType.EngineType - -class ElasticSearchEngineConnFactory extends MultiExecutorEngineConnFactory with Logging { - - private val executorFactoryArray = Array[ExecutorFactory]( - new ElasticSearchJsonExecutorFactory, - new ElasticSearchSqlExecutorFactory - ) - - override def getExecutorFactories: Array[ExecutorFactory] = { - executorFactoryArray - } - - override protected def getDefaultExecutorFactoryClass: Class[_ <: ExecutorFactory] = - classOf[ElasticSearchJsonExecutorFactory] - - override protected def getEngineConnType: EngineType = EngineType.ELASTICSEARCH - - override protected def createEngineConnSession( - engineCreationContext: EngineCreationContext - ): Any = null - -} diff --git a/linkis-engineconn-plugins/elasticsearch/src/main/scala/org/apache/linkis/engineplugin/elasticsearch/factory/ElasticSearchJsonExecutorFactory.scala b/linkis-engineconn-plugins/elasticsearch/src/main/scala/org/apache/linkis/engineplugin/elasticsearch/factory/ElasticSearchJsonExecutorFactory.scala deleted file mode 100644 index efee33041b..0000000000 --- a/linkis-engineconn-plugins/elasticsearch/src/main/scala/org/apache/linkis/engineplugin/elasticsearch/factory/ElasticSearchJsonExecutorFactory.scala +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.engineplugin.elasticsearch.factory - -import org.apache.linkis.engineconn.common.creation.EngineCreationContext -import org.apache.linkis.engineconn.common.engineconn.EngineConn -import org.apache.linkis.engineconn.computation.executor.creation.ComputationExecutorFactory -import org.apache.linkis.engineconn.computation.executor.execute.ComputationExecutor -import org.apache.linkis.engineplugin.elasticsearch.conf.ElasticSearchConfiguration -import org.apache.linkis.engineplugin.elasticsearch.executor.ElasticSearchEngineConnExecutor -import org.apache.linkis.governance.common.paser.JsonCodeParser -import org.apache.linkis.manager.label.entity.Label -import org.apache.linkis.manager.label.entity.engine.RunType -import org.apache.linkis.manager.label.entity.engine.RunType.RunType - -class ElasticSearchJsonExecutorFactory extends ComputationExecutorFactory { - - override protected def newExecutor( - id: Int, - engineCreationContext: EngineCreationContext, - engineConn: EngineConn, - labels: Array[Label[_]] - ): ComputationExecutor = { - val executor = new ElasticSearchEngineConnExecutor( - ElasticSearchConfiguration.ENGINE_DEFAULT_LIMIT.getValue, - id, - RunType.ES_JSON.toString - ) - executor.setCodeParser(new JsonCodeParser) - executor - } - - override protected def getRunType: RunType = RunType.ES_JSON - -} diff --git a/linkis-engineconn-plugins/elasticsearch/src/main/scala/org/apache/linkis/engineplugin/elasticsearch/factory/ElasticSearchSqlExecutorFactory.scala b/linkis-engineconn-plugins/elasticsearch/src/main/scala/org/apache/linkis/engineplugin/elasticsearch/factory/ElasticSearchSqlExecutorFactory.scala deleted file mode 100644 index a32d6c9379..0000000000 --- a/linkis-engineconn-plugins/elasticsearch/src/main/scala/org/apache/linkis/engineplugin/elasticsearch/factory/ElasticSearchSqlExecutorFactory.scala +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.engineplugin.elasticsearch.factory - -import org.apache.linkis.engineconn.common.creation.EngineCreationContext -import org.apache.linkis.engineconn.common.engineconn.EngineConn -import org.apache.linkis.engineconn.computation.executor.creation.ComputationExecutorFactory -import org.apache.linkis.engineconn.computation.executor.execute.ComputationExecutor -import org.apache.linkis.engineconn.core.executor.ExecutorManager -import org.apache.linkis.engineplugin.elasticsearch.conf.ElasticSearchConfiguration -import org.apache.linkis.engineplugin.elasticsearch.executor.ElasticSearchEngineConnExecutor -import org.apache.linkis.governance.common.paser.SQLCodeParser -import org.apache.linkis.manager.label.entity.Label -import org.apache.linkis.manager.label.entity.engine.RunType -import org.apache.linkis.manager.label.entity.engine.RunType.RunType - -class ElasticSearchSqlExecutorFactory extends ComputationExecutorFactory { - - override protected def newExecutor( - id: Int, - engineCreationContext: EngineCreationContext, - engineConn: EngineConn, - labels: Array[Label[_]] - ): ComputationExecutor = { - val executor = new ElasticSearchEngineConnExecutor( - ElasticSearchConfiguration.ENGINE_DEFAULT_LIMIT.getValue, - id, - RunType.ES_SQL.toString - ) - executor.setCodeParser(new SQLCodeParser) - executor - } - - override protected def getRunType: RunType = RunType.ES_SQL - -} diff --git a/linkis-engineconn-plugins/elasticsearch/src/test/scala/org/apache/linkis/engineplugin/elasticsearch/TestElasticSearchEngineConnPlugin.scala b/linkis-engineconn-plugins/elasticsearch/src/test/scala/org/apache/linkis/engineplugin/elasticsearch/TestElasticSearchEngineConnPlugin.scala deleted file mode 100644 index 4b854d9cf3..0000000000 --- a/linkis-engineconn-plugins/elasticsearch/src/test/scala/org/apache/linkis/engineplugin/elasticsearch/TestElasticSearchEngineConnPlugin.scala +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.engineplugin.elasticsearch - -import org.junit.jupiter.api.{Assertions, Test} - -class TestElasticSearchEngineConnPlugin { - - @Test - def testGetEngineResourceFactory: Unit = { - val elasticSearchEngineConnPlugin = new ElasticSearchEngineConnPlugin - elasticSearchEngineConnPlugin.init(null) - Assertions.assertNotNull(elasticSearchEngineConnPlugin.getEngineConnFactory) - Assertions.assertNotNull(elasticSearchEngineConnPlugin.getEngineConnLaunchBuilder) - Assertions.assertNotNull(elasticSearchEngineConnPlugin.getEngineResourceFactory) - Assertions.assertNotNull(elasticSearchEngineConnPlugin.getDefaultLabels) - } - -} diff --git a/linkis-engineconn-plugins/hive/src/main/assembly/distribution.xml b/linkis-engineconn-plugins/hive/src/main/assembly/distribution.xml index adcc6d7244..bd9d705013 100644 --- a/linkis-engineconn-plugins/hive/src/main/assembly/distribution.xml +++ b/linkis-engineconn-plugins/hive/src/main/assembly/distribution.xml @@ -66,9 +66,6 @@ com.google.guava:guava:jar com.google.protobuf:protobuf-java:jar com.ning:async-http-client:jar - com.sun.jersey:jersey-json:jar - com.sun.jersey:jersey-server:jar - com.sun.jersey:jersey-servlet:jar com.sun.xml.bind:jaxb-impl:jar commons-beanutils:commons-beanutils:jar commons-beanutils:commons-beanutils-core:jar @@ -107,7 +104,6 @@ javax.xml.bind:jaxb-api:jar javax.xml.stream:stax-api:jar mysql:mysql-connector-java:jar - org.antlr:antlr-runtime:jar org.antlr:stringtemplate:jar org.apache.commons:commons-compress:jar org.apache.commons:commons-math:jar @@ -132,6 +128,8 @@ org.aspectj:aspectjweaver:jar org.bouncycastle:bcpkix-jdk15on:jar org.bouncycastle:bcprov-jdk15on:jar + org.codehaus.jackson:jackson-jaxrs:jar + org.codehaus.jackson:jackson-xc:jar org.codehaus.jettison:jettison:jar org.codehaus.woodstox:stax2-api:jar org.codehaus.woodstox:woodstox-core-asl:jar diff --git a/linkis-engineconn-plugins/hive/src/main/resources/log4j2.xml b/linkis-engineconn-plugins/hive/src/main/resources/log4j2.xml index 4928593404..de932ee75f 100644 --- a/linkis-engineconn-plugins/hive/src/main/resources/log4j2.xml +++ b/linkis-engineconn-plugins/hive/src/main/resources/log4j2.xml @@ -38,7 +38,6 @@ - @@ -108,11 +107,5 @@ - - - - - - diff --git a/linkis-engineconn-plugins/jdbc/src/main/java/org/apache/linkis/manager/engineplugin/jdbc/ConnectionManager.java b/linkis-engineconn-plugins/jdbc/src/main/java/org/apache/linkis/manager/engineplugin/jdbc/ConnectionManager.java index b9cd479457..f7598d113d 100644 --- a/linkis-engineconn-plugins/jdbc/src/main/java/org/apache/linkis/manager/engineplugin/jdbc/ConnectionManager.java +++ b/linkis-engineconn-plugins/jdbc/src/main/java/org/apache/linkis/manager/engineplugin/jdbc/ConnectionManager.java @@ -19,20 +19,24 @@ import org.apache.linkis.common.utils.SecurityUtils; import org.apache.linkis.hadoop.common.utils.KerberosUtils; +import org.apache.linkis.manager.engineplugin.jdbc.conf.JDBCConfiguration$; import org.apache.linkis.manager.engineplugin.jdbc.constant.JDBCEngineConnConstant; import org.apache.linkis.manager.engineplugin.jdbc.exception.JDBCParamsIllegalException; import org.apache.linkis.manager.engineplugin.jdbc.utils.JdbcParamUtils; -import org.apache.commons.dbcp.BasicDataSource; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.security.UserGroupInformation; import javax.sql.DataSource; +import java.io.Closeable; import java.security.PrivilegedExceptionAction; -import java.sql.*; +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; import java.text.MessageFormat; -import java.util.*; +import java.util.HashMap; +import java.util.Map; import java.util.concurrent.Callable; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; @@ -42,7 +46,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import static org.apache.linkis.manager.engineplugin.jdbc.JdbcAuthType.*; +import static org.apache.linkis.manager.engineplugin.jdbc.JdbcAuthType.USERNAME; +import static org.apache.linkis.manager.engineplugin.jdbc.JdbcAuthType.of; import static org.apache.linkis.manager.engineplugin.jdbc.errorcode.JDBCErrorCodeSummary.*; public class ConnectionManager { @@ -103,8 +108,10 @@ public void close() { } for (DataSource dataSource : this.dataSourceFactories.values()) { try { - ((BasicDataSource) dataSource).close(); - } catch (SQLException e) { + if (dataSource instanceof Closeable) { + ((Closeable) dataSource).close(); + } + } catch (Exception e) { LOG.error("Error while closing datasource...", e); } } @@ -181,11 +188,20 @@ protected DataSource buildDataSource(String dbUrl, Map propertie JDBCPropertiesParser.getInt( properties, JDBCEngineConnConstant.JDBC_POOL_REMOVE_ABANDONED_TIMEOUT, 300); + int connectionTimeout = + JDBCPropertiesParser.getInt(properties, JDBCEngineConnConstant.JDBC_CONNECTION_TIMEOUT, 0); + int socketTimeout = + JDBCPropertiesParser.getInt(properties, JDBCEngineConnConstant.JDBC_SOCKET_TIMEOUT, 0); + int queryTimeout = + JDBCPropertiesParser.getInt( + properties, JDBCConfiguration$.MODULE$.JDBC_QUERY_TIMEOUT().key(), 0); + DruidDataSource datasource = new DruidDataSource(); LOG.info("Database connection address information(数据库连接地址信息)=" + dbUrl); datasource.setUrl(dbUrl); datasource.setUsername(username); datasource.setPassword(password); + datasource.setConnectProperties(SecurityUtils.getMysqlSecurityParams()); datasource.setDriverClassName(driverClassName); datasource.setInitialSize(initialSize); datasource.setMinIdle(minIdle); @@ -200,6 +216,15 @@ protected DataSource buildDataSource(String dbUrl, Map propertie datasource.setPoolPreparedStatements(poolPreparedStatements); datasource.setRemoveAbandoned(removeAbandoned); datasource.setRemoveAbandonedTimeout(removeAbandonedTimeout); + if (connectionTimeout > 0) { + datasource.setConnectTimeout(connectionTimeout); + } + if (socketTimeout > 0) { + datasource.setSocketTimeout(socketTimeout); + } + if (queryTimeout > 0) { + datasource.setQueryTimeout(queryTimeout); + } return datasource; } diff --git a/linkis-engineconn-plugins/jdbc/src/main/java/org/apache/linkis/manager/engineplugin/jdbc/constant/JDBCEngineConnConstant.java b/linkis-engineconn-plugins/jdbc/src/main/java/org/apache/linkis/manager/engineplugin/jdbc/constant/JDBCEngineConnConstant.java index 9e19ea9f50..16f6a7e324 100644 --- a/linkis-engineconn-plugins/jdbc/src/main/java/org/apache/linkis/manager/engineplugin/jdbc/constant/JDBCEngineConnConstant.java +++ b/linkis-engineconn-plugins/jdbc/src/main/java/org/apache/linkis/manager/engineplugin/jdbc/constant/JDBCEngineConnConstant.java @@ -34,6 +34,11 @@ private JDBCEngineConnConstant() {} "wds.linkis.jdbc.auth.kerberos.proxy.enable"; public static final String JDBC_PROXY_USER_PROPERTY = "wds.linkis.jdbc.proxy.user.property"; public static final String JDBC_PROXY_USER = "wds.linkis.jdbc.proxy.user"; + + public static final String JDBC_CONNECTION_TIMEOUT = "wds.linkis.jdbc.connection.timeout"; + + public static final String JDBC_SOCKET_TIMEOUT = "wds.linkis.jdbc.socket.timeout"; + public static final String JDBC_SCRIPTS_EXEC_USER = "execUser"; public static final String JDBC_ENGINE_RUN_TIME_DS = "wds.linkis.engine.runtime.datasource"; public static final String JDBC_ENGINE_RUN_TIME_DS_MAX_VERSION_ID = diff --git a/linkis-engineconn-plugins/jdbc/src/main/scala/org/apache/linkis/manager/engineplugin/jdbc/executor/JDBCEngineConnExecutor.scala b/linkis-engineconn-plugins/jdbc/src/main/scala/org/apache/linkis/manager/engineplugin/jdbc/executor/JDBCEngineConnExecutor.scala index 34d03a2b3a..195d4fa74c 100644 --- a/linkis-engineconn-plugins/jdbc/src/main/scala/org/apache/linkis/manager/engineplugin/jdbc/executor/JDBCEngineConnExecutor.scala +++ b/linkis-engineconn-plugins/jdbc/src/main/scala/org/apache/linkis/manager/engineplugin/jdbc/executor/JDBCEngineConnExecutor.scala @@ -81,6 +81,8 @@ class JDBCEngineConnExecutor(override val outputPrintLimit: Int, val id: Int) private val progressMonitors: util.Map[String, ProgressMonitor[_]] = new ConcurrentHashMap[String, ProgressMonitor[_]]() + private val connectionCache: util.Map[String, Connection] = new util.HashMap[String, Connection]() + override def init(): Unit = { logger.info("jdbc executor start init.") setCodeParser(new SQLCodeParser) @@ -90,53 +92,65 @@ class JDBCEngineConnExecutor(override val outputPrintLimit: Int, val id: Int) } } - override def executeLine( - engineExecutorContext: EngineExecutionContext, - code: String - ): ExecuteResponse = { - val realCode = code.trim() - val taskId = engineExecutorContext.getJobId.get + override def execute(engineConnTask: EngineConnTask): ExecuteResponse = { + val executeResponse = super.execute(engineConnTask) + if (StringUtils.isNotBlank(engineConnTask.getTaskId)) { + val connection = connectionCache.remove(engineConnTask.getTaskId) + logger.info(s"remove task ${engineConnTask.getTaskId} connection") + Utils.tryAndWarn(connection.close()) + } + executeResponse + } - var properties: util.Map[String, String] = Collections.emptyMap() + private def getConnection(engineExecutorContext: EngineExecutionContext): Connection = { - Utils.tryCatch({ - properties = getJDBCRuntimeParams(engineExecutorContext) - }) { e: Throwable => - logger.error(s"try to build JDBC runtime params error! $e") - return ErrorExecuteResponse(e.getMessage, e) + val taskId = engineExecutorContext.getJobId.orNull + if (StringUtils.isNotBlank(taskId) && connectionCache.containsKey(taskId)) { + logger.info( + s"Task ${taskId} paragraph ${engineExecutorContext.getCurrentParagraph} from cache get connection" + ) + return connectionCache.get(taskId) } - + val properties: util.Map[String, String] = getJDBCRuntimeParams(engineExecutorContext) logger.info(s"The jdbc properties is: $properties") val dataSourceName = properties.get(JDBCEngineConnConstant.JDBC_ENGINE_RUN_TIME_DS) val dataSourceMaxVersionId = properties.get(JDBCEngineConnConstant.JDBC_ENGINE_RUN_TIME_DS_MAX_VERSION_ID) logger.info( - s"The data source name is [$dataSourceName], and the jdbc client begins to run jdbc code:\n ${realCode.trim}" + s"The data source name is [$dataSourceName], and the jdbc client begins to run task ${taskId}" ) - var connection: Connection = null - var statement: Statement = null - var resultSet: ResultSet = null logger.info(s"The data source properties is $properties") - Utils.tryCatch({ - /* url + user as the cache key */ - val jdbcUrl: String = properties.get(JDBCEngineConnConstant.JDBC_URL) - val execUser: String = properties.get(JDBCEngineConnConstant.JDBC_SCRIPTS_EXEC_USER) - val proxyUser: String = properties.get(JDBCEngineConnConstant.JDBC_PROXY_USER_PROPERTY) - var dataSourceIdentifier = s"$jdbcUrl-$execUser-$proxyUser" - /* If datasource is used, use datasource name as the cache key */ - if (StringUtils.isNotBlank(dataSourceName)) { - dataSourceIdentifier = s"$dataSourceName-$dataSourceMaxVersionId" - } - connection = connectionManager.getConnection(dataSourceIdentifier, properties) - logger.info("The jdbc connection has created successfully!") - }) { e: Throwable => - logger.error(s"created data source connection error! $e") - return ErrorExecuteResponse(e.getMessage, e) + /* url + user as the cache key */ + val jdbcUrl: String = properties.get(JDBCEngineConnConstant.JDBC_URL) + val execUser: String = properties.get(JDBCEngineConnConstant.JDBC_SCRIPTS_EXEC_USER) + val proxyUser: String = properties.get(JDBCEngineConnConstant.JDBC_PROXY_USER_PROPERTY) + var dataSourceIdentifier = s"$jdbcUrl-$execUser-$proxyUser" + /* If datasource is used, use datasource name as the cache key */ + if (StringUtils.isNotBlank(dataSourceName)) { + dataSourceIdentifier = s"$dataSourceName-$dataSourceMaxVersionId" + } + val connection = connectionManager.getConnection(dataSourceIdentifier, properties) + if (StringUtils.isNotBlank(taskId)) { + connectionCache.put(taskId, connection) } + connection + } + + override def executeLine( + engineExecutorContext: EngineExecutionContext, + code: String + ): ExecuteResponse = { + + val taskId = engineExecutorContext.getJobId.get + val connection: Connection = getConnection(engineExecutorContext) + var statement: Statement = null + var resultSet: ResultSet = null try { statement = connection.createStatement() - statement.setQueryTimeout(JDBCConfiguration.JDBC_QUERY_TIMEOUT.getValue) + if (statement.getQueryTimeout == 0) { + statement.setQueryTimeout(JDBCConfiguration.JDBC_QUERY_TIMEOUT.getValue) + } statement.setFetchSize(outputPrintLimit) statement.setMaxRows(outputPrintLimit) @@ -170,14 +184,10 @@ class JDBCEngineConnExecutor(override val outputPrintLimit: Int, val id: Int) } } finally { if (resultSet != null) { - Utils.tryCatch({ resultSet.close() }) { case e: SQLException => - logger.warn(e.getMessage) - } + Utils.tryAndWarn(resultSet.close()) } if (statement != null) { - Utils.tryCatch({ statement.close() }) { case e: SQLException => - logger.warn(e.getMessage) - } + Utils.tryAndWarn(statement.close()) } } } catch { @@ -185,14 +195,6 @@ class JDBCEngineConnExecutor(override val outputPrintLimit: Int, val id: Int) logger.error(s"Cannot run $code", e) return ErrorExecuteResponse(e.getMessage, e) } finally { - if (connection != null) { - try { - if (!connection.getAutoCommit) connection.commit() - connection.close() - } catch { - case e: SQLException => logger.warn("close connection error.", e) - } - } connectionManager.removeStatement(taskId) } SuccessExecuteResponse() diff --git a/linkis-engineconn-plugins/openlookeng/pom.xml b/linkis-engineconn-plugins/openlookeng/pom.xml deleted file mode 100644 index 7f9f02f3af..0000000000 --- a/linkis-engineconn-plugins/openlookeng/pom.xml +++ /dev/null @@ -1,140 +0,0 @@ - - - - 4.0.0 - - - org.apache.linkis - linkis - ${revision} - ../../pom.xml - - - linkis-engineplugin-openlookeng - - - 1.5.0 - - - - - org.apache.linkis - linkis-engineconn-plugin-core - ${project.version} - - - - org.apache.linkis - linkis-computation-engineconn - ${project.version} - - - - org.apache.linkis - linkis-storage - ${project.version} - provided - - - - org.apache.linkis - linkis-common - ${project.version} - provided - - - - - io.hetu.core - presto-client - ${openlookeng.version} - - - io.hetu.core - presto-spi - - - io.hetu.core - hetu-transport - - - com.google.inject - guice - - - - - - org.apache.linkis - linkis-rpc - ${project.version} - provided - - - - - - - - - net.alchim31.maven - scala-maven-plugin - - - - org.apache.maven.plugins - maven-assembly-plugin - false - - false - out - false - false - - src/main/assembly/distribution.xml - - - - - make-assembly - - single - - package - - - src/main/assembly/distribution.xml - - - - - - - - - diff --git a/linkis-engineconn-plugins/openlookeng/src/main/assembly/distribution.xml b/linkis-engineconn-plugins/openlookeng/src/main/assembly/distribution.xml deleted file mode 100644 index ba67212cf8..0000000000 --- a/linkis-engineconn-plugins/openlookeng/src/main/assembly/distribution.xml +++ /dev/null @@ -1,70 +0,0 @@ - - - - - linkis-engineplugin-openlookeng - - dir - zip - - true - openlookeng - - - - - - /dist/${openlookeng.version}/lib - true - true - false - false - true - - - - - - - ${basedir}/src/main/resources - - linkis-engineconn.properties - log4j2.xml - - 0777 - dist/${openlookeng.version}/conf - unix - - - - ${basedir}/target - - *.jar - - - *doc.jar - - 0777 - plugin/${openlookeng.version} - - - - - - diff --git a/linkis-engineconn-plugins/openlookeng/src/main/java/org/apache/linkis/engineplugin/openlookeng/conf/OpenLooKengConfiguration.java b/linkis-engineconn-plugins/openlookeng/src/main/java/org/apache/linkis/engineplugin/openlookeng/conf/OpenLooKengConfiguration.java deleted file mode 100644 index b203a5c66e..0000000000 --- a/linkis-engineconn-plugins/openlookeng/src/main/java/org/apache/linkis/engineplugin/openlookeng/conf/OpenLooKengConfiguration.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.engineplugin.openlookeng.conf; - -import org.apache.linkis.common.conf.CommonVars; - -public class OpenLooKengConfiguration { - - public static final CommonVars OPENLOOKENG_CONCURRENT_LIMIT = - CommonVars.apply("linkis.openlookeng.engineconn.concurrent.limit", 100); - public static final CommonVars OPENLOOKENG_HTTP_CONNECT_TIME_OUT = - CommonVars.apply("linkis.openlookeng.http.connectTimeout", 60L); - public static final CommonVars OPENLOOKENG_HTTP_READ_TIME_OUT = - CommonVars.apply("linkis.openlookeng.http.readTimeout", 60L); - public static final CommonVars ENGINE_DEFAULT_LIMIT = - CommonVars.apply("linkis.openlookeng.default.limit", 5000); - public static final CommonVars OPENLOOKENG_URL = - CommonVars.apply("linkis.openlookeng.url", "http://127.0.0.1:8080"); - public static final CommonVars OPENLOOKENG_RESOURCE_CONFIG_PATH = - CommonVars.apply("linkis.openlookeng.resource.config", ""); - public static final CommonVars OPENLOOKENG_USER_NAME = - CommonVars.apply("linkis.openlookeng.username", "default"); - public static final CommonVars OPENLOOKENG_PASSWORD = - CommonVars.apply("linkis.openlookeng.password", ""); - public static final CommonVars OPENLOOKENG_CATALOG = - CommonVars.apply("linkis.openlookeng.catalog", "system"); - public static final CommonVars OPENLOOKENG_SCHEMA = - CommonVars.apply("linkis.openlookeng.schema", ""); - public static final CommonVars OPENLOOKENG_SOURCE = - CommonVars.apply("linkis.openlookeng.source", "global"); - public static final CommonVars OPENLOOKENG_REQUEST_MEMORY = - CommonVars.apply("openlookeng.session.query_max_total_memory", "8GB"); -} diff --git a/linkis-engineconn-plugins/openlookeng/src/main/java/org/apache/linkis/engineplugin/openlookeng/conf/OpenLooKengEngineConfCache.java b/linkis-engineconn-plugins/openlookeng/src/main/java/org/apache/linkis/engineplugin/openlookeng/conf/OpenLooKengEngineConfCache.java deleted file mode 100644 index 0f1a3f6c8f..0000000000 --- a/linkis-engineconn-plugins/openlookeng/src/main/java/org/apache/linkis/engineplugin/openlookeng/conf/OpenLooKengEngineConfCache.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.engineplugin.openlookeng.conf; - -import org.apache.linkis.common.conf.Configuration; -import org.apache.linkis.governance.common.protocol.conf.RequestQueryEngineConfigWithGlobalConfig; -import org.apache.linkis.governance.common.protocol.conf.ResponseQueryConfig; -import org.apache.linkis.manager.label.entity.engine.EngineTypeLabel; -import org.apache.linkis.manager.label.entity.engine.UserCreatorLabel; -import org.apache.linkis.rpc.Sender; - -import java.util.Map; - -public class OpenLooKengEngineConfCache { - - public static Map getConfMap( - UserCreatorLabel userCreatorLabel, EngineTypeLabel engineTypeLabel) { - Sender sender = - Sender.getSender( - Configuration.CLOUD_CONSOLE_CONFIGURATION_SPRING_APPLICATION_NAME().getValue()); - Object any = - sender.ask( - new RequestQueryEngineConfigWithGlobalConfig(userCreatorLabel, engineTypeLabel, null)); - if (any instanceof ResponseQueryConfig) { - return ((ResponseQueryConfig) any).getKeyAndValue(); - } - return null; - } -} diff --git a/linkis-engineconn-plugins/openlookeng/src/main/java/org/apache/linkis/engineplugin/openlookeng/errorcode/OpenLooKengErrorCodeSummary.java b/linkis-engineconn-plugins/openlookeng/src/main/java/org/apache/linkis/engineplugin/openlookeng/errorcode/OpenLooKengErrorCodeSummary.java deleted file mode 100644 index 68e85432f7..0000000000 --- a/linkis-engineconn-plugins/openlookeng/src/main/java/org/apache/linkis/engineplugin/openlookeng/errorcode/OpenLooKengErrorCodeSummary.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.engineplugin.openlookeng.errorcode; - -import org.apache.linkis.common.errorcode.ErrorCodeUtils; -import org.apache.linkis.common.errorcode.LinkisErrorCode; - -public enum OpenLooKengErrorCodeSummary implements LinkisErrorCode { - OPENLOOKENG_CLIENT_ERROR(26030, "openlookeng client error(openlookeng客户端异常)"), - - OPENLOOKENG_STATUS_ERROR( - 26031, "openlookeng status error,statement is not finished(openlookeng状态异常, 查询语句未完成)"); - - private final int errorCode; - - private final String errorDesc; - - OpenLooKengErrorCodeSummary(int errorCode, String errorDesc) { - ErrorCodeUtils.validateErrorCode(errorCode, 26000, 29999); - this.errorCode = errorCode; - this.errorDesc = errorDesc; - } - - @Override - public int getErrorCode() { - return errorCode; - } - - @Override - public String getErrorDesc() { - return errorDesc; - } -} diff --git a/linkis-engineconn-plugins/openlookeng/src/main/java/org/apache/linkis/engineplugin/openlookeng/executor/OpenLooKengEngineConnExecutor.java b/linkis-engineconn-plugins/openlookeng/src/main/java/org/apache/linkis/engineplugin/openlookeng/executor/OpenLooKengEngineConnExecutor.java deleted file mode 100644 index 9ce2703451..0000000000 --- a/linkis-engineconn-plugins/openlookeng/src/main/java/org/apache/linkis/engineplugin/openlookeng/executor/OpenLooKengEngineConnExecutor.java +++ /dev/null @@ -1,419 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.engineplugin.openlookeng.executor; - -import org.apache.linkis.common.io.MetaData; -import org.apache.linkis.common.io.Record; -import org.apache.linkis.common.io.resultset.ResultSetWriter; -import org.apache.linkis.engineconn.common.conf.EngineConnConf; -import org.apache.linkis.engineconn.common.conf.EngineConnConstant; -import org.apache.linkis.engineconn.computation.executor.entity.EngineConnTask; -import org.apache.linkis.engineconn.computation.executor.execute.ConcurrentComputationExecutor; -import org.apache.linkis.engineconn.computation.executor.execute.EngineExecutionContext; -import org.apache.linkis.engineconn.core.EngineConnObject; -import org.apache.linkis.engineplugin.openlookeng.conf.OpenLooKengConfiguration; -import org.apache.linkis.engineplugin.openlookeng.conf.OpenLooKengEngineConfCache; -import org.apache.linkis.engineplugin.openlookeng.exception.OpenLooKengClientException; -import org.apache.linkis.engineplugin.openlookeng.exception.OpenLooKengStateInvalidException; -import org.apache.linkis.governance.common.paser.SQLCodeParser; -import org.apache.linkis.manager.common.entity.resource.CommonNodeResource; -import org.apache.linkis.manager.common.entity.resource.NodeResource; -import org.apache.linkis.manager.engineplugin.common.util.NodeResourceUtils; -import org.apache.linkis.manager.label.entity.Label; -import org.apache.linkis.manager.label.entity.engine.EngineTypeLabel; -import org.apache.linkis.manager.label.entity.engine.UserCreatorLabel; -import org.apache.linkis.manager.label.utils.LabelUtil; -import org.apache.linkis.protocol.engine.JobProgressInfo; -import org.apache.linkis.rpc.Sender; -import org.apache.linkis.scheduler.executer.ErrorExecuteResponse; -import org.apache.linkis.scheduler.executer.ExecuteResponse; -import org.apache.linkis.scheduler.executer.SuccessExecuteResponse; -import org.apache.linkis.storage.domain.DataType; -import org.apache.linkis.storage.resultset.ResultSetFactory$; -import org.apache.linkis.storage.resultset.table.TableMetaData; -import org.apache.linkis.storage.resultset.table.TableRecord; - -import org.apache.commons.io.IOUtils; -import org.apache.commons.lang3.exception.ExceptionUtils; - -import java.io.IOException; -import java.net.URI; -import java.time.ZoneId; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.Set; -import java.util.TimeZone; -import java.util.concurrent.TimeUnit; - -import com.google.common.cache.Cache; -import com.google.common.cache.CacheBuilder; -import io.prestosql.client.ClientSelectedRole; -import io.prestosql.client.ClientSession; -import io.prestosql.client.QueryError; -import io.prestosql.client.QueryStatusInfo; -import io.prestosql.client.SocketChannelSocketFactory; -import io.prestosql.client.StatementClient; -import io.prestosql.client.StatementClientFactory; -import okhttp3.OkHttpClient; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import static org.apache.linkis.engineplugin.openlookeng.conf.OpenLooKengConfiguration.OPENLOOKENG_HTTP_CONNECT_TIME_OUT; -import static org.apache.linkis.engineplugin.openlookeng.conf.OpenLooKengConfiguration.OPENLOOKENG_HTTP_READ_TIME_OUT; -import static org.apache.linkis.engineplugin.openlookeng.errorcode.OpenLooKengErrorCodeSummary.OPENLOOKENG_CLIENT_ERROR; -import static org.apache.linkis.engineplugin.openlookeng.errorcode.OpenLooKengErrorCodeSummary.OPENLOOKENG_STATUS_ERROR; - -public class OpenLooKengEngineConnExecutor extends ConcurrentComputationExecutor { - - private static final Logger LOG = LoggerFactory.getLogger(OpenLooKengEngineConnExecutor.class); - - private int id; - - private OkHttpClient okHttpClient = - new OkHttpClient.Builder() - .socketFactory(new SocketChannelSocketFactory()) - .connectTimeout(OPENLOOKENG_HTTP_CONNECT_TIME_OUT.getValue(), TimeUnit.SECONDS) - .readTimeout(OPENLOOKENG_HTTP_READ_TIME_OUT.getValue(), TimeUnit.SECONDS) - .build(); - - private List> executorLabels = new ArrayList>(); - - private Cache clientSessionCache = - CacheBuilder.newBuilder() - .expireAfterAccess( - Long.valueOf(EngineConnConf.ENGINE_TASK_EXPIRE_TIME().getValue().toString()), - TimeUnit.MILLISECONDS) - .maximumSize(EngineConnConstant.MAX_TASK_NUM()) - .build(); - - public OpenLooKengEngineConnExecutor(int outputPrintLimit, int id) { - super(outputPrintLimit); - this.id = id; - } - - @Override - public void init() { - setCodeParser(new SQLCodeParser()); - super.init(); - } - - @Override - public ExecuteResponse execute(EngineConnTask engineConnTask) { - List> labelList = Arrays.asList(engineConnTask.getLables()); - UserCreatorLabel userCreatorLabel = LabelUtil.getUserCreatorLabel(labelList); - String user = userCreatorLabel.getUser(); - EngineTypeLabel engineTypeLabel = LabelUtil.getEngineTypeLabel(labelList); - clientSessionCache.put( - engineConnTask.getTaskId(), - getClientSession( - user, - engineConnTask.getProperties(), - OpenLooKengEngineConfCache.getConfMap(userCreatorLabel, engineTypeLabel))); - return super.execute(engineConnTask); - } - - @Override - public ExecuteResponse executeLine(EngineExecutionContext engineExecutorContext, String code) { - - String taskId = engineExecutorContext.getJobId().get(); - - ClientSession clientSession = clientSessionCache.getIfPresent(taskId); - StatementClient statement = - StatementClientFactory.newStatementClient(okHttpClient, clientSession, code); - - initialStatusUpdates(taskId, engineExecutorContext, statement); - - try { - if (statement.isRunning() - || (statement.isFinished() && statement.finalStatusInfo().getError() == null)) { - queryOutput(taskId, engineExecutorContext, statement); - } - - ErrorExecuteResponse errorResponse = - verifyServerError(taskId, engineExecutorContext, statement); - if (errorResponse == null) { - // update session - clientSessionCache.put(taskId, updateSession(clientSession, statement)); - return new SuccessExecuteResponse(); - } else { - return errorResponse; - } - } catch (Exception e) { - return new ErrorExecuteResponse(e.getMessage(), e); - } - } - - @Override - public ExecuteResponse executeCompletely( - EngineExecutionContext engineExecutorContext, String code, String completedLine) { - return null; - } - - @Override - public float progress(String taskID) { - return 0; - } - - @Override - public JobProgressInfo[] getProgressInfo(String taskID) { - return new JobProgressInfo[0]; - } - - @Override - public boolean supportCallBackLogs() { - return false; - } - - @Override - public String getId() { - return Sender.getThisServiceInstance().getInstance() + this.id; - } - - @Override - public void killAll() {} - - @Override - public List> getExecutorLabels() { - return executorLabels; - } - - @Override - public void setExecutorLabels(List> labels) { - if (null != labels && !labels.isEmpty()) { - executorLabels.clear(); - executorLabels.addAll(labels); - } - } - - @Override - public NodeResource requestExpectedResource(NodeResource expectedResource) { - return null; - } - - @Override - public NodeResource getCurrentNodeResource() { - CommonNodeResource resource = new CommonNodeResource(); - resource.setUsedResource( - NodeResourceUtils.applyAsLoadInstanceResource( - EngineConnObject.getEngineCreationContext().getOptions())); - return resource; - } - - private ClientSession getClientSession( - String user, Map taskParams, Map cacheMap) { - Map configMap = new HashMap<>(); - if (null != cacheMap && !cacheMap.isEmpty()) { - configMap.putAll(cacheMap); - } - - for (Map.Entry keyValue : taskParams.entrySet()) { - configMap.put(keyValue.getKey(), String.valueOf(keyValue.getValue())); - } - - URI httpUri = URI.create(OpenLooKengConfiguration.OPENLOOKENG_URL.getValue(configMap)); - String source = OpenLooKengConfiguration.OPENLOOKENG_SOURCE.getValue(configMap); - String catalog = OpenLooKengConfiguration.OPENLOOKENG_CATALOG.getValue(configMap); - String schema = OpenLooKengConfiguration.OPENLOOKENG_SCHEMA.getValue(configMap); - - Map properties = new HashMap<>(); - - for (Map.Entry keyValue : configMap.entrySet()) { - if (keyValue.getKey().startsWith("presto.session.")) { - properties.put( - keyValue.getKey().substring("presto.session.".length()), keyValue.getValue()); - } - } - - String clientInfo = "Linkis"; - String transactionId = null; - Optional traceToken = Optional.empty(); - Set clientTags = Collections.emptySet(); - ZoneId timeZonId = TimeZone.getDefault().toZoneId(); - Locale locale = Locale.getDefault(); - Map resourceEstimates = Collections.emptyMap(); - Map preparedStatements = Collections.emptyMap(); - Map roles = Collections.emptyMap(); - Map extraCredentials = Collections.emptyMap(); - - io.airlift.units.Duration clientRequestTimeout = - new io.airlift.units.Duration(0, TimeUnit.MILLISECONDS); - - ClientSession session = - new ClientSession( - httpUri, - user, - source, - traceToken, - clientTags, - clientInfo, - catalog, - schema, - "", - timeZonId, - locale, - resourceEstimates, - properties, - preparedStatements, - roles, - extraCredentials, - transactionId, - clientRequestTimeout); - return session; - } - - private void initialStatusUpdates( - String taskId, EngineExecutionContext engineExecutorContext, StatementClient statement) { - while (statement.isRunning() - && (statement.currentData().getData() == null - || statement.currentStatusInfo().getUpdateType() != null)) { - engineExecutorContext.pushProgress(progress(taskId), getProgressInfo(taskId)); - statement.advance(); - } - } - - private void queryOutput( - String taskId, EngineExecutionContext engineExecutorContext, StatementClient statement) - throws IOException { - int columnCount = 0; - int rows = 0; - ResultSetWriter resultSetWriter = - engineExecutorContext.createResultSetWriter(ResultSetFactory$.MODULE$.TABLE_TYPE()); - try { - QueryStatusInfo results = null; - if (statement.isRunning()) { - results = statement.currentStatusInfo(); - } else { - results = statement.finalStatusInfo(); - } - if (results.getColumns() == null) { - throw new RuntimeException("openlookeng columns is null."); - } - org.apache.linkis.storage.domain.Column[] columns = - results.getColumns().stream() - .map( - column -> - new org.apache.linkis.storage.domain.Column( - column.getName(), DataType.toDataType(column.getType()), "")) - .toArray(org.apache.linkis.storage.domain.Column[]::new); - columnCount = columns.length; - resultSetWriter.addMetaData(new TableMetaData(columns)); - while (statement.isRunning()) { - Iterable> data = statement.currentData().getData(); - if (data != null) { - for (List row : data) { - Object[] rowArray = row.stream().map(String::valueOf).toArray(); - resultSetWriter.addRecord(new TableRecord(rowArray)); - rows += 1; - } - } - engineExecutorContext.pushProgress(progress(taskId), getProgressInfo(taskId)); - statement.advance(); - } - LOG.warn("Fetched {} col(s) : {} row(s) in openlookeng", columnCount, rows); - engineExecutorContext.sendResultSet(resultSetWriter); - } catch (Exception e) { - IOUtils.closeQuietly(resultSetWriter); - throw e; - } - } - - // check error - private ErrorExecuteResponse verifyServerError( - String taskId, EngineExecutionContext engineExecutorContext, StatementClient statement) - throws OpenLooKengClientException, OpenLooKengStateInvalidException { - engineExecutorContext.pushProgress(progress(taskId), getProgressInfo(taskId)); - if (statement.isFinished()) { - QueryStatusInfo info = statement.finalStatusInfo(); - if (info.getError() != null) { - QueryError error = Objects.requireNonNull(info.getError()); - String message = "openlookeng execute failed (#" + info.getId() + "):" + error.getMessage(); - Throwable cause = null; - if (error.getFailureInfo() != null) { - cause = error.getFailureInfo().toException(); - } - String errorString = ""; - if (cause == null) { - errorString = ExceptionUtils.getStackTrace(cause); - } - return new ErrorExecuteResponse(ExceptionUtils.getMessage(cause), cause); - } - } else if (statement.isClientAborted()) { - LOG.warn("openlookeng statement is killed."); - } else if (statement.isClientError()) { - throw new OpenLooKengClientException( - OPENLOOKENG_CLIENT_ERROR.getErrorCode(), OPENLOOKENG_CLIENT_ERROR.getErrorDesc()); - } else { - throw new OpenLooKengStateInvalidException( - OPENLOOKENG_STATUS_ERROR.getErrorCode(), OPENLOOKENG_STATUS_ERROR.getErrorDesc()); - } - return null; - } - - private ClientSession updateSession(ClientSession clientSession, StatementClient statement) { - ClientSession newSession = clientSession; - // update catalog and schema if present - if (statement.getSetCatalog().isPresent() || statement.getSetSchema().isPresent()) { - newSession = - ClientSession.builder(newSession) - .withCatalog(statement.getSetCatalog().orElse(newSession.getCatalog())) - .withSchema(statement.getSetSchema().orElse(newSession.getSchema())) - .build(); - } - - // update transaction ID if necessary - if (statement.isClearTransactionId()) newSession = ClientSession.stripTransactionId(newSession); - - ClientSession.Builder builder = ClientSession.builder(newSession); - - if (statement.getStartedTransactionId() != null) - builder = builder.withTransactionId(statement.getStartedTransactionId()); - - // update session properties if present - if (!statement.getSetSessionProperties().isEmpty() - || !statement.getResetSessionProperties().isEmpty()) { - Map sessionProperties = new HashMap(newSession.getProperties()); - sessionProperties.putAll(statement.getSetSessionProperties()); - sessionProperties.keySet().removeAll(statement.getResetSessionProperties()); - builder = builder.withProperties(sessionProperties); - } - - // update session roles - if (!statement.getSetRoles().isEmpty()) { - Map roles = new HashMap(newSession.getRoles()); - roles.putAll(statement.getSetRoles()); - builder = builder.withRoles(roles); - } - - // update prepared statements if present - if (!statement.getAddedPreparedStatements().isEmpty() - || !statement.getDeallocatedPreparedStatements().isEmpty()) { - Map preparedStatements = new HashMap(newSession.getPreparedStatements()); - preparedStatements.putAll(statement.getAddedPreparedStatements()); - preparedStatements.keySet().removeAll(statement.getDeallocatedPreparedStatements()); - builder = builder.withPreparedStatements(preparedStatements); - } - return builder.build(); - } -} diff --git a/linkis-engineconn-plugins/openlookeng/src/main/resources/linkis-engineconn.properties b/linkis-engineconn-plugins/openlookeng/src/main/resources/linkis-engineconn.properties deleted file mode 100644 index c20fc24e6f..0000000000 --- a/linkis-engineconn-plugins/openlookeng/src/main/resources/linkis-engineconn.properties +++ /dev/null @@ -1,27 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# -wds.linkis.server.version=v1 - -wds.linkis.engineconn.debug.enable=true - -#wds.linkis.keytab.enable=true - -wds.linkis.engineconn.plugin.default.class=org.apache.linkis.engineplugin.openlookeng.OpenLooKengECPlugin - -wds.linkis.engineconn.support.parallelism=true - -wds.linkis.engineconn.max.free.time=0 diff --git a/linkis-engineconn-plugins/openlookeng/src/main/resources/log4j2.xml b/linkis-engineconn-plugins/openlookeng/src/main/resources/log4j2.xml deleted file mode 100644 index c59ec8ac61..0000000000 --- a/linkis-engineconn-plugins/openlookeng/src/main/resources/log4j2.xml +++ /dev/null @@ -1,84 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/linkis-engineconn-plugins/openlookeng/src/main/scala/org/apache/linkis/engineplugin/openlookeng/OpenLooKengECPlugin.scala b/linkis-engineconn-plugins/openlookeng/src/main/scala/org/apache/linkis/engineplugin/openlookeng/OpenLooKengECPlugin.scala deleted file mode 100644 index dc3197bd70..0000000000 --- a/linkis-engineconn-plugins/openlookeng/src/main/scala/org/apache/linkis/engineplugin/openlookeng/OpenLooKengECPlugin.scala +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.engineplugin.openlookeng - -import org.apache.linkis.engineplugin.openlookeng.builder.OpenLooKengProcessECLaunchBuilder -import org.apache.linkis.engineplugin.openlookeng.factory.OpenLooKengEngineConnFactory -import org.apache.linkis.manager.engineplugin.common.EngineConnPlugin -import org.apache.linkis.manager.engineplugin.common.creation.EngineConnFactory -import org.apache.linkis.manager.engineplugin.common.launch.EngineConnLaunchBuilder -import org.apache.linkis.manager.engineplugin.common.resource.{ - EngineResourceFactory, - GenericEngineResourceFactory -} -import org.apache.linkis.manager.label.entity.Label -import org.apache.linkis.manager.label.entity.engine.EngineType -import org.apache.linkis.manager.label.utils.EngineTypeLabelCreator - -import java.util -import java.util.List - -class OpenLooKengECPlugin extends EngineConnPlugin { - - private val resourceLocker = new Object() - - private val engineLaunchBuilderLocker = new Object() - - private val engineFactoryLocker = new Object() - - private var engineResourceFactory: EngineResourceFactory = _ - - private var engineLaunchBuilder: EngineConnLaunchBuilder = _ - - private var engineFactory: EngineConnFactory = _ - - private val defaultLabels: List[Label[_]] = new util.ArrayList[Label[_]]() - - override def init(params: util.Map[String, AnyRef]): Unit = { - val engineTypeLabel = - EngineTypeLabelCreator.createEngineTypeLabel(EngineType.OPENLOOKENG.toString) - this.defaultLabels.add(engineTypeLabel) - } - - override def getEngineResourceFactory: EngineResourceFactory = { - if (null == engineResourceFactory) resourceLocker synchronized { - engineResourceFactory = new GenericEngineResourceFactory - } - engineResourceFactory - } - - override def getEngineConnLaunchBuilder: EngineConnLaunchBuilder = { - new OpenLooKengProcessECLaunchBuilder - } - - override def getEngineConnFactory: EngineConnFactory = { - if (null == engineFactory) engineFactoryLocker synchronized { - engineFactory = new OpenLooKengEngineConnFactory - } - engineFactory - } - - override def getDefaultLabels: util.List[Label[_]] = { - this.defaultLabels - } - -} diff --git a/linkis-engineconn-plugins/openlookeng/src/main/scala/org/apache/linkis/engineplugin/openlookeng/factory/OpenLooKengEngineConnFactory.scala b/linkis-engineconn-plugins/openlookeng/src/main/scala/org/apache/linkis/engineplugin/openlookeng/factory/OpenLooKengEngineConnFactory.scala deleted file mode 100644 index 098f5f0374..0000000000 --- a/linkis-engineconn-plugins/openlookeng/src/main/scala/org/apache/linkis/engineplugin/openlookeng/factory/OpenLooKengEngineConnFactory.scala +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.engineplugin.openlookeng.factory - -import org.apache.linkis.common.utils.Logging -import org.apache.linkis.engineconn.common.creation.EngineCreationContext -import org.apache.linkis.engineconn.common.engineconn.EngineConn -import org.apache.linkis.engineconn.computation.executor.creation.ComputationSingleExecutorEngineConnFactory -import org.apache.linkis.engineconn.executor.entity.LabelExecutor -import org.apache.linkis.engineplugin.openlookeng.executor.OpenLooKengEngineConnExecutor -import org.apache.linkis.manager.label.entity.engine.{EngineType, RunType} -import org.apache.linkis.manager.label.entity.engine.EngineType.EngineType -import org.apache.linkis.manager.label.entity.engine.RunType.RunType - -class OpenLooKengEngineConnFactory extends ComputationSingleExecutorEngineConnFactory with Logging { - - override protected def getEngineConnType: EngineType = EngineType.OPENLOOKENG - - override protected def getRunType: RunType = RunType.SQL - - override def newExecutor( - id: Int, - engineCreationContext: EngineCreationContext, - engineConn: EngineConn - ): LabelExecutor = { - new OpenLooKengEngineConnExecutor(100, id) - } - -} diff --git a/linkis-engineconn-plugins/openlookeng/src/test/java/org/apache/linkis/engineplugin/openlookeng/errorcode/OpenLooKengErrorCodeSummaryTest.java b/linkis-engineconn-plugins/openlookeng/src/test/java/org/apache/linkis/engineplugin/openlookeng/errorcode/OpenLooKengErrorCodeSummaryTest.java deleted file mode 100644 index b1a67a54ea..0000000000 --- a/linkis-engineconn-plugins/openlookeng/src/test/java/org/apache/linkis/engineplugin/openlookeng/errorcode/OpenLooKengErrorCodeSummaryTest.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.engineplugin.openlookeng.errorcode; - -import org.junit.jupiter.api.Test; - -import static org.apache.linkis.engineplugin.openlookeng.errorcode.OpenLooKengErrorCodeSummary.OPENLOOKENG_CLIENT_ERROR; -import static org.apache.linkis.engineplugin.openlookeng.errorcode.OpenLooKengErrorCodeSummary.OPENLOOKENG_STATUS_ERROR; -import static org.junit.jupiter.api.Assertions.assertEquals; - -public class OpenLooKengErrorCodeSummaryTest { - @Test - void testGetErrorCode() { - assertEquals(26030, OPENLOOKENG_CLIENT_ERROR.getErrorCode()); - assertEquals(26031, OPENLOOKENG_STATUS_ERROR.getErrorCode()); - } - - @Test - void testGetErrorDesc() { - assertEquals( - "openlookeng client error(openlookeng客户端异常)", OPENLOOKENG_CLIENT_ERROR.getErrorDesc()); - assertEquals( - "openlookeng status error,statement is not finished(openlookeng状态异常, 查询语句未完成)", - OPENLOOKENG_STATUS_ERROR.getErrorDesc()); - } -} diff --git a/linkis-engineconn-plugins/pom.xml b/linkis-engineconn-plugins/pom.xml index b383c540e1..cc86286204 100644 --- a/linkis-engineconn-plugins/pom.xml +++ b/linkis-engineconn-plugins/pom.xml @@ -30,7 +30,6 @@ hive spark python - openlookeng io_file shell pipeline @@ -39,7 +38,6 @@ sqoop presto trino - elasticsearch seatunnel hbase nebula diff --git a/linkis-engineconn-plugins/spark/pom.xml b/linkis-engineconn-plugins/spark/pom.xml index 96131955d9..c011f84975 100644 --- a/linkis-engineconn-plugins/spark/pom.xml +++ b/linkis-engineconn-plugins/spark/pom.xml @@ -34,11 +34,7 @@ com.sun.jersey - jersey-core - - - com.sun.jersey - jersey-client + * @@ -50,11 +46,7 @@ com.sun.jersey - jersey-core - - - com.sun.jersey - jersey-client + * @@ -104,14 +96,9 @@ org.apache.hadoop * - com.sun.jersey - jersey-core - - - com.sun.jersey - jersey-client + * com.google.protobuf @@ -176,8 +163,12 @@ ${project.version} - org.codehaus.jackson - jackson-mapper-asl + io.micrometer + micrometer-core + + + io.micrometer + micrometer-registry-prometheus @@ -197,23 +188,11 @@ com.fasterxml.jackson.core - jackson-annotations - - - com.fasterxml.jackson.core - jackson-core - - - com.fasterxml.jackson.core - jackson-databind - - - org.codehaus.jackson - jackson-core-asl + * org.codehaus.jackson - jackson-mapper-asl + * com.fasterxml.jackson.module @@ -221,11 +200,7 @@ com.sun.jersey - jersey-core - - - com.sun.jersey - jersey-client + * org.apache.commons @@ -262,31 +237,15 @@ com.fasterxml.jackson.core - jackson-annotations - - - com.fasterxml.jackson.core - jackson-core - - - com.fasterxml.jackson.core - jackson-databind - - - org.codehaus.jackson - jackson-core-asl + * org.codehaus.jackson - jackson-mapper-asl - - - com.sun.jersey - jersey-core + * com.sun.jersey - jersey-client + * com.google.protobuf @@ -320,31 +279,15 @@ com.fasterxml.jackson.core - jackson-annotations - - - com.fasterxml.jackson.core - jackson-core - - - com.fasterxml.jackson.core - jackson-databind - - - org.codehaus.jackson - jackson-core-asl + * org.codehaus.jackson - jackson-mapper-asl - - - com.sun.jersey - jersey-core + * com.sun.jersey - jersey-client + * org.codehaus.janino @@ -356,6 +299,10 @@ + + com.sun.jersey.contribs + jersey-apache-client4 + org.apache.hadoop hadoop-auth @@ -444,11 +391,33 @@ ${hadoop.version} provided + - com.sun.jersey.contribs - jersey-apache-client4 - ${jersey.version} + io.fabric8 + kubernetes-client + ${kubernetes-client.version} + + + io.fabric8 + kubernetes-model-common + + + io.fabric8 + kubernetes-model-core + + + + io.fabric8 + kubernetes-model-common + ${kubernetes-client.version} + + + io.fabric8 + kubernetes-model-core + ${kubernetes-client.version} + + diff --git a/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/client/context/SparkConfig.java b/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/client/context/SparkConfig.java index 20b0749d83..1768b77d04 100644 --- a/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/client/context/SparkConfig.java +++ b/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/client/context/SparkConfig.java @@ -17,6 +17,8 @@ package org.apache.linkis.engineplugin.spark.client.context; +import org.apache.commons.lang3.StringUtils; + import java.util.HashMap; import java.util.Map; @@ -25,6 +27,33 @@ public class SparkConfig { private String javaHome; // ("") private String sparkHome; // ("") private String master = "yarn"; // ("yarn") + + private String k8sConfigFile; + + private String k8sServiceAccount; + + private String k8sMasterUrl; + + private String k8sUsername; + + private String k8sPassword; + + private String k8sImage; + + private String k8sImagePullPolicy; + + private String k8sLanguageType; + + private String k8sRestartPolicy; + + private String k8sSparkVersion; + + private String k8sNamespace; + private String k8sFileUploadPath; + + private String k8sDriverRequestCores; + private String k8sExecutorRequestCores; + private String k8sSparkUIPort; private String deployMode = "client"; // ("client") // todo cluster private String appResource; // ("") private String appName; // ("") @@ -50,6 +79,131 @@ public class SparkConfig { private String principal; // ("--principal", "") private String keytab; // ("--keytab", "") private String queue; // ("--queue", "") + private String pyFiles; // ("--py-files", "") + + public String getK8sFileUploadPath() { + return k8sFileUploadPath; + } + + public void setK8sFileUploadPath(String k8sFileUploadPath) { + this.k8sFileUploadPath = k8sFileUploadPath; + } + + public String getK8sImagePullPolicy() { + return k8sImagePullPolicy; + } + + public void setK8sImagePullPolicy(String k8sImagePullPolicy) { + this.k8sImagePullPolicy = k8sImagePullPolicy; + } + + public String getK8sLanguageType() { + return k8sLanguageType; + } + + public void setK8sLanguageType(String k8sLanguageType) { + this.k8sLanguageType = k8sLanguageType; + } + + public String getK8sRestartPolicy() { + return k8sRestartPolicy; + } + + public void setK8sRestartPolicy(String k8sRestartPolicy) { + this.k8sRestartPolicy = k8sRestartPolicy; + } + + public String getK8sSparkVersion() { + return k8sSparkVersion; + } + + public void setK8sSparkVersion(String k8sSparkVersion) { + this.k8sSparkVersion = k8sSparkVersion; + } + + public String getK8sNamespace() { + return k8sNamespace; + } + + public void setK8sNamespace(String k8sNamespace) { + this.k8sNamespace = k8sNamespace; + } + + public String getK8sMasterUrl() { + return k8sMasterUrl; + } + + public String getK8sConfigFile() { + return k8sConfigFile; + } + + public void setK8sConfigFile(String k8sConfigFile) { + if (StringUtils.isNotBlank(k8sConfigFile) && k8sConfigFile.startsWith("~")) { + String user = System.getProperty("user.home"); + k8sConfigFile = k8sConfigFile.replaceFirst("~", user); + } + this.k8sConfigFile = k8sConfigFile; + } + + public String getK8sServiceAccount() { + return k8sServiceAccount; + } + + public void setK8sServiceAccount(String k8sServiceAccount) { + this.k8sServiceAccount = k8sServiceAccount; + } + + public void setK8sMasterUrl(String k8sMasterUrl) { + this.k8sMasterUrl = k8sMasterUrl; + } + + public String getK8sUsername() { + return k8sUsername; + } + + public void setK8sUsername(String k8sUsername) { + this.k8sUsername = k8sUsername; + } + + public String getK8sPassword() { + return k8sPassword; + } + + public void setK8sPassword(String k8sPassword) { + this.k8sPassword = k8sPassword; + } + + public String getK8sImage() { + return k8sImage; + } + + public void setK8sImage(String k8sImage) { + this.k8sImage = k8sImage; + } + + public String getK8sDriverRequestCores() { + return k8sDriverRequestCores; + } + + public void setK8sDriverRequestCores(String k8sDriverRequestCores) { + this.k8sDriverRequestCores = k8sDriverRequestCores; + } + + public String getK8sExecutorRequestCores() { + return k8sExecutorRequestCores; + } + + public void setK8sExecutorRequestCores(String k8sExecutorRequestCores) { + this.k8sExecutorRequestCores = k8sExecutorRequestCores; + } + + public String getK8sSparkUIPort() { + return k8sSparkUIPort; + } + + public void setK8sSparkUIPort(String k8sSparkUIPort) { + this.k8sSparkUIPort = k8sSparkUIPort; + } public String getJavaHome() { return javaHome; @@ -275,6 +429,14 @@ public void setQueue(String queue) { this.queue = queue; } + public String getPyFiles() { + return pyFiles; + } + + public void setPyFiles(String pyFiles) { + this.pyFiles = pyFiles; + } + @Override public String toString() { return "SparkConfig{" @@ -287,6 +449,45 @@ public String toString() { + ", master='" + master + '\'' + + ", k8sConfigFile='" + + k8sConfigFile + + '\'' + + ", k8sServiceAccount='" + + k8sServiceAccount + + '\'' + + ", k8sMasterUrl='" + + k8sMasterUrl + + '\'' + + ", k8sImage='" + + k8sImage + + '\'' + + ", k8sImagePullPolicy='" + + k8sImagePullPolicy + + '\'' + + ", k8sLanguageType='" + + k8sLanguageType + + '\'' + + ", k8sRestartPolicy='" + + k8sRestartPolicy + + '\'' + + ", k8sSparkVersion='" + + k8sSparkVersion + + '\'' + + ", k8sFileUploadPath='" + + k8sFileUploadPath + + '\'' + + ", k8sNamespace='" + + k8sNamespace + + '\'' + + ", k8sDriverRequestCores='" + + k8sDriverRequestCores + + '\'' + + ", k8sExecutorRequestCores='" + + k8sExecutorRequestCores + + '\'' + + ", k8sSparkUIPort='" + + k8sSparkUIPort + + '\'' + ", deployMode='" + deployMode + '\'' @@ -354,6 +555,9 @@ public String toString() { + ", queue='" + queue + '\'' + + ", pyFiles='" + + pyFiles + + '\'' + '}'; } } diff --git a/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/client/deployment/ClusterDescriptorAdapter.java b/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/client/deployment/ClusterDescriptorAdapter.java index 16b7613cfe..1ee44b27c7 100644 --- a/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/client/deployment/ClusterDescriptorAdapter.java +++ b/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/client/deployment/ClusterDescriptorAdapter.java @@ -23,6 +23,8 @@ import org.apache.spark.launcher.SparkAppHandle; import java.io.Closeable; +import java.io.IOException; +import java.util.Map; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -58,6 +60,11 @@ public String toString() { return "ClusterDescriptorAdapter{" + "applicationId=" + sparkAppHandle.getAppId() + '}'; } + public void deployCluster(String mainClass, String args, Map confMap) + throws IOException {} + + public abstract boolean initJobId(); + @Override public void close() { if (sparkAppHandle != null) { diff --git a/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/client/deployment/ClusterDescriptorAdapterFactory.java b/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/client/deployment/ClusterDescriptorAdapterFactory.java index 8e9540dcbc..bc67a33e9f 100644 --- a/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/client/deployment/ClusterDescriptorAdapterFactory.java +++ b/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/client/deployment/ClusterDescriptorAdapterFactory.java @@ -19,11 +19,25 @@ import org.apache.linkis.engineplugin.spark.client.context.ExecutionContext; +import org.apache.commons.lang3.StringUtils; + public class ClusterDescriptorAdapterFactory { public static ClusterDescriptorAdapter create(ExecutionContext executionContext) { + String master = executionContext.getSparkConfig().getMaster(); + ClusterDescriptorAdapter clusterDescriptorAdapter = new YarnApplicationClusterDescriptorAdapter(executionContext); + + if (StringUtils.isNotBlank(master)) { + if (master.equalsIgnoreCase("k8s-operator")) { + clusterDescriptorAdapter = new KubernetesOperatorClusterDescriptorAdapter(executionContext); + } else if (master.equalsIgnoreCase("k8s-native")) { + clusterDescriptorAdapter = + new KubernetesApplicationClusterDescriptorAdapter(executionContext); + } + } + return clusterDescriptorAdapter; } } diff --git a/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/client/deployment/KubernetesApplicationClusterDescriptorAdapter.java b/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/client/deployment/KubernetesApplicationClusterDescriptorAdapter.java new file mode 100644 index 0000000000..73892117ad --- /dev/null +++ b/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/client/deployment/KubernetesApplicationClusterDescriptorAdapter.java @@ -0,0 +1,237 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineplugin.spark.client.deployment; + +import org.apache.linkis.engineplugin.spark.client.context.ExecutionContext; +import org.apache.linkis.engineplugin.spark.client.context.SparkConfig; +import org.apache.linkis.engineplugin.spark.client.deployment.util.KubernetesHelper; + +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.util.Strings; +import org.apache.spark.launcher.CustomSparkSubmitLauncher; +import org.apache.spark.launcher.SparkAppHandle; +import org.apache.spark.launcher.SparkLauncher; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Map; +import java.util.UUID; + +import io.fabric8.kubernetes.api.model.Pod; +import io.fabric8.kubernetes.client.KubernetesClient; +import io.fabric8.kubernetes.client.dsl.PodResource; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class KubernetesApplicationClusterDescriptorAdapter extends ClusterDescriptorAdapter { + private static final Logger logger = + LoggerFactory.getLogger(KubernetesApplicationClusterDescriptorAdapter.class); + + protected SparkConfig sparkConfig; + protected KubernetesClient client; + protected String driverPodName; + protected String namespace; + + public KubernetesApplicationClusterDescriptorAdapter(ExecutionContext executionContext) { + super(executionContext); + this.sparkConfig = executionContext.getSparkConfig(); + this.client = + KubernetesHelper.getKubernetesClient( + this.sparkConfig.getK8sConfigFile(), + this.sparkConfig.getK8sMasterUrl(), + this.sparkConfig.getK8sUsername(), + this.sparkConfig.getK8sPassword()); + } + + public void deployCluster(String mainClass, String args, Map confMap) + throws IOException { + SparkConfig sparkConfig = executionContext.getSparkConfig(); + sparkLauncher = new CustomSparkSubmitLauncher(); + sparkLauncher + .setJavaHome(sparkConfig.getJavaHome()) + .setSparkHome(sparkConfig.getSparkHome()) + .setMaster(sparkConfig.getK8sMasterUrl()) + .setDeployMode("cluster") + .setAppName(sparkConfig.getAppName()) + .setVerbose(true); + this.driverPodName = generateDriverPodName(sparkConfig.getAppName()); + this.namespace = sparkConfig.getK8sNamespace(); + setConf(sparkLauncher, "spark.app.name", sparkConfig.getAppName()); + setConf(sparkLauncher, "spark.ui.port", sparkConfig.getK8sSparkUIPort()); + setConf(sparkLauncher, "spark.kubernetes.namespace", this.namespace); + setConf(sparkLauncher, "spark.kubernetes.container.image", sparkConfig.getK8sImage()); + setConf(sparkLauncher, "spark.kubernetes.driver.pod.name", this.driverPodName); + setConf( + sparkLauncher, + "spark.kubernetes.driver.request.cores", + sparkConfig.getK8sDriverRequestCores()); + setConf( + sparkLauncher, + "spark.kubernetes.executor.request.cores", + sparkConfig.getK8sExecutorRequestCores()); + setConf( + sparkLauncher, + "spark.kubernetes.container.image.pullPolicy", + sparkConfig.getK8sImagePullPolicy()); + setConf( + sparkLauncher, + "spark.kubernetes.authenticate.driver.serviceAccountName", + sparkConfig.getK8sServiceAccount()); + if (confMap != null) confMap.forEach((k, v) -> sparkLauncher.setConf(k, v)); + + addSparkArg(sparkLauncher, "--jars", sparkConfig.getJars()); + addSparkArg(sparkLauncher, "--packages", sparkConfig.getPackages()); + addSparkArg(sparkLauncher, "--exclude-packages", sparkConfig.getExcludePackages()); + addSparkArg(sparkLauncher, "--repositories", sparkConfig.getRepositories()); + addSparkArg(sparkLauncher, "--files", sparkConfig.getFiles()); + addSparkArg(sparkLauncher, "--archives", sparkConfig.getArchives()); + addSparkArg(sparkLauncher, "--driver-memory", sparkConfig.getDriverMemory()); + addSparkArg(sparkLauncher, "--driver-java-options", sparkConfig.getDriverJavaOptions()); + addSparkArg(sparkLauncher, "--driver-library-path", sparkConfig.getDriverLibraryPath()); + addSparkArg(sparkLauncher, "--driver-class-path", sparkConfig.getDriverClassPath()); + addSparkArg(sparkLauncher, "--executor-memory", sparkConfig.getExecutorMemory()); + addSparkArg(sparkLauncher, "--proxy-user", sparkConfig.getProxyUser()); + addSparkArg(sparkLauncher, "--driver-cores", sparkConfig.getDriverCores().toString()); + addSparkArg(sparkLauncher, "--total-executor-cores", sparkConfig.getTotalExecutorCores()); + addSparkArg(sparkLauncher, "--executor-cores", sparkConfig.getExecutorCores().toString()); + addSparkArg(sparkLauncher, "--num-executors", sparkConfig.getNumExecutors().toString()); + addSparkArg(sparkLauncher, "--principal", sparkConfig.getPrincipal()); + addSparkArg(sparkLauncher, "--keytab", sparkConfig.getKeytab()); + addSparkArg(sparkLauncher, "--py-files", sparkConfig.getPyFiles()); + sparkLauncher.setAppResource(sparkConfig.getAppResource()); + sparkLauncher.setMainClass(mainClass); + Arrays.stream(args.split("\\s+")) + .filter(StringUtils::isNotBlank) + .forEach(arg -> sparkLauncher.addAppArgs(arg)); + sparkAppHandle = + sparkLauncher.startApplication( + new SparkAppHandle.Listener() { + @Override + public void stateChanged(SparkAppHandle sparkAppHandle) {} + + @Override + public void infoChanged(SparkAppHandle sparkAppHandle) {} + }); + sparkLauncher.setSparkAppHandle(sparkAppHandle); + } + + private void addSparkArg(SparkLauncher sparkLauncher, String key, String value) { + if (StringUtils.isNotBlank(key) && StringUtils.isNotBlank(value)) { + sparkLauncher.addSparkArg(key, value); + } + } + + private void setConf(SparkLauncher sparkLauncher, String key, String value) { + if (StringUtils.isNotBlank(key) && StringUtils.isNotBlank(value)) { + sparkLauncher.setConf(key, value); + } + } + + public boolean initJobId() { + Pod sparkDriverPod = getSparkDriverPod(); + if (null == sparkDriverPod) { + return false; + } + String sparkDriverPodPhase = sparkDriverPod.getStatus().getPhase(); + String sparkApplicationId = sparkDriverPod.getMetadata().getLabels().get("spark-app-selector"); + + if (Strings.isNotBlank(sparkApplicationId)) { + this.applicationId = sparkApplicationId; + } + if (Strings.isNotBlank(sparkDriverPodPhase)) { + this.jobState = kubernetesPodStateConvertSparkState(sparkDriverPodPhase); + } + + // When the job is not finished, the appId is monitored; otherwise, the status is + // monitored(当任务没结束时，监控appId，反之，则监控状态，这里主要防止任务过早结束，导致一直等待) + return null != getApplicationId() || (jobState != null && jobState.isFinal()); + } + + protected Pod getSparkDriverPod() { + return client.pods().inNamespace(namespace).withName(driverPodName).get(); + } + + public String getSparkUIUrl() { + Pod sparkDriverPod = getSparkDriverPod(); + if (null != sparkDriverPod) { + String sparkDriverPodIP = sparkDriverPod.getStatus().getPodIP(); + if (StringUtils.isNotBlank(sparkDriverPodIP)) { + return sparkDriverPodIP + ":" + this.sparkConfig.getK8sSparkUIPort(); + } else { + logger.info("spark driver pod IP is null, the application may be pending"); + } + } else { + logger.info("spark driver pod is not exist"); + } + return ""; + } + + @Override + public SparkAppHandle.State getJobState() { + Pod sparkDriverPod = getSparkDriverPod(); + if (null != sparkDriverPod) { + String sparkDriverPodPhase = sparkDriverPod.getStatus().getPhase(); + this.jobState = kubernetesPodStateConvertSparkState(sparkDriverPodPhase); + logger.info("Job {} state is {}.", getApplicationId(), this.jobState); + return this.jobState; + } + return null; + } + + @Override + public void close() { + logger.info("Start to close job {}.", getApplicationId()); + client.close(); + if (isDisposed()) { + logger.info("Job has finished, close action return."); + return; + } + PodResource sparkDriverPodResource = + client.pods().inNamespace(namespace).withName(driverPodName); + if (null != sparkDriverPodResource.get()) { + sparkDriverPodResource.delete(); + } + } + + @Override + public boolean isDisposed() { + return this.jobState.isFinal(); + } + + public SparkAppHandle.State kubernetesPodStateConvertSparkState(String kubernetesState) { + if (StringUtils.isBlank(kubernetesState)) { + return SparkAppHandle.State.UNKNOWN; + } + switch (kubernetesState.toUpperCase()) { + case "PENDING": + return SparkAppHandle.State.CONNECTED; + case "RUNNING": + return SparkAppHandle.State.RUNNING; + case "SUCCEEDED": + return SparkAppHandle.State.FINISHED; + case "FAILED": + return SparkAppHandle.State.FAILED; + default: + return SparkAppHandle.State.UNKNOWN; + } + } + + public String generateDriverPodName(String appName) { + return appName + "-" + UUID.randomUUID().toString().replace("-", "") + "-driver"; + } +} diff --git a/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/client/deployment/KubernetesOperatorClusterDescriptorAdapter.java b/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/client/deployment/KubernetesOperatorClusterDescriptorAdapter.java new file mode 100644 index 0000000000..eafa8abec5 --- /dev/null +++ b/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/client/deployment/KubernetesOperatorClusterDescriptorAdapter.java @@ -0,0 +1,249 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineplugin.spark.client.deployment; + +import org.apache.linkis.engineplugin.spark.client.context.ExecutionContext; +import org.apache.linkis.engineplugin.spark.client.context.SparkConfig; +import org.apache.linkis.engineplugin.spark.client.deployment.crds.*; +import org.apache.linkis.engineplugin.spark.client.deployment.util.KubernetesHelper; +import org.apache.linkis.engineplugin.spark.config.SparkConfiguration; + +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.spark.launcher.SparkAppHandle; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; + +import io.fabric8.kubernetes.api.model.ObjectMeta; +import io.fabric8.kubernetes.api.model.apiextensions.v1.CustomResourceDefinition; +import io.fabric8.kubernetes.api.model.apiextensions.v1.CustomResourceDefinitionList; +import io.fabric8.kubernetes.client.CustomResource; +import io.fabric8.kubernetes.client.KubernetesClient; +import io.fabric8.kubernetes.client.Watcher; +import io.fabric8.kubernetes.client.WatcherException; +import io.fabric8.kubernetes.client.dsl.MixedOperation; +import io.fabric8.kubernetes.client.dsl.NonNamespaceOperation; +import io.fabric8.kubernetes.client.dsl.Resource; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class KubernetesOperatorClusterDescriptorAdapter extends ClusterDescriptorAdapter { + private static final Logger logger = + LoggerFactory.getLogger(KubernetesOperatorClusterDescriptorAdapter.class); + + protected SparkConfig sparkConfig; + protected KubernetesClient client; + + public KubernetesOperatorClusterDescriptorAdapter(ExecutionContext executionContext) { + super(executionContext); + this.sparkConfig = executionContext.getSparkConfig(); + this.client = + KubernetesHelper.getKubernetesClient( + this.sparkConfig.getK8sConfigFile(), + this.sparkConfig.getK8sMasterUrl(), + this.sparkConfig.getK8sUsername(), + this.sparkConfig.getK8sPassword()); + } + + public void deployCluster(String mainClass, String args, Map confMap) { + + logger.info( + "The spark k8s operator task start，k8sNamespace: {},appName: {}", + this.sparkConfig.getK8sNamespace(), + this.sparkConfig.getAppName()); + CustomResourceDefinitionList crds = + client.apiextensions().v1().customResourceDefinitions().list(); + + String sparkApplicationCRDName = CustomResource.getCRDName(SparkApplication.class); + List sparkCRDList = + crds.getItems().stream() + .filter(crd -> crd.getMetadata().getName().equals(sparkApplicationCRDName)) + .collect(Collectors.toList()); + if (CollectionUtils.isEmpty(sparkCRDList)) { + throw new RuntimeException("The Spark operator crd does not exist"); + } + + NonNamespaceOperation> + sparkApplicationClient = getSparkApplicationClient(client); + + SparkApplication sparkApplication = + getSparkApplication(sparkConfig.getAppName(), sparkConfig.getK8sNamespace()); + + SparkPodSpec driver = + SparkPodSpec.Builder() + .cores(sparkConfig.getDriverCores()) + .memory(sparkConfig.getDriverMemory()) + .serviceAccount(sparkConfig.getK8sServiceAccount()) + .build(); + + SparkPodSpec executor = + SparkPodSpec.Builder() + .cores(sparkConfig.getExecutorCores()) + .instances(sparkConfig.getNumExecutors()) + .memory(sparkConfig.getExecutorMemory()) + .build(); + + Map sparkConfMap = new HashMap<>(); + sparkConfMap.put( + SparkConfiguration.SPARK_KUBERNETES_FILE_UPLOAD_PATH().key(), + sparkConfig.getK8sFileUploadPath()); + + SparkApplicationSpec sparkApplicationSpec = + SparkApplicationSpec.Builder() + .type(sparkConfig.getK8sLanguageType()) + // todo An error occurs when the client mode is used. The cause has not been found + .mode("cluster") + .image(sparkConfig.getK8sImage()) + .imagePullPolicy(sparkConfig.getK8sImagePullPolicy()) + .mainClass(mainClass) + .mainApplicationFile(sparkConfig.getAppResource()) + .sparkVersion(sparkConfig.getK8sSparkVersion()) + .restartPolicy(new RestartPolicy(sparkConfig.getK8sRestartPolicy())) + .driver(driver) + .executor(executor) + .sparkConf(sparkConfMap) + .build(); + + logger.info("Spark k8s operator task parameters: {}", sparkApplicationSpec); + sparkApplication.setSpec(sparkApplicationSpec); + + SparkApplication created = sparkApplicationClient.createOrReplace(sparkApplication); + logger.info("Preparing to submit the Spark k8s operator Task: {}", created); + + // Wait three seconds to get the status + try { + Thread.sleep(3000); + } catch (InterruptedException e) { + + } + + SparkApplicationList list = getSparkApplicationClient(client).list(); + + List sparkApplicationList = + list.getItems().stream() + .filter(crd -> crd.getMetadata().getName().equals(sparkConfig.getAppName())) + .collect(Collectors.toList()); + + if (CollectionUtils.isNotEmpty(sparkApplicationList)) { + SparkApplicationStatus status = sparkApplicationList.get(0).getStatus(); + if (Objects.nonNull(status)) { + logger.info( + "Spark k8s task: {},status: {}", + sparkConfig.getAppName(), + status.getApplicationState().getState()); + } + } + } + + public boolean initJobId() { + try { + getKubernetesOperatorState(); + } catch (Exception e) { + try { + // Prevent watch interruption due to network interruption.Restart Watcher. + Thread.sleep(5000); + getKubernetesOperatorState(); + } catch (InterruptedException interruptedException) { + logger.error("Use k8s watch obtain the status failed"); + } + } + // When the job is not finished, the appId is monitored; otherwise, the status is + // monitored(当任务没结束时，监控appId，反之，则监控状态，这里主要防止任务过早结束，导致一直等待) + return null != getApplicationId() || (jobState != null && jobState.isFinal()); + } + + private void getKubernetesOperatorState() { + getSparkApplicationClient(client) + .inNamespace(this.sparkConfig.getK8sNamespace()) + .withName(this.sparkConfig.getAppName()) + .watch( + new Watcher() { + @Override + public void eventReceived(Action action, SparkApplication sparkApplication) { + // todo get status + if (Objects.nonNull(sparkApplication.getStatus())) { + applicationId = sparkApplication.getStatus().getSparkApplicationId(); + jobState = + kubernetesOperatorStateConvertSparkState( + sparkApplication.getStatus().getApplicationState().getState()); + } + } + + @Override + public void onClose(WatcherException e) { + // Invoked when the watcher closes due to an Exception.Restart Watcher. + logger.error("Use k8s watch obtain the status failed", e); + getKubernetesOperatorState(); + } + }); + } + + public SparkAppHandle.State kubernetesOperatorStateConvertSparkState(String kubernetesState) { + if (StringUtils.isBlank(kubernetesState)) { + return SparkAppHandle.State.UNKNOWN; + } + switch (kubernetesState) { + case "PENDING": + return SparkAppHandle.State.CONNECTED; + case "RUNNING": + return SparkAppHandle.State.RUNNING; + case "COMPLETED": + return SparkAppHandle.State.FINISHED; + case "FAILED": + return SparkAppHandle.State.FAILED; + default: + return SparkAppHandle.State.UNKNOWN; + } + } + + public boolean isDisposed() { + return this.jobState.isFinal(); + } + + @Override + public String toString() { + return "ClusterDescriptorAdapter{" + "applicationId=" + getApplicationId() + '}'; + } + + @Override + public void close() { + logger.info("Start to close job {}.", getApplicationId()); + SparkApplication SparkApplication = + getSparkApplication(this.sparkConfig.getAppName(), this.sparkConfig.getK8sNamespace()); + getSparkApplicationClient(client).delete(SparkApplication); + client.close(); + } + + public static MixedOperation> + getSparkApplicationClient(KubernetesClient client) { + return client.customResources(SparkApplication.class, SparkApplicationList.class); + } + + public static SparkApplication getSparkApplication(String sparkOperatorName, String namespace) { + SparkApplication sparkApplication = new SparkApplication(); + ObjectMeta metadata = new ObjectMeta(); + metadata.setName(sparkOperatorName); + metadata.setNamespace(namespace); + sparkApplication.setMetadata(metadata); + return sparkApplication; + } +} diff --git a/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/client/deployment/crds/ApplicationState.java b/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/client/deployment/crds/ApplicationState.java new file mode 100644 index 0000000000..ac59f75be0 --- /dev/null +++ b/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/client/deployment/crds/ApplicationState.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineplugin.spark.client.deployment.crds; + +public class ApplicationState { + + private String state; + private String errorMessage; + + public String getState() { + return state; + } + + public void setState(String state) { + this.state = state; + } + + public String getErrorMessage() { + return errorMessage; + } + + public void setErrorMessage(String errorMessage) { + this.errorMessage = errorMessage; + } +} diff --git a/linkis-engineconn-plugins/elasticsearch/src/main/scala/org/apache/linkis/engineplugin/elasticsearch/exception/EsEngineException.scala b/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/client/deployment/crds/ConfigMap.java similarity index 77% rename from linkis-engineconn-plugins/elasticsearch/src/main/scala/org/apache/linkis/engineplugin/elasticsearch/exception/EsEngineException.scala rename to linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/client/deployment/crds/ConfigMap.java index 1632ae412d..4ea04d7c52 100644 --- a/linkis-engineconn-plugins/elasticsearch/src/main/scala/org/apache/linkis/engineplugin/elasticsearch/exception/EsEngineException.scala +++ b/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/client/deployment/crds/ConfigMap.java @@ -15,8 +15,17 @@ * limitations under the License. */ -package org.apache.linkis.engineplugin.elasticsearch.exception +package org.apache.linkis.engineplugin.spark.client.deployment.crds; -import org.apache.linkis.common.exception.ErrorException +public class ConfigMap { -case class EsEngineException(errorMsg: String) extends ErrorException(70114, errorMsg) + private String name; + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } +} diff --git a/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/client/deployment/crds/DriverInfo.java b/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/client/deployment/crds/DriverInfo.java new file mode 100644 index 0000000000..0484e23900 --- /dev/null +++ b/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/client/deployment/crds/DriverInfo.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineplugin.spark.client.deployment.crds; + +public class DriverInfo { + + private String podName; + private String webUIAddress; + private String webUIPort; + private String webUIServiceName; + + public String getWebUIServiceName() { + return webUIServiceName; + } + + public void setWebUIServiceName(String webUIServiceName) { + this.webUIServiceName = webUIServiceName; + } + + public String getWebUIPort() { + return webUIPort; + } + + public void setWebUIPort(String webUIPort) { + this.webUIPort = webUIPort; + } + + public String getWebUIAddress() { + return webUIAddress; + } + + public void setWebUIAddress(String webUIAddress) { + this.webUIAddress = webUIAddress; + } + + public String getPodName() { + return podName; + } + + public void setPodName(String podName) { + this.podName = podName; + } +} diff --git a/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/client/deployment/crds/HostPath.java b/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/client/deployment/crds/HostPath.java new file mode 100644 index 0000000000..053150352d --- /dev/null +++ b/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/client/deployment/crds/HostPath.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineplugin.spark.client.deployment.crds; + +public class HostPath { + + private String path; + + private String type; + + public String getPath() { + return path; + } + + public void setPath(String path) { + this.path = path; + } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } +} diff --git a/linkis-engineconn-plugins/openlookeng/src/main/java/org/apache/linkis/engineplugin/openlookeng/exception/OpenLooKengClientException.java b/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/client/deployment/crds/RestartPolicy.java similarity index 73% rename from linkis-engineconn-plugins/openlookeng/src/main/java/org/apache/linkis/engineplugin/openlookeng/exception/OpenLooKengClientException.java rename to linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/client/deployment/crds/RestartPolicy.java index 8c1c4a1541..276505af85 100644 --- a/linkis-engineconn-plugins/openlookeng/src/main/java/org/apache/linkis/engineplugin/openlookeng/exception/OpenLooKengClientException.java +++ b/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/client/deployment/crds/RestartPolicy.java @@ -15,13 +15,21 @@ * limitations under the License. */ -package org.apache.linkis.engineplugin.openlookeng.exception; +package org.apache.linkis.engineplugin.spark.client.deployment.crds; -import org.apache.linkis.common.exception.ErrorException; +public class RestartPolicy { -public class OpenLooKengClientException extends ErrorException { + private String type; - public OpenLooKengClientException(int errCode, String desc) { - super(errCode, desc); + public RestartPolicy(String type) { + this.type = type; + } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; } } diff --git a/linkis-engineconn-plugins/openlookeng/src/main/java/org/apache/linkis/engineplugin/openlookeng/builder/OpenLooKengProcessECLaunchBuilder.java b/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/client/deployment/crds/SparkApplication.java similarity index 52% rename from linkis-engineconn-plugins/openlookeng/src/main/java/org/apache/linkis/engineplugin/openlookeng/builder/OpenLooKengProcessECLaunchBuilder.java rename to linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/client/deployment/crds/SparkApplication.java index babeb1f6cc..6a0b143a75 100644 --- a/linkis-engineconn-plugins/openlookeng/src/main/java/org/apache/linkis/engineplugin/openlookeng/builder/OpenLooKengProcessECLaunchBuilder.java +++ b/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/client/deployment/crds/SparkApplication.java @@ -15,16 +15,21 @@ * limitations under the License. */ -package org.apache.linkis.engineplugin.openlookeng.builder; +package org.apache.linkis.engineplugin.spark.client.deployment.crds; -import org.apache.linkis.manager.engineplugin.common.launch.process.JavaProcessEngineConnLaunchBuilder; -import org.apache.linkis.manager.label.entity.engine.UserCreatorLabel; -import org.apache.linkis.storage.utils.StorageConfiguration; +import io.fabric8.kubernetes.api.model.Namespaced; +import io.fabric8.kubernetes.client.CustomResource; +import io.fabric8.kubernetes.model.annotation.Group; +import io.fabric8.kubernetes.model.annotation.Kind; +import io.fabric8.kubernetes.model.annotation.Version; -public class OpenLooKengProcessECLaunchBuilder extends JavaProcessEngineConnLaunchBuilder { +@Version(SparkApplication.VERSION) +@Group(SparkApplication.GROUP) +@Kind(SparkApplication.Kind) +public class SparkApplication extends CustomResource + implements Namespaced { + public static final String GROUP = "sparkoperator.k8s.io"; + public static final String VERSION = "v1beta2"; - @Override - public String getEngineStartUser(UserCreatorLabel label) { - return StorageConfiguration.HDFS_ROOT_USER().getValue(); - } + public static final String Kind = "SparkApplication"; } diff --git a/linkis-engineconn-plugins/elasticsearch/src/main/scala/org/apache/linkis/engineplugin/elasticsearch/executor/ElasticSearchExecutorOrder.scala b/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/client/deployment/crds/SparkApplicationList.java similarity index 79% rename from linkis-engineconn-plugins/elasticsearch/src/main/scala/org/apache/linkis/engineplugin/elasticsearch/executor/ElasticSearchExecutorOrder.scala rename to linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/client/deployment/crds/SparkApplicationList.java index ea49f9fe2d..4e70f6bc1d 100644 --- a/linkis-engineconn-plugins/elasticsearch/src/main/scala/org/apache/linkis/engineplugin/elasticsearch/executor/ElasticSearchExecutorOrder.scala +++ b/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/client/deployment/crds/SparkApplicationList.java @@ -15,11 +15,8 @@ * limitations under the License. */ -package org.apache.linkis.engineplugin.elasticsearch.executor +package org.apache.linkis.engineplugin.spark.client.deployment.crds; -object ElasticSearchExecutorOrder extends Enumeration { +import io.fabric8.kubernetes.client.CustomResourceList; - type ElasticSearchExecutorOrder = Value - val SQL = Value(1) - val JSON = Value(3) -} +public class SparkApplicationList extends CustomResourceList {} diff --git a/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/client/deployment/crds/SparkApplicationSpec.java b/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/client/deployment/crds/SparkApplicationSpec.java new file mode 100644 index 0000000000..be705ce400 --- /dev/null +++ b/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/client/deployment/crds/SparkApplicationSpec.java @@ -0,0 +1,294 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineplugin.spark.client.deployment.crds; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import io.fabric8.kubernetes.api.model.KubernetesResource; + +public class SparkApplicationSpec implements KubernetesResource { + + private String type; + + private String mode; + + private String image; + + private String imagePullPolicy; + + private String mainClass; + + private String mainApplicationFile; + + private String sparkVersion; + + private RestartPolicy restartPolicy; + + private List volumes; + + private SparkPodSpec driver; + + private SparkPodSpec executor; + + private Map sparkConf; + + public Map getSparkConf() { + return sparkConf; + } + + public void setSparkConf(Map sparkConf) { + this.sparkConf = sparkConf; + } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public String getMode() { + return mode; + } + + public void setMode(String mode) { + this.mode = mode; + } + + public String getImage() { + return image; + } + + public void setImage(String image) { + this.image = image; + } + + public String getImagePullPolicy() { + return imagePullPolicy; + } + + public void setImagePullPolicy(String imagePullPolicy) { + this.imagePullPolicy = imagePullPolicy; + } + + public String getMainClass() { + return mainClass; + } + + public void setMainClass(String mainClass) { + this.mainClass = mainClass; + } + + public String getMainApplicationFile() { + return mainApplicationFile; + } + + public void setMainApplicationFile(String mainApplicationFile) { + this.mainApplicationFile = mainApplicationFile; + } + + public String getSparkVersion() { + return sparkVersion; + } + + public void setSparkVersion(String sparkVersion) { + this.sparkVersion = sparkVersion; + } + + public RestartPolicy getRestartPolicy() { + return restartPolicy; + } + + public void setRestartPolicy(RestartPolicy restartPolicy) { + this.restartPolicy = restartPolicy; + } + + public List getVolumes() { + return volumes; + } + + public void setVolumes(List volumes) { + this.volumes = volumes; + } + + public SparkPodSpec getDriver() { + return driver; + } + + public void setDriver(SparkPodSpec driver) { + this.driver = driver; + } + + public SparkPodSpec getExecutor() { + return executor; + } + + public void setExecutor(SparkPodSpec executor) { + this.executor = executor; + } + + @Override + public String toString() { + return "SparkApplicationSpec{" + + "type='" + + type + + '\'' + + ", mode='" + + mode + + '\'' + + ", image='" + + image + + '\'' + + ", imagePullPolicy='" + + imagePullPolicy + + '\'' + + ", mainClass='" + + mainClass + + '\'' + + ", mainApplicationFile='" + + mainApplicationFile + + '\'' + + ", sparkVersion='" + + sparkVersion + + '\'' + + ", restartPolicy=" + + restartPolicy + + ", volumes=" + + volumes + + ", driver=" + + driver + + ", executor=" + + executor + + ", sparkConf=" + + sparkConf + + '}'; + } + + public static SparkApplicationSpecBuilder Builder() { + return new SparkApplicationSpecBuilder(); + } + + public static class SparkApplicationSpecBuilder { + private String type; + private String mode; + private String image; + private String imagePullPolicy; + private String mainClass; + private String mainApplicationFile; + private String sparkVersion; + private RestartPolicy restartPolicy; + private List volumes; + private SparkPodSpec driver; + private SparkPodSpec executor; + + private Map sparkConf; + + private SparkApplicationSpecBuilder() {} + + public SparkApplicationSpecBuilder type(String type) { + this.type = type; + return this; + } + + public SparkApplicationSpecBuilder mode(String mode) { + this.mode = mode; + return this; + } + + public SparkApplicationSpecBuilder image(String image) { + this.image = image; + return this; + } + + public SparkApplicationSpecBuilder imagePullPolicy(String imagePullPolicy) { + this.imagePullPolicy = imagePullPolicy; + return this; + } + + public SparkApplicationSpecBuilder mainClass(String mainClass) { + this.mainClass = mainClass; + return this; + } + + public SparkApplicationSpecBuilder mainApplicationFile(String mainApplicationFile) { + this.mainApplicationFile = mainApplicationFile; + return this; + } + + public SparkApplicationSpecBuilder sparkVersion(String sparkVersion) { + this.sparkVersion = sparkVersion; + return this; + } + + public SparkApplicationSpecBuilder restartPolicy(RestartPolicy restartPolicy) { + this.restartPolicy = restartPolicy; + return this; + } + + public SparkApplicationSpecBuilder volumes(List volumes) { + this.volumes = volumes; + return this; + } + + public SparkApplicationSpecBuilder driver(SparkPodSpec driver) { + this.driver = driver; + return this; + } + + public SparkApplicationSpecBuilder executor(SparkPodSpec executor) { + this.executor = executor; + return this; + } + + public SparkApplicationSpecBuilder sparkConf(Map sparkConf) { + if (sparkConf == null || sparkConf.size() == 0) { + return this; + } + + if (this.sparkConf == null) { + this.sparkConf = new HashMap<>(); + } + + for (Map.Entry entry : sparkConf.entrySet()) { + this.sparkConf.put(entry.getKey(), entry.getValue()); + } + + return this; + } + + public SparkApplicationSpec build() { + SparkApplicationSpec sparkApplicationSpec = new SparkApplicationSpec(); + sparkApplicationSpec.type = this.type; + sparkApplicationSpec.mainClass = this.mainClass; + sparkApplicationSpec.imagePullPolicy = this.imagePullPolicy; + sparkApplicationSpec.volumes = this.volumes; + sparkApplicationSpec.driver = this.driver; + sparkApplicationSpec.sparkVersion = this.sparkVersion; + sparkApplicationSpec.mode = this.mode; + sparkApplicationSpec.mainApplicationFile = this.mainApplicationFile; + sparkApplicationSpec.executor = this.executor; + sparkApplicationSpec.image = this.image; + sparkApplicationSpec.restartPolicy = this.restartPolicy; + sparkApplicationSpec.sparkConf = this.sparkConf; + return sparkApplicationSpec; + } + } +} diff --git a/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/client/deployment/crds/SparkApplicationStatus.java b/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/client/deployment/crds/SparkApplicationStatus.java new file mode 100644 index 0000000000..c7668ea338 --- /dev/null +++ b/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/client/deployment/crds/SparkApplicationStatus.java @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineplugin.spark.client.deployment.crds; + +import java.util.Map; + +public class SparkApplicationStatus { + + private String sparkApplicationId; + private String terminationTime; + private String lastSubmissionAttemptTime; + private String submissionID; + private ApplicationState applicationState; + private Integer executionAttempts; + private Integer submissionAttempts; + private DriverInfo driverInfo; + private Map executorState; + + public String getSubmissionID() { + return submissionID; + } + + public void setSubmissionID(String submissionID) { + this.submissionID = submissionID; + } + + public Integer getSubmissionAttempts() { + return submissionAttempts; + } + + public void setSubmissionAttempts(Integer submissionAttempts) { + this.submissionAttempts = submissionAttempts; + } + + public String getLastSubmissionAttemptTime() { + return lastSubmissionAttemptTime; + } + + public void setLastSubmissionAttemptTime(String lastSubmissionAttemptTime) { + this.lastSubmissionAttemptTime = lastSubmissionAttemptTime; + } + + public String getSparkApplicationId() { + return sparkApplicationId; + } + + public void setSparkApplicationId(String sparkApplicationId) { + this.sparkApplicationId = sparkApplicationId; + } + + public String getTerminationTime() { + return terminationTime; + } + + public void setTerminationTime(String terminationTime) { + this.terminationTime = terminationTime; + } + + public ApplicationState getApplicationState() { + return applicationState; + } + + public void setApplicationState(ApplicationState applicationState) { + this.applicationState = applicationState; + } + + public Integer getExecutionAttempts() { + return executionAttempts; + } + + public void setExecutionAttempts(Integer executionAttempts) { + this.executionAttempts = executionAttempts; + } + + public DriverInfo getDriverInfo() { + return driverInfo; + } + + public void setDriverInfo(DriverInfo driverInfo) { + this.driverInfo = driverInfo; + } + + public Map getExecutorState() { + return executorState; + } + + public void setExecutorState(Map executorState) { + this.executorState = executorState; + } +} diff --git a/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/client/deployment/crds/SparkPodSpec.java b/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/client/deployment/crds/SparkPodSpec.java new file mode 100644 index 0000000000..6c7b281a92 --- /dev/null +++ b/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/client/deployment/crds/SparkPodSpec.java @@ -0,0 +1,238 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineplugin.spark.client.deployment.crds; + +import java.util.List; +import java.util.Map; + +public class SparkPodSpec { + + private Integer cores; + + private String coreLimit; + + private String memory; + + private Map labels; + + private Map annotations; + + private String serviceAccount; + + private List volumeMounts; + + private Integer instances; + + public Integer getCores() { + return cores; + } + + public void setCores(Integer cores) { + this.cores = cores; + } + + public String getCoreLimit() { + return coreLimit; + } + + public void setCoreLimit(String coreLimit) { + this.coreLimit = coreLimit; + } + + public String getMemory() { + return memory; + } + + public void setMemory(String memory) { + this.memory = memory; + } + + public Map getLabels() { + return labels; + } + + public void setLabels(Map labels) { + this.labels = labels; + } + + public Map getAnnotations() { + return annotations; + } + + public void setAnnotations(Map annotations) { + this.annotations = annotations; + } + + public String getServiceAccount() { + return serviceAccount; + } + + public void setServiceAccount(String serviceAccount) { + this.serviceAccount = serviceAccount; + } + + public List getVolumeMounts() { + return volumeMounts; + } + + public void setVolumeMounts(List volumeMounts) { + this.volumeMounts = volumeMounts; + } + + public Integer getInstances() { + return instances; + } + + public void setInstances(Integer instances) { + this.instances = instances; + } + + public static SparkPodSpecBuilder Builder() { + return new SparkPodSpecBuilder(); + } + + public static class SparkPodSpecBuilder { + private Integer cores; + private String coreLimit; + private String memory; + private Map labels; + private Map annotations; + private String serviceAccount; + private List volumeMounts; + private Integer instances; + + public Integer getCores() { + return cores; + } + + public void setCores(Integer cores) { + this.cores = cores; + } + + public String getCoreLimit() { + return coreLimit; + } + + public void setCoreLimit(String coreLimit) { + this.coreLimit = coreLimit; + } + + public String getMemory() { + return memory; + } + + public void setMemory(String memory) { + this.memory = memory; + } + + public Map getLabels() { + return labels; + } + + public void setLabels(Map labels) { + this.labels = labels; + } + + public Map getAnnotations() { + return annotations; + } + + public void setAnnotations(Map annotations) { + this.annotations = annotations; + } + + public String getServiceAccount() { + return serviceAccount; + } + + public void setServiceAccount(String serviceAccount) { + this.serviceAccount = serviceAccount; + } + + public List getVolumeMounts() { + return volumeMounts; + } + + public void setVolumeMounts(List volumeMounts) { + this.volumeMounts = volumeMounts; + } + + public Integer getInstances() { + return instances; + } + + public void setInstances(Integer instances) { + this.instances = instances; + } + + private SparkPodSpecBuilder() {} + + public SparkPodSpecBuilder cores(Integer cores) { + this.cores = cores; + return this; + } + + public SparkPodSpecBuilder coreLimit(String coreLimit) { + this.coreLimit = coreLimit; + return this; + } + + public SparkPodSpecBuilder memory(String memory) { + this.memory = memory; + return this; + } + + public SparkPodSpecBuilder labels(Map labels) { + this.labels = labels; + return this; + } + + public SparkPodSpecBuilder annotations(Map annotations) { + this.annotations = annotations; + return this; + } + + public SparkPodSpecBuilder serviceAccount(String serviceAccount) { + this.serviceAccount = serviceAccount; + return this; + } + + public SparkPodSpecBuilder volumeMounts(List volumeMounts) { + this.volumeMounts = volumeMounts; + return this; + } + + public SparkPodSpecBuilder instances(Integer instances) { + this.instances = instances; + return this; + } + + public SparkPodSpec build() { + SparkPodSpec sparkPodSpec = new SparkPodSpec(); + sparkPodSpec.annotations = this.annotations; + sparkPodSpec.coreLimit = this.coreLimit; + sparkPodSpec.instances = this.instances; + sparkPodSpec.labels = this.labels; + sparkPodSpec.serviceAccount = this.serviceAccount; + sparkPodSpec.cores = this.cores; + sparkPodSpec.memory = this.memory; + sparkPodSpec.volumeMounts = this.volumeMounts; + return sparkPodSpec; + } + } +} diff --git a/linkis-engineconn-plugins/elasticsearch/src/main/scala/org/apache/linkis/engineplugin/elasticsearch/builder/ElasticSearchProcessEngineConnLaunchBuilder.scala b/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/client/deployment/crds/Volume.java similarity index 58% rename from linkis-engineconn-plugins/elasticsearch/src/main/scala/org/apache/linkis/engineplugin/elasticsearch/builder/ElasticSearchProcessEngineConnLaunchBuilder.scala rename to linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/client/deployment/crds/Volume.java index a35b9107be..4410fe80f9 100644 --- a/linkis-engineconn-plugins/elasticsearch/src/main/scala/org/apache/linkis/engineplugin/elasticsearch/builder/ElasticSearchProcessEngineConnLaunchBuilder.scala +++ b/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/client/deployment/crds/Volume.java @@ -15,16 +15,37 @@ * limitations under the License. */ -package org.apache.linkis.engineplugin.elasticsearch.builder +package org.apache.linkis.engineplugin.spark.client.deployment.crds; -import org.apache.linkis.manager.engineplugin.common.launch.process.JavaProcessEngineConnLaunchBuilder -import org.apache.linkis.manager.label.entity.engine.UserCreatorLabel -import org.apache.linkis.storage.utils.StorageConfiguration +public class Volume { -class ElasticSearchProcessEngineConnLaunchBuilder extends JavaProcessEngineConnLaunchBuilder { + private String name; - override def getEngineStartUser(label: UserCreatorLabel): String = { - StorageConfiguration.HDFS_ROOT_USER.getValue + private HostPath hostPath; + + private ConfigMap configMap; + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public HostPath getHostPath() { + return hostPath; } + public void setHostPath(HostPath hostPath) { + this.hostPath = hostPath; + } + + public ConfigMap getConfigMap() { + return configMap; + } + + public void setConfigMap(ConfigMap configMap) { + this.configMap = configMap; + } } diff --git a/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/client/deployment/crds/VolumeMount.java b/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/client/deployment/crds/VolumeMount.java new file mode 100644 index 0000000000..a4b2fb70cb --- /dev/null +++ b/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/client/deployment/crds/VolumeMount.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineplugin.spark.client.deployment.crds; + +public class VolumeMount { + + private String name; + + private String mountPath; + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getMountPath() { + return mountPath; + } + + public void setMountPath(String mountPath) { + this.mountPath = mountPath; + } +} diff --git a/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/client/deployment/util/FileUtils.java b/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/client/deployment/util/FileUtils.java new file mode 100644 index 0000000000..2b311b91bd --- /dev/null +++ b/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/client/deployment/util/FileUtils.java @@ -0,0 +1,201 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineplugin.spark.client.deployment.util; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; +import java.nio.channels.Channels; +import java.nio.channels.SeekableByteChannel; +import java.nio.channels.WritableByteChannel; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.Arrays; +import java.util.Random; +import java.util.stream.Stream; + +/** + * This is a utility class to deal files and directories. Contains utilities for recursive deletion + * and creation of temporary files. + */ +public final class FileUtils { + + /** Global lock to prevent concurrent directory deletes under Windows and MacOS. */ + private static final Object DELETE_LOCK = new Object(); + + /** The alphabet to construct the random part of the filename from. */ + private static final char[] ALPHABET = { + '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '0', 'a', 'b', 'c', 'd', 'e', 'f' + }; + + /** The length of the random part of the filename. */ + private static final int RANDOM_FILE_NAME_LENGTH = 12; + + /** + * The maximum size of array to allocate for reading. See {@code MAX_BUFFER_SIZE} in {@link Files} + * for more. + */ + private static final int MAX_BUFFER_SIZE = Integer.MAX_VALUE - 8; + + /** The size of the buffer used for reading. */ + private static final int BUFFER_SIZE = 4096; + + private static final String JAR_FILE_EXTENSION = "jar"; + + public static final String CLASS_FILE_EXTENSION = "class"; + + public static final String PACKAGE_SEPARATOR = "."; + + // ------------------------------------------------------------------------ + + public static void writeCompletely(WritableByteChannel channel, ByteBuffer src) + throws IOException { + while (src.hasRemaining()) { + channel.write(src); + } + } + + // ------------------------------------------------------------------------ + + /** Lists the given directory in a resource-leak-safe way. */ + public static Path[] listDirectory(Path directory) throws IOException { + try (Stream stream = Files.list(directory)) { + return stream.toArray(Path[]::new); + } + } + + // ------------------------------------------------------------------------ + + /** + * Constructs a random filename with the given prefix and a random part generated from hex + * characters. + * + * @param prefix the prefix to the filename to be constructed + * @return the generated random filename with the given prefix + */ + public static String getRandomFilename(final String prefix) { + final Random rnd = new Random(); + final StringBuilder stringBuilder = new StringBuilder(prefix); + + for (int i = 0; i < RANDOM_FILE_NAME_LENGTH; i++) { + stringBuilder.append(ALPHABET[rnd.nextInt(ALPHABET.length)]); + } + + return stringBuilder.toString(); + } + + // ------------------------------------------------------------------------ + // Simple reading and writing of files + // ------------------------------------------------------------------------ + + public static String readFile(File file, String charsetName) throws IOException { + byte[] bytes = readAllBytes(file.toPath()); + return new String(bytes, charsetName); + } + + public static String readFileUtf8(File file) throws IOException { + return readFile(file, "UTF-8"); + } + + public static void writeFile(File file, String contents, String encoding) throws IOException { + byte[] bytes = contents.getBytes(encoding); + Files.write(file.toPath(), bytes, StandardOpenOption.WRITE); + } + + public static void writeFileUtf8(File file, String contents) throws IOException { + writeFile(file, contents, "UTF-8"); + } + + /** + * Reads all the bytes from a file. The method ensures that the file is closed when all bytes have + * been read or an I/O error, or other runtime exception, is thrown. + * + *

This is an implementation that follow {@link Files#readAllBytes(Path)}, and the difference + * is that it limits the size of the direct buffer to avoid direct-buffer OutOfMemoryError. When + * {@link Files#readAllBytes(Path)} or other interfaces in java API can do this in the future, we + * should remove it. + * + * @param path the path to the file + * @return a byte array containing the bytes read from the file + * @throws IOException if an I/O error occurs reading from the stream + * @throws OutOfMemoryError if an array of the required size cannot be allocated, for example the + * file is larger that {@code 2GB} + */ + public static byte[] readAllBytes(Path path) throws IOException { + try (SeekableByteChannel channel = Files.newByteChannel(path); + InputStream in = Channels.newInputStream(channel)) { + + long size = channel.size(); + if (size > (long) MAX_BUFFER_SIZE) { + throw new OutOfMemoryError("Required array size too large"); + } + + return read(in, (int) size); + } + } + + /** + * Reads all the bytes from an input stream. Uses {@code initialSize} as a hint about how many + * bytes the stream will have and uses {@code directBufferSize} to limit the size of the direct + * buffer used to read. + * + * @param source the input stream to read from + * @param initialSize the initial size of the byte array to allocate + * @return a byte array containing the bytes read from the file + * @throws IOException if an I/O error occurs reading from the stream + * @throws OutOfMemoryError if an array of the required size cannot be allocated + */ + private static byte[] read(InputStream source, int initialSize) throws IOException { + int capacity = initialSize; + byte[] buf = new byte[capacity]; + int nread = 0; + int n; + + for (; ; ) { + // read to EOF which may read more or less than initialSize (eg: file + // is truncated while we are reading) + while ((n = source.read(buf, nread, Math.min(capacity - nread, BUFFER_SIZE))) > 0) { + nread += n; + } + + // if last call to source.read() returned -1, we are done + // otherwise, try to read one more byte; if that failed we're done too + if (n < 0 || (n = source.read()) < 0) { + break; + } + + // one more byte was read; need to allocate a larger buffer + if (capacity <= MAX_BUFFER_SIZE - capacity) { + capacity = Math.max(capacity << 1, BUFFER_SIZE); + } else { + if (capacity == MAX_BUFFER_SIZE) { + throw new OutOfMemoryError("Required array size too large"); + } + capacity = MAX_BUFFER_SIZE; + } + buf = Arrays.copyOf(buf, capacity); + buf[nread++] = (byte) n; + } + return (capacity == nread) ? buf : Arrays.copyOf(buf, nread); + } + + /** Private default constructor to avoid instantiation. */ + private FileUtils() {} +} diff --git a/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/client/deployment/util/KubernetesHelper.java b/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/client/deployment/util/KubernetesHelper.java new file mode 100644 index 0000000000..709308d084 --- /dev/null +++ b/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/client/deployment/util/KubernetesHelper.java @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineplugin.spark.client.deployment.util; + +import org.apache.commons.lang3.StringUtils; + +import java.io.File; +import java.io.IOException; + +import io.fabric8.kubernetes.client.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class KubernetesHelper { + private static final Logger logger = LoggerFactory.getLogger(KubernetesHelper.class); + + public static KubernetesClient getKubernetesClientByUrl( + String k8sMasterUrl, String k8sUsername, String k8sPassword) { + Config config = + new ConfigBuilder() + .withMasterUrl(k8sMasterUrl) + .withUsername(k8sUsername) + .withPassword(k8sPassword) + .build(); + DefaultKubernetesClient kubernetesClient = new DefaultKubernetesClient(config); + logger.info( + "KubernetesClient Create success,kubernetesClient masterUrl: {}", + kubernetesClient.getMasterUrl().toString()); + return kubernetesClient; + } + + public static KubernetesClient getKubernetesClientByUrl(String k8sMasterUrl) { + Config config = new ConfigBuilder().withMasterUrl(k8sMasterUrl).build(); + DefaultKubernetesClient kubernetesClient = new DefaultKubernetesClient(config); + logger.info( + "KubernetesClient Create success,kubernetesClient masterUrl: {}", + kubernetesClient.getMasterUrl().toString()); + return kubernetesClient; + } + + public static KubernetesClient getKubernetesClient( + String kubeConfigFile, String k8sMasterUrl, String k8sUsername, String k8sPassword) { + logger.info( + "Start create KubernetesClient,kubeConfigFile: {},k8sMasterUrl: {}", + kubeConfigFile, + k8sMasterUrl); + + if (StringUtils.isBlank(kubeConfigFile) && StringUtils.isBlank(kubeConfigFile)) { + throw new KubernetesClientException( + "Both kubeConfigFile and k8sMasterUrl are empty. Initializing KubernetesClient failed."); + } + // The ConfigFile mode is preferred + if (StringUtils.isNotBlank(kubeConfigFile)) { + return getKubernetesClientByKubeConfigFile(kubeConfigFile); + } + + if (StringUtils.isNotBlank(k8sMasterUrl) + && StringUtils.isNotBlank(k8sUsername) + && StringUtils.isNotBlank(k8sPassword)) { + return getKubernetesClientByUrl(k8sMasterUrl, k8sUsername, k8sPassword); + } + + if (StringUtils.isNotBlank(k8sMasterUrl)) { + return getKubernetesClientByUrl(k8sMasterUrl); + } + + throw new KubernetesClientException("Initializing KubernetesClient failed."); + } + + public static KubernetesClient getKubernetesClientByKubeConfigFile(String kubeConfigFile) { + final Config config; + + if (kubeConfigFile != null) { + try { + config = + Config.fromKubeconfig(null, FileUtils.readFileUtf8(new File(kubeConfigFile)), null); + } catch (IOException e) { + throw new KubernetesClientException("Load kubernetes config failed.", e); + } + } else { + config = Config.autoConfigure(null); + } + + DefaultKubernetesClient kubernetesClient = new DefaultKubernetesClient(config); + logger.info( + "KubernetesClient Create success,kubernetesClient masterUrl: {}", + kubernetesClient.getMasterUrl().toString()); + return kubernetesClient; + } +} diff --git a/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/datacalc/exception/ElasticsearchSinkException.java b/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/datacalc/exception/ElasticsearchSinkException.java new file mode 100644 index 0000000000..0d77d6ab7a --- /dev/null +++ b/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/datacalc/exception/ElasticsearchSinkException.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineplugin.spark.datacalc.exception; + +import org.apache.linkis.common.exception.ExceptionLevel; +import org.apache.linkis.common.exception.LinkisRuntimeException; + +public class ElasticsearchSinkException extends LinkisRuntimeException { + + public ElasticsearchSinkException(int errCode, String desc) { + super(errCode, desc); + } + + @Override + public ExceptionLevel getLevel() { + return ExceptionLevel.ERROR; + } +} diff --git a/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/datacalc/service/LinkisDataSourceContext.java b/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/datacalc/service/LinkisDataSourceContext.java index 3fa4237486..974c0bbf07 100644 --- a/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/datacalc/service/LinkisDataSourceContext.java +++ b/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/datacalc/service/LinkisDataSourceContext.java @@ -39,8 +39,12 @@ public class LinkisDataSourceContext { // https://docs.pingcap.com/tidb/dev/dev-guide-connect-to-tidb dsStrategyMap.put("tidb", new TiDBStrategy()); dsStrategyMap.put("doris", new DorisStrategy()); + dsStrategyMap.put("starrocks", new DorisStrategy()); + dsStrategyMap.put("dm", new DmStrategy()); + dsStrategyMap.put("kingbase", new KingbaseStrategy()); // https://jdbc.postgresql.org/documentation/use/ dsStrategyMap.put("postgresql", new PostgreSqlStrategy()); + dsStrategyMap.put("gaussdb", new PostgreSqlStrategy()); // https://github.com/ClickHouse/clickhouse-jdbc/tree/master/clickhouse-jdbc dsStrategyMap.put("clickhouse", new ClickHouseStrategy()); // https://docs.oracle.com/en/database/oracle/oracle-database/21/jajdb/oracle/jdbc/OracleDriver.html diff --git a/linkis-engineconn-plugins/openlookeng/src/main/java/org/apache/linkis/engineplugin/openlookeng/exception/OpenLooKengStateInvalidException.java b/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/datacalc/service/strategy/DmStrategy.java similarity index 70% rename from linkis-engineconn-plugins/openlookeng/src/main/java/org/apache/linkis/engineplugin/openlookeng/exception/OpenLooKengStateInvalidException.java rename to linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/datacalc/service/strategy/DmStrategy.java index b2b2cbc6f0..1f7905fb23 100644 --- a/linkis-engineconn-plugins/openlookeng/src/main/java/org/apache/linkis/engineplugin/openlookeng/exception/OpenLooKengStateInvalidException.java +++ b/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/datacalc/service/strategy/DmStrategy.java @@ -15,13 +15,22 @@ * limitations under the License. */ -package org.apache.linkis.engineplugin.openlookeng.exception; +package org.apache.linkis.engineplugin.spark.datacalc.service.strategy; -import org.apache.linkis.common.exception.ErrorException; +public class DmStrategy extends NormalStrategy { -public class OpenLooKengStateInvalidException extends ErrorException { + @Override + public String defaultDriver() { + return "dm.jdbc.driver.DmDriver"; + } + + @Override + public String defaultPort() { + return "5236"; + } - public OpenLooKengStateInvalidException(int errCode, String desc) { - super(errCode, desc); + @Override + public String getDatabaseType() { + return "dm"; } } diff --git a/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/datacalc/service/strategy/KingbaseStrategy.java b/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/datacalc/service/strategy/KingbaseStrategy.java new file mode 100644 index 0000000000..c72b37c72c --- /dev/null +++ b/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/datacalc/service/strategy/KingbaseStrategy.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineplugin.spark.datacalc.service.strategy; + +public class KingbaseStrategy extends NormalStrategy { + + @Override + public String defaultDriver() { + return "com.kingbase8.Driver"; + } + + @Override + public String defaultPort() { + return "54321"; + } + + @Override + public String getDatabaseType() { + return "kingbase8"; + } +} diff --git a/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/datacalc/sink/DataLakeSinkConfig.java b/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/datacalc/sink/DataLakeSinkConfig.java new file mode 100644 index 0000000000..03f8088494 --- /dev/null +++ b/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/datacalc/sink/DataLakeSinkConfig.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineplugin.spark.datacalc.sink; + +import org.apache.linkis.engineplugin.spark.datacalc.model.SinkConfig; + +import javax.validation.constraints.NotBlank; +import javax.validation.constraints.Pattern; + +public class DataLakeSinkConfig extends SinkConfig { + + @NotBlank + @Pattern( + regexp = "^(((file|hdfs)://)|/).*", + message = + "Invalid path URI, please set the following allowed schemas: 'file://' or 'hdfs://'(default).") + private String path; + + @NotBlank + @Pattern( + regexp = "^(delta|hudi)$", + message = "Unknown table format: {saveMode}. Accepted save modes are 'delta', 'hudi'.") + private String tableFormat = "delta"; + + @NotBlank + @Pattern( + regexp = "^(overwrite|append|ignore|error|errorifexists)$", + message = + "Unknown save mode: {saveMode}. Accepted save modes are 'overwrite', 'append', 'ignore', 'error', 'errorifexists'.") + private String saveMode = "overwrite"; + + public String getPath() { + if (path.startsWith("/")) return "hdfs://" + path; + return path; + } + + public void setPath(String path) { + this.path = path; + } + + public String getSaveMode() { + return saveMode; + } + + public void setSaveMode(String saveMode) { + this.saveMode = saveMode; + } + + public String getTableFormat() { + return tableFormat; + } + + public void setTableFormat(String tableFormat) { + this.tableFormat = tableFormat; + } +} diff --git a/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/datacalc/sink/DorisSinkConfig.java b/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/datacalc/sink/DorisSinkConfig.java new file mode 100644 index 0000000000..3c8227faed --- /dev/null +++ b/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/datacalc/sink/DorisSinkConfig.java @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineplugin.spark.datacalc.sink; + +import org.apache.linkis.engineplugin.spark.datacalc.model.SinkConfig; + +import javax.validation.constraints.NotBlank; +import javax.validation.constraints.Pattern; + +public class DorisSinkConfig extends SinkConfig { + + @NotBlank private String url; + + @NotBlank private String user; + + private String password; + + @NotBlank private String targetDatabase; + + @NotBlank private String targetTable; + + @NotBlank + @Pattern( + regexp = "^(overwrite|append|ignore|error|errorifexists)$", + message = + "Unknown save mode: {saveMode}. Accepted save modes are 'overwrite', 'append', 'ignore', 'error', 'errorifexists'.") + private String saveMode = "overwrite"; + + public String getUrl() { + return url; + } + + public void setUrl(String url) { + this.url = url; + } + + public String getUser() { + return user; + } + + public void setUser(String user) { + this.user = user; + } + + public String getPassword() { + return password; + } + + public void setPassword(String password) { + this.password = password; + } + + public String getTargetDatabase() { + return targetDatabase; + } + + public void setTargetDatabase(String targetDatabase) { + this.targetDatabase = targetDatabase; + } + + public String getTargetTable() { + return targetTable; + } + + public void setTargetTable(String targetTable) { + this.targetTable = targetTable; + } + + public String getSaveMode() { + return saveMode; + } + + public void setSaveMode(String saveMode) { + this.saveMode = saveMode; + } +} diff --git a/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/datacalc/sink/ElasticsearchSinkConfig.java b/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/datacalc/sink/ElasticsearchSinkConfig.java new file mode 100644 index 0000000000..9f4f26e641 --- /dev/null +++ b/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/datacalc/sink/ElasticsearchSinkConfig.java @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineplugin.spark.datacalc.sink; + +import org.apache.linkis.engineplugin.spark.datacalc.model.SinkConfig; + +import javax.validation.constraints.NotBlank; +import javax.validation.constraints.Pattern; + +public class ElasticsearchSinkConfig extends SinkConfig { + + @NotBlank private String node; + + @NotBlank private String port; + + @NotBlank private String index; + + private String type = "_doc"; + + private String user = ""; + + private String password = ""; + + private String primaryKey = ""; + + @NotBlank + @Pattern( + regexp = "^(overwrite|append|upsert|ignore|error|errorifexists)$", + message = + "Unknown save mode: {saveMode}. Accepted save modes are 'overwrite', 'append','upsert', 'ignore', 'error', 'errorifexists'.") + private String saveMode = "overwrite"; + + public String getPrimaryKey() { + return primaryKey; + } + + public void setPrimaryKey(String primaryKey) { + this.primaryKey = primaryKey; + } + + public String getUser() { + return user; + } + + public void setUser(String user) { + this.user = user; + } + + public String getPassword() { + return password; + } + + public void setPassword(String password) { + this.password = password; + } + + public String getSaveMode() { + return saveMode; + } + + public void setSaveMode(String saveMode) { + this.saveMode = saveMode; + } + + public String getNode() { + return node; + } + + public void setNode(String node) { + this.node = node; + } + + public String getPort() { + return port; + } + + public void setPort(String port) { + this.port = port; + } + + public String getIndex() { + return index; + } + + public void setIndex(String index) { + this.index = index; + } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } +} diff --git a/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/datacalc/sink/KafkaSinkConfig.java b/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/datacalc/sink/KafkaSinkConfig.java new file mode 100644 index 0000000000..ed3e94a27e --- /dev/null +++ b/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/datacalc/sink/KafkaSinkConfig.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineplugin.spark.datacalc.sink; + +import org.apache.linkis.engineplugin.spark.datacalc.model.SinkConfig; + +import javax.validation.constraints.NotBlank; +import javax.validation.constraints.Pattern; + +public class KafkaSinkConfig extends SinkConfig { + + @NotBlank private String servers; + + @NotBlank private String topic; + + private String checkpointLocation = "./ck"; + + @NotBlank + @Pattern( + regexp = "^(batch|stream)$", + message = "Unknown mode: {saveMode}. Accepted modes are 'batch', 'stream'.") + private String mode = "stream"; + + public String getCheckpointLocation() { + return checkpointLocation; + } + + public void setCheckpointLocation(String checkpointLocation) { + this.checkpointLocation = checkpointLocation; + } + + public String getMode() { + return mode; + } + + public void setMode(String mode) { + this.mode = mode; + } + + public String getServers() { + return servers; + } + + public void setServers(String servers) { + this.servers = servers; + } + + public String getTopic() { + return topic; + } + + public void setTopic(String topic) { + this.topic = topic; + } +} diff --git a/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/datacalc/sink/MongoSinkConfig.java b/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/datacalc/sink/MongoSinkConfig.java new file mode 100644 index 0000000000..072bad67c9 --- /dev/null +++ b/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/datacalc/sink/MongoSinkConfig.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineplugin.spark.datacalc.sink; + +import org.apache.linkis.engineplugin.spark.datacalc.model.SinkConfig; + +import javax.validation.constraints.NotBlank; +import javax.validation.constraints.Pattern; + +public class MongoSinkConfig extends SinkConfig { + + @NotBlank private String uri; + + @NotBlank private String database; + + @NotBlank private String collection; + + @NotBlank + @Pattern( + regexp = "^(overwrite|append|ignore|error|errorifexists)$", + message = + "Unknown save mode: {saveMode}. Accepted save modes are 'overwrite', 'append', 'ignore', 'error', 'errorifexists'.") + private String saveMode = "overwrite"; + + public String getSaveMode() { + return saveMode; + } + + public void setSaveMode(String saveMode) { + this.saveMode = saveMode; + } + + public String getUri() { + return uri; + } + + public void setUri(String uri) { + this.uri = uri; + } + + public String getDatabase() { + return database; + } + + public void setDatabase(String database) { + this.database = database; + } + + public String getCollection() { + return collection; + } + + public void setCollection(String collection) { + this.collection = collection; + } +} diff --git a/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/datacalc/sink/RedisSinkConfig.java b/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/datacalc/sink/RedisSinkConfig.java new file mode 100644 index 0000000000..cb87b9080e --- /dev/null +++ b/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/datacalc/sink/RedisSinkConfig.java @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineplugin.spark.datacalc.sink; + +import org.apache.linkis.engineplugin.spark.datacalc.model.SinkConfig; + +import javax.validation.constraints.NotBlank; +import javax.validation.constraints.Pattern; + +public class RedisSinkConfig extends SinkConfig { + + @NotBlank private String host; + + @NotBlank private String port; + + private String dbNum = "0"; + private String auth = "password"; + + @NotBlank private String targetTable; + + @NotBlank + @Pattern( + regexp = "^(overwrite|append|ignore|error|errorifexists)$", + message = + "Unknown save mode: {saveMode}. Accepted save modes are 'overwrite', 'append', 'ignore', 'error', 'errorifexists'.") + private String saveMode = "overwrite"; + + public String getSaveMode() { + return saveMode; + } + + public void setSaveMode(String saveMode) { + this.saveMode = saveMode; + } + + public String getHost() { + return host; + } + + public void setHost(String host) { + this.host = host; + } + + public String getPort() { + return port; + } + + public void setPort(String port) { + this.port = port; + } + + public String getDbNum() { + return dbNum; + } + + public void setDbNum(String dbNum) { + this.dbNum = dbNum; + } + + public String getTargetTable() { + return targetTable; + } + + public void setTargetTable(String targetTable) { + this.targetTable = targetTable; + } + + public String getAuth() { + return auth; + } + + public void setAuth(String auth) { + this.auth = auth; + } +} diff --git a/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/datacalc/sink/RocketmqSinkConfig.java b/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/datacalc/sink/RocketmqSinkConfig.java new file mode 100644 index 0000000000..344af7ab95 --- /dev/null +++ b/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/datacalc/sink/RocketmqSinkConfig.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineplugin.spark.datacalc.sink; + +import org.apache.linkis.engineplugin.spark.datacalc.model.SinkConfig; + +import javax.validation.constraints.NotBlank; +import javax.validation.constraints.Pattern; + +public class RocketmqSinkConfig extends SinkConfig { + + @NotBlank private String nameServer; + + @NotBlank private String topic; + + private String checkpointLocation = "./ck"; + + @NotBlank + @Pattern( + regexp = "^(batch|stream)$", + message = "Unknown mode: {saveMode}. Accepted modes are 'batch', 'stream'.") + private String mode = "stream"; + + public String getNameServer() { + return nameServer; + } + + public String getCheckpointLocation() { + return checkpointLocation; + } + + public void setCheckpointLocation(String checkpointLocation) { + this.checkpointLocation = checkpointLocation; + } + + public String getMode() { + return mode; + } + + public void setMode(String mode) { + this.mode = mode; + } + + public String getTopic() { + return topic; + } + + public void setTopic(String topic) { + this.topic = topic; + } +} diff --git a/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/datacalc/sink/SolrSinkConfig.java b/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/datacalc/sink/SolrSinkConfig.java new file mode 100644 index 0000000000..7b7a6daf66 --- /dev/null +++ b/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/datacalc/sink/SolrSinkConfig.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineplugin.spark.datacalc.sink; + +import org.apache.linkis.engineplugin.spark.datacalc.model.SinkConfig; + +import javax.validation.constraints.NotBlank; +import javax.validation.constraints.Pattern; + +public class SolrSinkConfig extends SinkConfig { + + @NotBlank private String zkhost; + + @NotBlank private String collection; + + private String softCommitSecs = "5"; + + private String batchSize = "500"; + + @NotBlank + @Pattern( + regexp = "^(overwrite|append|ignore|error|errorifexists)$", + message = + "Unknown save mode: {saveMode}. Accepted save modes are 'overwrite', 'append', 'ignore', 'error', 'errorifexists'.") + private String saveMode = "overwrite"; + + public String getSoftCommitSecs() { + return softCommitSecs; + } + + public void setSoftCommitSecs(String softCommitSecs) { + this.softCommitSecs = softCommitSecs; + } + + public String getBatchSize() { + return batchSize; + } + + public void setBatchSize(String batchSize) { + this.batchSize = batchSize; + } + + public String getSaveMode() { + return saveMode; + } + + public void setSaveMode(String saveMode) { + this.saveMode = saveMode; + } + + public String getZkhost() { + return zkhost; + } + + public void setZkhost(String zkhost) { + this.zkhost = zkhost; + } + + public String getCollection() { + return collection; + } + + public void setCollection(String collection) { + this.collection = collection; + } +} diff --git a/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/datacalc/sink/StarrocksSinkConfig.java b/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/datacalc/sink/StarrocksSinkConfig.java new file mode 100644 index 0000000000..f95b108be5 --- /dev/null +++ b/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/datacalc/sink/StarrocksSinkConfig.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineplugin.spark.datacalc.sink; + +import org.apache.linkis.engineplugin.spark.datacalc.model.SinkConfig; + +import javax.validation.constraints.NotBlank; + +public class StarrocksSinkConfig extends SinkConfig { + + @NotBlank private String url; + + @NotBlank private String jdbcUrl; + + @NotBlank private String user; + + private String password; + + @NotBlank private String targetDatabase; + + @NotBlank private String targetTable; + + public String getUrl() { + return url; + } + + public void setUrl(String url) { + this.url = url; + } + + public String getJdbcUrl() { + return jdbcUrl; + } + + public void setJdbcUrl(String jdbcUrl) { + this.jdbcUrl = jdbcUrl; + } + + public String getUser() { + return user; + } + + public void setUser(String user) { + this.user = user; + } + + public String getPassword() { + return password; + } + + public void setPassword(String password) { + this.password = password; + } + + public String getTargetDatabase() { + return targetDatabase; + } + + public void setTargetDatabase(String targetDatabase) { + this.targetDatabase = targetDatabase; + } + + public String getTargetTable() { + return targetTable; + } + + public void setTargetTable(String targetTable) { + this.targetTable = targetTable; + } +} diff --git a/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/datacalc/source/DataLakeSourceConfig.java b/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/datacalc/source/DataLakeSourceConfig.java new file mode 100644 index 0000000000..b8fed0a7a1 --- /dev/null +++ b/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/datacalc/source/DataLakeSourceConfig.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineplugin.spark.datacalc.source; + +import org.apache.linkis.engineplugin.spark.datacalc.model.SourceConfig; + +import javax.validation.constraints.NotBlank; +import javax.validation.constraints.Pattern; + +public class DataLakeSourceConfig extends SourceConfig { + + @NotBlank + @Pattern( + regexp = "^(((file|hdfs)://)|/).*", + message = + "Invalid path URI, please set the following allowed schemas: 'file://' or 'hdfs://'(default).") + private String path; + + @NotBlank + @Pattern( + regexp = "^(delta|hudi)$", + message = "Unknown table format: {saveMode}. Accepted save modes are 'delta', 'hudi'.") + private String tableFormat = "delta"; + + public String getPath() { + if (path.startsWith("/")) return "hdfs://" + path; + return path; + } + + public void setPath(String path) { + this.path = path; + } + + public String getTableFormat() { + return tableFormat; + } + + public void setTableFormat(String tableFormat) { + this.tableFormat = tableFormat; + } +} diff --git a/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/datacalc/source/DorisSourceConfig.java b/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/datacalc/source/DorisSourceConfig.java new file mode 100644 index 0000000000..95a11d89df --- /dev/null +++ b/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/datacalc/source/DorisSourceConfig.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineplugin.spark.datacalc.source; + +import org.apache.linkis.engineplugin.spark.datacalc.model.SourceConfig; + +import javax.validation.constraints.NotBlank; + +public class DorisSourceConfig extends SourceConfig { + + @NotBlank private String url; + + @NotBlank private String user; + private String password; + + @NotBlank private String sourceDatabase; + + @NotBlank private String sourceTable; + + public String getUrl() { + return url; + } + + public void setUrl(String url) { + this.url = url; + } + + public String getUser() { + return user; + } + + public void setUser(String user) { + this.user = user; + } + + public String getPassword() { + return password; + } + + public void setPassword(String password) { + this.password = password; + } + + public String getSourceDatabase() { + return sourceDatabase; + } + + public void setSourceDatabase(String sourceDatabase) { + this.sourceDatabase = sourceDatabase; + } + + public String getSourceTable() { + return sourceTable; + } + + public void setSourceTable(String sourceTable) { + this.sourceTable = sourceTable; + } +} diff --git a/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/datacalc/source/ElasticsearchSourceConfig.java b/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/datacalc/source/ElasticsearchSourceConfig.java new file mode 100644 index 0000000000..feed085ac3 --- /dev/null +++ b/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/datacalc/source/ElasticsearchSourceConfig.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineplugin.spark.datacalc.source; + +import org.apache.linkis.engineplugin.spark.datacalc.model.SourceConfig; + +import javax.validation.constraints.NotBlank; + +public class ElasticsearchSourceConfig extends SourceConfig { + + @NotBlank private String node; + + @NotBlank private String port; + + @NotBlank private String index; + + private String user = ""; + + private String password = ""; + + private String query = "{\"query\":{\"match_all\":{}}}"; + + private String type = "_doc"; + + public String getUser() { + return user; + } + + public void setUser(String user) { + this.user = user; + } + + public String getPassword() { + return password; + } + + public void setPassword(String password) { + this.password = password; + } + + public String getNode() { + return node; + } + + public void setNode(String node) { + this.node = node; + } + + public String getPort() { + return port; + } + + public void setPort(String port) { + this.port = port; + } + + public String getIndex() { + return index; + } + + public void setIndex(String index) { + this.index = index; + } + + public String getQuery() { + return query; + } + + public void setQuery(String query) { + this.query = query; + } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } +} diff --git a/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/datacalc/source/KafkaSourceConfig.java b/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/datacalc/source/KafkaSourceConfig.java new file mode 100644 index 0000000000..489565e9ce --- /dev/null +++ b/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/datacalc/source/KafkaSourceConfig.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineplugin.spark.datacalc.source; + +import org.apache.linkis.engineplugin.spark.datacalc.model.SourceConfig; + +import javax.validation.constraints.NotBlank; + +public class KafkaSourceConfig extends SourceConfig { + + @NotBlank private String servers; + + @NotBlank private String topic; + + private String consumeMode = "earliest"; + + public String getServers() { + return servers; + } + + public void setServers(String servers) { + this.servers = servers; + } + + public String getTopic() { + return topic; + } + + public void setTopic(String topic) { + this.topic = topic; + } + + public String getConsumeMode() { + return consumeMode; + } + + public void setConsumeMode(String consumeMode) { + this.consumeMode = consumeMode; + } +} diff --git a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/Interpreter/Interpreter.scala b/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/datacalc/source/MongoSourceConfig.java similarity index 52% rename from linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/Interpreter/Interpreter.scala rename to linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/datacalc/source/MongoSourceConfig.java index f6d6c797ec..83deed369d 100644 --- a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/Interpreter/Interpreter.scala +++ b/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/datacalc/source/MongoSourceConfig.java @@ -15,29 +15,41 @@ * limitations under the License. */ -package org.apache.linkis.engineplugin.spark.Interpreter +package org.apache.linkis.engineplugin.spark.datacalc.source; -import org.apache.linkis.common.utils.Utils -import org.apache.linkis.engineplugin.spark.common.State -import org.apache.linkis.scheduler.executer.ExecuteResponse +import org.apache.linkis.engineplugin.spark.datacalc.model.SourceConfig; -import scala.concurrent.TimeoutException -import scala.concurrent.duration.Duration +import javax.validation.constraints.NotBlank; -/** - */ +public class MongoSourceConfig extends SourceConfig { + + @NotBlank private String uri; + + @NotBlank private String database; + + @NotBlank private String collection; + + public String getUri() { + return uri; + } -trait Interpreter { - def state: State + public void setUri(String uri) { + this.uri = uri; + } - def execute(code: String): ExecuteResponse + public String getDatabase() { + return database; + } - def close(): Unit + public void setDatabase(String database) { + this.database = database; + } - @throws(classOf[TimeoutException]) - @throws(classOf[InterruptedException]) - final def waitForStateChange(oldState: State, atMost: Duration): Unit = { - Utils.waitUntil({ () => state != oldState }, atMost) + public String getCollection() { + return collection; } + public void setCollection(String collection) { + this.collection = collection; + } } diff --git a/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/datacalc/source/RedisSourceConfig.java b/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/datacalc/source/RedisSourceConfig.java new file mode 100644 index 0000000000..d13628c820 --- /dev/null +++ b/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/datacalc/source/RedisSourceConfig.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineplugin.spark.datacalc.source; + +import org.apache.linkis.engineplugin.spark.datacalc.model.SourceConfig; + +import javax.validation.constraints.NotBlank; + +public class RedisSourceConfig extends SourceConfig { + + @NotBlank private String host; + + @NotBlank private String port; + + @NotBlank private String serializer = "table"; + + private String keysPattern; + + private String sourceTable; + + private String dbNum = "0"; + private String auth = "password"; + + public String getHost() { + return host; + } + + public void setHost(String host) { + this.host = host; + } + + public String getPort() { + return port; + } + + public void setPort(String port) { + this.port = port; + } + + public String getKeysPattern() { + return keysPattern; + } + + public void setKeysPattern(String keysPattern) { + this.keysPattern = keysPattern; + } + + public String getDbNum() { + return dbNum; + } + + public void setDbNum(String dbNum) { + this.dbNum = dbNum; + } + + public String getAuth() { + return auth; + } + + public void setAuth(String auth) { + this.auth = auth; + } + + public String getSourceTable() { + return sourceTable; + } + + public void setSourceTable(String sourceTable) { + this.sourceTable = sourceTable; + } + + public String getSerializer() { + return serializer; + } + + public void setSerializer(String serializer) { + this.serializer = serializer; + } +} diff --git a/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/datacalc/source/RocketmqSourceConfig.java b/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/datacalc/source/RocketmqSourceConfig.java new file mode 100644 index 0000000000..83d79ec1ad --- /dev/null +++ b/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/datacalc/source/RocketmqSourceConfig.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineplugin.spark.datacalc.source; + +import org.apache.linkis.engineplugin.spark.datacalc.model.SourceConfig; + +import javax.validation.constraints.NotBlank; + +public class RocketmqSourceConfig extends SourceConfig { + + @NotBlank private String nameServer; + + @NotBlank private String topic; + + private String consumeMode = "earliest"; + + public String getNameServer() { + return nameServer; + } + + public void setNameServer(String nameServer) { + this.nameServer = nameServer; + } + + public String getTopic() { + return topic; + } + + public void setTopic(String topic) { + this.topic = topic; + } + + public String getConsumeMode() { + return consumeMode; + } + + public void setConsumeMode(String consumeMode) { + this.consumeMode = consumeMode; + } +} diff --git a/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/datacalc/source/StarrocksSourceConfig.java b/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/datacalc/source/StarrocksSourceConfig.java new file mode 100644 index 0000000000..89d23c4d98 --- /dev/null +++ b/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/datacalc/source/StarrocksSourceConfig.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineplugin.spark.datacalc.source; + +import org.apache.linkis.engineplugin.spark.datacalc.model.SourceConfig; + +import javax.validation.constraints.NotBlank; + +public class StarrocksSourceConfig extends SourceConfig { + + @NotBlank private String url; + @NotBlank private String user; + private String password; + + @NotBlank private String sourceDatabase; + + @NotBlank private String sourceTable; + + public String getUrl() { + return url; + } + + public void setUrl(String url) { + this.url = url; + } + + public String getSourceDatabase() { + return sourceDatabase; + } + + public void setSourceDatabase(String sourceDatabase) { + this.sourceDatabase = sourceDatabase; + } + + public String getSourceTable() { + return sourceTable; + } + + public void setSourceTable(String sourceTable) { + this.sourceTable = sourceTable; + } + + public String getUser() { + return user; + } + + public void setUser(String user) { + this.user = user; + } + + public String getPassword() { + return password; + } + + public void setPassword(String password) { + this.password = password; + } +} diff --git a/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/datacalc/util/PluginUtil.java b/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/datacalc/util/PluginUtil.java index 8cb7beb4f3..2d29c1b551 100644 --- a/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/datacalc/util/PluginUtil.java +++ b/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/datacalc/util/PluginUtil.java @@ -44,6 +44,15 @@ private static Map> getSourcePlugins() { classMap.put("managed_jdbc", ManagedJdbcSource.class); classMap.put("jdbc", JdbcSource.class); classMap.put("file", FileSource.class); + classMap.put("redis", RedisSource.class); + classMap.put("datalake", DataLakeSource.class); + classMap.put("rocketmq", RocketmqSource.class); + classMap.put("mongo", MongoSource.class); + classMap.put("elasticsearch", ElasticsearchSource.class); + classMap.put("solr", SolrSource.class); + classMap.put("kafka", KafkaSource.class); + classMap.put("starrocks", StarrocksSource.class); + classMap.put("doris", DorisSource.class); return classMap; } @@ -59,6 +68,15 @@ private static Map> getSinkPlugins() { classMap.put("jdbc", JdbcSink.class); classMap.put("hive", HiveSink.class); classMap.put("file", FileSink.class); + classMap.put("redis", RedisSink.class); + classMap.put("datalake", DataLakeSink.class); + classMap.put("rocketmq", RocketmqSink.class); + classMap.put("mongo", MongoSink.class); + classMap.put("elasticsearch", ElasticsearchSink.class); + classMap.put("solr", SolrSink.class); + classMap.put("kafka", KafkaSink.class); + classMap.put("starrocks", StarrocksSink.class); + classMap.put("doris", DorisSink.class); return classMap; } diff --git a/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/errorcode/SparkErrorCodeSummary.java b/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/errorcode/SparkErrorCodeSummary.java index 936e773e40..42f0b66e4d 100644 --- a/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/errorcode/SparkErrorCodeSummary.java +++ b/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/errorcode/SparkErrorCodeSummary.java @@ -66,6 +66,10 @@ public enum SparkErrorCodeSummary implements LinkisErrorCode { 43032, "The application start failed, since yarn applicationId is null."), NOT_SUPPORT_METHOD(43040, "Not support method for requestExpectedResource."), + + LINKIS_SPARK_YARN_CLUSTER_JARS_ERROR( + 43042, + "linkis.spark.yarn.cluster.jars parameters configuration errors(linkis.spark.yarn.cluster.jars 参数配置错误)."), ; /** (errorCode)错误码 */ diff --git a/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/executor/SecureRandomStringUtils.java b/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/executor/SecureRandomStringUtils.java new file mode 100644 index 0000000000..de5ac0b60f --- /dev/null +++ b/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/executor/SecureRandomStringUtils.java @@ -0,0 +1,221 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineplugin.spark.executor; + +import org.apache.commons.lang3.StringUtils; + +import java.security.NoSuchAlgorithmException; +import java.security.SecureRandom; + +/** + * Based on Commons Lang3 RandomStringUtils, but with a SecureRandom. + * + *

For internal Apache Linkis use only. + */ +final class SecureRandomStringUtils { + // Based on + // https://github.com/apache/commons-lang/blob/5e07d873e6b45714d29bf47634adffa3b5aef098/src/main/java/org/apache/commons/lang3/RandomStringUtils.java + + private static SecureRandom RANDOM_INSTANCE; + + private static SecureRandom random() { + if (RANDOM_INSTANCE != null) { + return RANDOM_INSTANCE; + } + try { + RANDOM_INSTANCE = SecureRandom.getInstanceStrong(); + return RANDOM_INSTANCE; + } catch (NoSuchAlgorithmException e) { + throw new IllegalStateException("Cannot create SecureRandom.getInstanceStrong()", e); + } + } + + // Random + /** + * Creates a random string whose length is the number of characters specified. + * + *

Characters will be chosen from the set of Latin alphabetic characters (a-z, A-Z) and the + * digits 0-9. + * + * @param count the length of random string to create + * @return the random string + * @throws IllegalArgumentException if {@code count} < 0. + */ + public static String randomAlphanumeric(final int count) { + return random(count, true, true); + } + + /** + * Creates a random string whose length is the number of characters specified. + * + *

Characters will be chosen from the set of alpha-numeric characters as indicated by the + * arguments. + * + * @param count the length of random string to create + * @param letters if {@code true}, generated string may include alphabetic characters + * @param numbers if {@code true}, generated string may include numeric characters + * @return the random string + * @throws IllegalArgumentException if {@code count} < 0. + */ + private static String random(final int count, final boolean letters, final boolean numbers) { + return random(count, 0, 0, letters, numbers); + } + + /** + * Creates a random string whose length is the number of characters specified. + * + *

Characters will be chosen from the set of alpha-numeric characters as indicated by the + * arguments. + * + * @param count the length of random string to create + * @param start the position in set of chars to start at + * @param end the position in set of chars to end before + * @param letters if {@code true}, generated string may include alphabetic characters + * @param numbers if {@code true}, generated string may include numeric characters + * @return the random string + * @throws IllegalArgumentException if {@code count} < 0. + */ + private static String random( + final int count, + final int start, + final int end, + final boolean letters, + final boolean numbers) { + return random(count, start, end, letters, numbers, null, random()); + } + + /** + * Creates a random string based on a variety of options, using supplied source of randomness. + * + *

If start and end are both {@code 0}, start and end are set to {@code ' '} and {@code 'z'}, + * the ASCII printable characters, will be used, unless letters and numbers are both {@code + * false}, in which case, start and end are set to {@code 0} and {@link Character#MAX_CODE_POINT}. + * + *

If set is not {@code null}, characters between start and end are chosen. + * + *

This method accepts a user-supplied {@link SecureRandom} instance to use as a source of + * randomness. By seeding a single {@link SecureRandom} instance with a fixed seed and using it + * for each call, the same random sequence of strings can be generated repeatedly and predictably. + * + * @param count the length of random string to create + * @param start the position in set of chars to start at (inclusive) + * @param end the position in set of chars to end before (exclusive) + * @param letters if {@code true}, generated string may include alphabetic characters + * @param numbers if {@code true}, generated string may include numeric characters + * @param chars the set of chars to choose randoms from, must not be empty. If {@code null}, then + * it will use the set of all chars. + * @param random a source of randomness. + * @return the random string + * @throws ArrayIndexOutOfBoundsException if there are not {@code (end - start) + 1} characters in + * the set array. + * @throws IllegalArgumentException if {@code count} < 0 or the provided chars array is empty. + */ + private static String random( + int count, + int start, + int end, + final boolean letters, + final boolean numbers, + final char[] chars, + final SecureRandom random) { + if (count == 0) { + return StringUtils.EMPTY; + } + if (count < 0) { + throw new IllegalArgumentException( + "Requested random string length " + count + " is less than 0."); + } + if (chars != null && chars.length == 0) { + throw new IllegalArgumentException("The chars array must not be empty"); + } + + if (start == 0 && end == 0) { + if (chars != null) { + end = chars.length; + } else if (!letters && !numbers) { + end = Character.MAX_CODE_POINT; + } else { + end = 'z' + 1; + start = ' '; + } + } else if (end <= start) { + throw new IllegalArgumentException( + "Parameter end (" + end + ") must be greater than start (" + start + ")"); + } + + final int zeroDigitAscii = 48; + final int firstLetterAscii = 65; + + if (chars == null && (numbers && end <= zeroDigitAscii || letters && end <= firstLetterAscii)) { + throw new IllegalArgumentException( + "Parameter end (" + + end + + ") must be greater then (" + + zeroDigitAscii + + ") for generating digits " + + "or greater then (" + + firstLetterAscii + + ") for generating letters."); + } + + final StringBuilder builder = new StringBuilder(count); + final int gap = end - start; + + while (count-- != 0) { + final int codePoint; + if (chars == null) { + codePoint = random.nextInt(gap) + start; + + switch (Character.getType(codePoint)) { + case Character.UNASSIGNED: + case Character.PRIVATE_USE: + case Character.SURROGATE: + count++; + continue; + } + + } else { + codePoint = chars[random.nextInt(gap) + start]; + } + + final int numberOfChars = Character.charCount(codePoint); + if (count == 0 && numberOfChars > 1) { + count++; + continue; + } + + if (letters && Character.isLetter(codePoint) + || numbers && Character.isDigit(codePoint) + || !letters && !numbers) { + builder.appendCodePoint(codePoint); + + if (numberOfChars == 2) { + count--; + } + + } else { + count++; + } + } + return builder.toString(); + } + + private SecureRandomStringUtils() { + // empty + } +} diff --git a/linkis-engineconn-plugins/spark/src/main/resources/log4j2.xml b/linkis-engineconn-plugins/spark/src/main/resources/log4j2.xml index 11c4574bab..298cf8cf70 100644 --- a/linkis-engineconn-plugins/spark/src/main/resources/log4j2.xml +++ b/linkis-engineconn-plugins/spark/src/main/resources/log4j2.xml @@ -15,7 +15,7 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - + - - - diff --git a/linkis-engineconn-plugins/spark/src/main/resources/python/mix_pyspark.py b/linkis-engineconn-plugins/spark/src/main/resources/python/mix_pyspark.py index b3675c15c8..b8844bb11b 100644 --- a/linkis-engineconn-plugins/spark/src/main/resources/python/mix_pyspark.py +++ b/linkis-engineconn-plugins/spark/src/main/resources/python/mix_pyspark.py @@ -126,6 +126,7 @@ def handler_stop_signals(sig, frame): java_import(gateway.jvm, "org.apache.spark.api.java.*") java_import(gateway.jvm, "org.apache.spark.api.python.*") java_import(gateway.jvm, "org.apache.spark.mllib.api.python.*") +java_import(gateway.jvm, "org.apache.spark.sql.api.python.*") intp = gateway.entry_point diff --git a/linkis-engineconn-plugins/spark/src/main/resources/spark-k8s-operator.md b/linkis-engineconn-plugins/spark/src/main/resources/spark-k8s-operator.md new file mode 100644 index 0000000000..9a0921f299 --- /dev/null +++ b/linkis-engineconn-plugins/spark/src/main/resources/spark-k8s-operator.md @@ -0,0 +1,109 @@ + +### 1. spark-on-k8s-operator document + +```text +https://github.com/GoogleCloudPlatform/spark-on-k8s-operator/blob/master/docs/quick-start-guide.md +``` + + +### 2. spark-on-k8s-operator install + +```text +helm repo add spark-operator https://googlecloudplatform.github.io/spark-on-k8s-operator + +helm install my-release spark-operator/spark-operator --namespace spark-operator --create-namespace --set webhook.enable=true +``` + +### 3. spark-on-k8s-operator test task submit + +```text +kubectl apply -f examples/spark-pi.yaml +``` + +### 4. If an error is reported: Message: Forbidden!Configured service account doesn't have access. Service account may have been revoked. pods "spark-pi-driver" is forbidden: error looking up service account spark/spark: serviceaccount "spark" not found. + +```text +kubectl create serviceaccount spark + +kubectl create clusterrolebinding spark-role --clusterrole=edit --serviceaccount=default:spark --namespace=default +``` + +### 5. spark-on-k8s-operator Uninstall (usually not required, uninstall after installation problems) + +```text +helm uninstall my-release --namespace spark-operator + +kubectl delete serviceaccounts my-release-spark-operator --namespace spark-operator + +kubectl delete clusterrole my-release-spark-operator --namespace spark-operator + +kubectl delete clusterrolebindings my-release-spark-operator --namespace spark-operator +``` + +### 6. Submitting tasks with Restful API +```text +POST /api/rest_j/v1/entrance/submit +``` + +```json +{ + "executionContent": { + "runType": "jar", + "code": "show databases" + }, + "params": { + "variable": { + }, + "configuration": { + "startup": { + "spark.executor.memory": "1g", + "spark.driver.memory": "1g", + "spark.executor.cores": "1", + "spark.app.main.class": "org.apache.spark.examples.SparkPi", + "spark.app.name": "spark-submit-jar-cjtest", + "spark.app.resource": "local:///opt/spark/examples/jars/spark-examples_2.12-3.2.1.jar", + "spark.executor.instances": 1, + "spark.master":"k8s-operator", + "linkis.spark.k8s.config.file":"~/.kube/config", + "linkis.spark.k8s.serviceAccount":"spark" + } + } + }, + "source": { + "scriptPath": "file:///tmp/hadoop/test.sql" + }, + "labels": { + "engineType": "spark-3.2.1", + "engineConnMode": "once", + "userCreator": "linkis-IDE" + } +} +``` + +### 7. Submitting tasks via Linkis-cli + +```text +sh ./bin/linkis-cli --mode once -labelMap engineConnMode=once -engineType spark-3.2.1 -codeType jar -submitUser hadoop -proxyUser hadoop -jobContentMap runType=jar -jobContentMap spark.app.main.class=org.apache.spark.examples.SparkPi -confMap spark.app.name=spark-submit-jar-test -confMap spark.app.resource=local:///opt/spark/examples/jars/spark-examples_2.12-3.2.1.jar -confMap spark.executor.instances=1 -confMap spark.kubernetes.file.upload.path=local:///opt/spark/tmp -confMap spark.executor.memory=1g -confMap spark.driver.memory=1g -confMap spark.executor.cores=1 -confMap spark.master=k8s-operator -confMap linkis.spark.k8s.config.file=/home/hadoop/.kube/config -confMap linkis.spark.k8s.serviceAccount=spark +``` + +### 8. Matters needing attention + +```text +You need to check whether hosts is configured,Such as: +k8s-master-ip lb.kubesphere.local +``` + +### 9. Reference document +```text +https://github.com/GoogleCloudPlatform/spark-on-k8s-operator + +https://github.com/fabric8io/kubernetes-client/ + +https://github.com/apple/batch-processing-gateway + +https://www.lightbend.com/blog/how-to-manage-monitor-spark-on-kubernetes-introduction-spark-submit-kubernetes-operator + +https://www.lightbend.com/blog/how-to-manage-monitor-spark-on-kubernetes-deep-dive-kubernetes-operator-for-spark +``` + + diff --git a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/Interpreter/ProcessInterpreter.scala b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/Interpreter/ProcessInterpreter.scala deleted file mode 100644 index 171d48e4f1..0000000000 --- a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/Interpreter/ProcessInterpreter.scala +++ /dev/null @@ -1,125 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.engineplugin.spark.Interpreter - -import org.apache.linkis.common.utils.{Logging, Utils} -import org.apache.linkis.engineplugin.spark.common._ -import org.apache.linkis.scheduler.executer.{ - ErrorExecuteResponse, - ExecuteResponse, - SuccessExecuteResponse -} - -import org.apache.commons.io.IOUtils - -import java.io.{BufferedReader, InputStreamReader, PrintWriter} -import java.util.concurrent.TimeUnit - -import scala.concurrent.{Await, ExecutionContext, Future} -import scala.concurrent.duration.Duration - -import org.json4s._ - -/** - */ -abstract class ProcessInterpreter(process: Process) extends Interpreter with Logging { - - implicit val executor: ExecutionContext = ExecutionContext.global - - protected[this] var _state: State = Starting() - - protected[this] val stdin = new PrintWriter(process.getOutputStream) - - protected[this] val stdout = - new BufferedReader(new InputStreamReader(process.getInputStream()), 1) - - protected[this] val errOut = new LineBufferedStream(process.getErrorStream()) - - override def state: State = _state - - override def execute(code: String): ExecuteResponse = { - if (code == "sc.cancelAllJobs" || code == "sc.cancelAllJobs()") { - sendExecuteRequest(code) - } - _state match { - case (Dead() | ShuttingDown() | Error() | Success()) => - throw new IllegalStateException("interpreter is not running") - case Idle() => - require(state == Idle()) - code match { - case "SHUTDOWN" => - sendShutdownRequest() - close() - ErrorExecuteResponse("shutdown", new Exception("shutdown")) - case _ => - _state = Busy() - sendExecuteRequest(code) match { - case Some(rep) => - _state = Idle() - // ExecuteComplete(rep) - SuccessExecuteResponse() - case None => - _state = Error() - val errorMsg = errOut.lines.mkString(", ") - throw new Exception(errorMsg) - } - } - case _ => - throw new IllegalStateException(s"interpreter is in ${_state} state, cannot do query.") - } - } - - Future { - val exitCode = process.waitFor() - if (exitCode != 0) { - // scalastyle:off println - errOut.lines.foreach(println) - println(getClass.getSimpleName + " has stopped with exit code " + process.exitValue) - _state = Error() - } else { - println(getClass.getSimpleName + " has finished.") - _state = Success() - } - } - - protected def waitUntilReady(): Unit - - protected def sendExecuteRequest(request: String): Option[JValue] - - protected def sendShutdownRequest(): Unit = {} - - override def close(): Unit = { - val future = Future { - _state match { - case (Dead() | ShuttingDown() | Success()) => - Future.successful() - case _ => - sendShutdownRequest() - } - } - _state = Dead() - IOUtils.closeQuietly(stdin) - IOUtils.closeQuietly(stdout) - errOut.close - // scalastyle:off awaitresult - Utils.tryFinally(Await.result(future, Duration(10, TimeUnit.SECONDS))) { - process.destroy() - } - } - -} diff --git a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/Interpreter/PythonInterpreter.scala b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/Interpreter/PythonInterpreter.scala index 4223db8ba7..df6fe55f7e 100644 --- a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/Interpreter/PythonInterpreter.scala +++ b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/Interpreter/PythonInterpreter.scala @@ -17,54 +17,27 @@ package org.apache.linkis.engineplugin.spark.Interpreter -import org.apache.linkis.common.conf.CommonVars import org.apache.linkis.common.io.FsPath import org.apache.linkis.common.utils.{ClassUtils, Logging, Utils} -import org.apache.linkis.engineplugin.spark.common.LineBufferedStream import org.apache.linkis.engineplugin.spark.config.SparkConfiguration import org.apache.linkis.storage.FSFactory import org.apache.commons.io.IOUtils -import org.apache.spark.{SparkContext, SparkException} +import org.apache.spark.SparkContext import org.apache.spark.sql.DataFrame import org.apache.spark.sql.catalyst.expressions.Attribute import java.io._ import java.nio.file.Files -import scala.collection.JavaConverters._ import scala.collection.mutable import scala.collection.mutable.ArrayBuffer -import org.json4s.{DefaultFormats, JValue} -import org.json4s.jackson.JsonMethods._ -import org.json4s.jackson.Serialization -import py4j.GatewayServer - /** */ object PythonInterpreter { - def create(): Interpreter = { - val pythonExec = CommonVars("PYSPARK_DRIVER_PYTHON", "python").getValue - - val gatewayServer = new GatewayServer(SQLSession, 0) - gatewayServer.start() - - val builder = new ProcessBuilder(Array(pythonExec, createFakeShell().toString).toList.asJava) - - val env = builder.environment() - env.put("PYTHONPATH", pythonPath) - env.put("PYTHONUNBUFFERED", "YES") - env.put("PYSPARK_GATEWAY_PORT", "" + gatewayServer.getListeningPort) - env.put("SPARK_HOME", SparkConfiguration.SPARK_HOME.getValue) - - val process = builder.start() - - new PythonInterpreter(process, gatewayServer) - } - def pythonPath: String = { val pythonPath = new ArrayBuffer[String] val pythonHomePath = new File(SparkConfiguration.SPARK_HOME.getValue, "python").getPath @@ -102,203 +75,4 @@ object PythonInterpreter { file } - private def createFakePySpark(): File = { - val source: InputStream = getClass.getClassLoader.getResourceAsStream("fake_pyspark.sh") - - val file = Files.createTempFile("", "").toFile - file.deleteOnExit() - - file.setExecutable(true) - - val sink = new FileOutputStream(file) - val buf = new Array[Byte](1024) - var n = source.read(buf) - - while (n > 0) { - sink.write(buf, 0, n) - n = source.read(buf) - } - - source.close() - sink.close() - - file - } - -} - -private class PythonInterpreter(process: Process, gatewayServer: GatewayServer) - extends ProcessInterpreter(process) - with Logging { - implicit val formats = DefaultFormats - - override def close(): Unit = { - try { - super.close() - } finally { - gatewayServer.shutdown() - } - } - - final override protected def waitUntilReady(): Unit = { - var running = false - val code = - try process.exitValue - catch { case t: IllegalThreadStateException => running = true; -1 } - if (!running) { - throw new SparkException( - s"Spark python application has already finished with exit code $code, now exit..." - ) - } - var continue = true - val initOut = new LineBufferedStream(process.getInputStream) - val iterable = initOut.iterator - while (continue && iterable.hasNext) { - iterable.next match { - // scalastyle:off println - case "READY" => println("Start python application succeed."); continue = false - case str: String => println(str) - case _ => - } - } - initOut.close - } - - override protected def sendExecuteRequest(code: String): Option[JValue] = { - val rep = sendRequest(Map("msg_type" -> "execute_request", "content" -> Map("code" -> code))) - rep.map { rep => - assert((rep \ "msg_type").extract[String] == "execute_reply") - - val content: JValue = rep \ "content" - - content - } - } - - override protected def sendShutdownRequest(): Unit = { - sendRequest(Map("msg_type" -> "shutdown_request", "content" -> ())).foreach { rep => - logger.warn(f"process failed to shut down while returning $rep") - } - } - - private def sendRequest(request: Map[String, Any]): Option[JValue] = { - // scalastyle:off println - stdin.println(Serialization.write(request)) - stdin.flush() - - Option(stdout.readLine()).map { line => parse(line) } - } - - def pythonPath: String = { - val pythonPath = new ArrayBuffer[String] - val pythonHomePath = new File(SparkConfiguration.SPARK_HOME.getValue, "python").getPath - val pythonParentPath = new File(pythonHomePath, "lib") - pythonPath += pythonHomePath - pythonParentPath - .listFiles(new FileFilter { - override def accept(pathname: File): Boolean = pathname.getName.endsWith(".zip") - }) - .foreach(f => pythonPath += f.getPath) - ClassUtils.jarOfClass(classOf[SparkContext]).foreach(pythonPath += _) - pythonPath.mkString(File.pathSeparator) - } - - def createFakeShell(): File = createFakeShell("python/fake_shell.py") - - def createFakeShell(script: String, fileType: String = ".py"): File = { - val source: InputStream = getClass.getClassLoader.getResourceAsStream(script) - - val file = Files.createTempFile("", fileType).toFile - file.deleteOnExit() - - val sink = new FileOutputStream(file) - val buf = new Array[Byte](1024) - var n = source.read(buf) - - while (n > 0) { - sink.write(buf, 0, n) - n = source.read(buf) - } - - source.close() - sink.close() - - file - } - -} - -object SQLSession extends Logging { - - def showDF(sc: SparkContext, jobGroup: String, df: Any, maxResult: Int = Int.MaxValue): String = { - val startTime = System.currentTimeMillis() - - val iterator = Utils.tryThrow(df.asInstanceOf[DataFrame].toLocalIterator)(t => { - sc.clearJobGroup() - t - }) - - var columns: List[Attribute] = null - // get field names - Utils.tryThrow({ - val qe = df.getClass.getMethod("queryExecution").invoke(df) - val a = qe.getClass.getMethod("analyzed").invoke(qe) - val seq = a.getClass.getMethod("output").invoke(a).asInstanceOf[Seq[Any]] - columns = seq.toList.asInstanceOf[List[Attribute]] - })(t => { - sc.clearJobGroup() - t - }) - var schema = new StringBuilder - schema ++= "%TABLE\n" - val nameSet = new mutable.HashSet[String]() - for (col <- columns) { - nameSet.add(col.name) - schema ++= col.name ++ "\t" - } - val trim = if (nameSet.size < columns.length) { - var schemaWithAlis = new StringBuilder - schemaWithAlis ++= "%TABLE\n" - for (col <- columns) { - val colName = col.qualifiedName - schemaWithAlis ++= colName ++ "\t" - } - logger.info("I AM IN LESS") - logger.info(schemaWithAlis.toString.trim) - schemaWithAlis.toString.trim - } else { - logger.info("I AM IN MORE") - logger.info(schema.toString.trim) - schema.toString.trim - } - val msg = FSFactory.getFs("").write(new FsPath(""), true) - msg.write(trim.getBytes("utf-8")) - - var index = 0 - Utils.tryThrow({ - while (index < maxResult && iterator.hasNext) { - msg.write("\n".getBytes("utf-8")) - val row = iterator.next() - columns.indices.foreach { i => - if (row.isNullAt(i)) msg.write("NULL".getBytes("utf-8")) - else msg.write(row.apply(i).asInstanceOf[Object].toString.getBytes("utf-8")) - if (i != columns.size - 1) { - msg.write("\t".getBytes("utf-8")) - } - } - index += 1 - } - })(t => { - sc.clearJobGroup() - t - }) - val colCount = if (columns != null) columns.size else 0 - logger.warn(s"Fetched $colCount col(s) : $index row(s).") - sc.clearJobGroup() - Utils.tryFinally({ - msg.flush(); - msg.toString - }) { () => IOUtils.closeQuietly(msg) } - } - } diff --git a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/config/SparkConfiguration.scala b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/config/SparkConfiguration.scala index 45a5640f69..a493c5ff37 100644 --- a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/config/SparkConfiguration.scala +++ b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/config/SparkConfiguration.scala @@ -30,6 +30,10 @@ object SparkConfiguration extends Logging { val SPARK_HOME_ENV = "SPARK_HOME" val SPARK_CONF_DIR_ENV = "SPARK_CONF_DIR" + val SPARK_YARN_CLIENT = "client" + + val SPARK_YARN_CLUSTER = "cluster" + val PROCESS_MAX_THREADS = CommonVars[Int]("wds.linkis.process.threadpool.max", 100) val SPARK_SESSION_HOOK = CommonVars[String]("wds.linkis.engine.spark.session.hook", "") @@ -46,12 +50,39 @@ object SparkConfiguration extends Logging { val SPARK_DEPLOY_MODE = CommonVars[String]("spark.submit.deployMode", "client") + val SPARK_YARN_CLUSTER_JARS = + CommonVars[String]("linkis.spark.yarn.cluster.jars", "hdfs:///spark/cluster") + val SPARK_APP_NAME = CommonVars[String]("spark.app.name", "Linkis-EngineConn-Spark") val SPARK_APP_RESOURCE = CommonVars[String]("spark.app.resource", "") val SPARK_APP_CONF = CommonVars[String]("spark.extconf", "") + val SPARK_K8S_CONFIG_FILE = CommonVars[String]("linkis.spark.k8s.config.file", "~/.kube/config") + val SPARK_K8S_SERVICE_ACCOUNT = CommonVars[String]("linkis.spark.k8s.serviceAccount", "") + val SPARK_K8S_MASTER_URL = CommonVars[String]("linkis.spark.k8s.master.url", "") + val SPARK_K8S_USERNAME = CommonVars[String]("linkis.spark.k8s.username", "") + val SPARK_K8S_PASSWORD = CommonVars[String]("linkis.spark.k8s.password", "") + val SPARK_K8S_IMAGE = CommonVars[String]("linkis.spark.k8s.image", "apache/spark:v3.2.1") + val SPARK_K8S_IMAGE_PULL_POLICY = CommonVars[String]("linkis.spark.k8s.imagePullPolicy", "Always") + val SPARK_K8S_LANGUAGE_TYPE = CommonVars[String]("linkis.spark.k8s.languageType", "Scala") + val SPARK_K8S_RESTART_POLICY = CommonVars[String]("linkis.spark.k8s.restartPolicy", "Never") + val SPARK_K8S_SPARK_VERSION = CommonVars[String]("linkis.spark.k8s.sparkVersion", "3.2.1") + val SPARK_K8S_NAMESPACE = CommonVars[String]("linkis.spark.k8s.namespace", "default") + val SPARK_K8S_UI_PORT = CommonVars[String]("linkis.spark.k8s.ui.port", "4040") + + val SPARK_K8S_EXECUTOR_REQUEST_CORES = + CommonVars[String]("linkis.spark.k8s.executor.request.cores", "1") + + val SPARK_K8S_DRIVER_REQUEST_CORES = + CommonVars[String]("linkis.spark.k8s.driver.request.cores", "1") + + val SPARK_KUBERNETES_FILE_UPLOAD_PATH = + CommonVars[String]("spark.kubernetes.file.upload.path", "local:///opt/spark/tmp") + val SPARK_PYTHON_VERSION = CommonVars[String]("spark.python.version", "python") + val SPARK_PYTHON_FILES = CommonVars[String]("spark.submit.pyFiles", "") + val SPARK_PYTHON_TEST_MODE_ENABLE = CommonVars[Boolean]("linkis.spark.python.test.mode.enable", false) @@ -83,14 +114,14 @@ object SparkConfiguration extends Logging { "Map output compression method（map输出结果压缩方式）" ) - val SPARK_MASTER = CommonVars[String]("spark.master", "yarn", "Default master（默认master）") + val SPARK_MASTER = CommonVars[String]("spark.master", "yarn", "Default yarn（默认yarn）") val SPARK_CONSOLE_OUTPUT_NUM = CommonVars[Int]("wds.linkis.spark.output.line.limit", 10) val LINKIS_SPARK_USEHIVECONTEXT = CommonVars[Boolean]("wds.linkis.spark.useHiveContext", true) val DEFAULT_SPARK_JAR_NAME = - CommonVars[String]("wds.linkis.ecp.spark.default.jar", "linkis-engineconn-core-1.3.1.jar") + CommonVars[String]("wds.linkis.ecp.spark.default.jar", "linkis-engineconn-core-1.3.2.jar") val ENGINE_JAR = CommonVars[String]("wds.linkis.enginemanager.core.jar", getMainJarName) @@ -149,6 +180,11 @@ object SparkConfiguration extends Logging { val SPARK_ONCE_YARN_RESTFUL_URL = CommonVars[String]("linkis.spark.once.yarn.restful.url", "") + val LINKIS_SPARK_ETL_SUPPORT_HUDI = CommonVars[Boolean]("linkis.spark.etl.support.hudi", false) + + val SCALA_PARSE_APPEND_CODE = + CommonVars("linkis.scala.parse.append.code", "val linkisVar=1").getValue + private def getMainJarName(): String = { val somePath = ClassUtils.jarOfClass(classOf[SparkEngineConnFactory]) if (somePath.isDefined) { diff --git a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/datacalc/DataCalcExecution.scala b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/datacalc/DataCalcExecution.scala index 447005f5c6..e1b17342e9 100644 --- a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/datacalc/DataCalcExecution.scala +++ b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/datacalc/DataCalcExecution.scala @@ -52,9 +52,13 @@ object DataCalcExecution { val sources = mapleData.getSources.map(source => PluginUtil.createSource[SR](source.getName, source.getConfig) ) - val transformations = mapleData.getTransformations.map(sink => - PluginUtil.createTransform[TR](sink.getName, sink.getConfig) - ) + val transformations = if (mapleData.getTransformations == null) { + Array.empty[DataCalcTransform[TR]] + } else { + mapleData.getTransformations.map(sink => + PluginUtil.createTransform[TR](sink.getName, sink.getConfig) + ) + } val sinks = mapleData.getSinks.map(sink => PluginUtil.createSink[SK](sink.getName, sink.getConfig)) diff --git a/linkis-engineconn-plugins/elasticsearch/src/main/scala/org/apache/linkis/engineplugin/elasticsearch/executor/client/ElasticSearchExecutor.scala b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/datacalc/sink/DataLakeSink.scala similarity index 56% rename from linkis-engineconn-plugins/elasticsearch/src/main/scala/org/apache/linkis/engineplugin/elasticsearch/executor/client/ElasticSearchExecutor.scala rename to linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/datacalc/sink/DataLakeSink.scala index 72e1953e33..a2deb99a63 100644 --- a/linkis-engineconn-plugins/elasticsearch/src/main/scala/org/apache/linkis/engineplugin/elasticsearch/executor/client/ElasticSearchExecutor.scala +++ b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/datacalc/sink/DataLakeSink.scala @@ -15,32 +15,23 @@ * limitations under the License. */ -package org.apache.linkis.engineplugin.elasticsearch.executor.client +package org.apache.linkis.engineplugin.spark.datacalc.sink import org.apache.linkis.common.utils.Logging -import org.apache.linkis.engineplugin.elasticsearch.executor.client.impl.ElasticSearchExecutorImpl -import org.apache.linkis.scheduler.executer.ExecuteResponse +import org.apache.linkis.engineplugin.spark.datacalc.api.DataCalcSink -import java.io.IOException -import java.util +import org.apache.spark.sql.{Dataset, Row, SparkSession} -import scala.collection.JavaConverters._ +class DataLakeSink extends DataCalcSink[DataLakeSinkConfig] with Logging { -trait ElasticSearchExecutor extends Logging { + def output(spark: SparkSession, ds: Dataset[Row]): Unit = { + logger.info(s"Save data to ${config.getTableFormat} tablePath: ${config.getPath}") + val writer = ds.write.format(config.getTableFormat).mode(config.getSaveMode) - @throws(classOf[IOException]) - def open: Unit - - def executeLine(code: String): ElasticSearchResponse - - def close: Unit - -} - -object ElasticSearchExecutor { - - def apply(runType: String, properties: util.Map[String, String]): ElasticSearchExecutor = { - new ElasticSearchExecutorImpl(runType, properties) + if (config.getOptions != null && !config.getOptions.isEmpty) { + writer.options(config.getOptions) + } + writer.save(config.getPath) } } diff --git a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/datacalc/sink/DorisSink.scala b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/datacalc/sink/DorisSink.scala new file mode 100644 index 0000000000..9d5301ced9 --- /dev/null +++ b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/datacalc/sink/DorisSink.scala @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineplugin.spark.datacalc.sink + +import org.apache.linkis.common.utils.Logging +import org.apache.linkis.engineplugin.spark.datacalc.api.DataCalcSink + +import org.apache.commons.lang3.StringUtils +import org.apache.spark.sql.{Dataset, Row, SparkSession} + +import scala.collection.JavaConverters._ + +class DorisSink extends DataCalcSink[DorisSinkConfig] with Logging { + + def output(spark: SparkSession, ds: Dataset[Row]): Unit = { + var options = Map( + "doris.fenodes" -> config.getUrl, + "user" -> config.getUser, + "password" -> config.getPassword, + "doris.table.identifier" -> String.format( + "%s.%s", + config.getTargetDatabase, + config.getTargetTable + ) + ) + + if (config.getOptions != null && !config.getOptions.isEmpty) { + options = config.getOptions.asScala.toMap ++ options + } + + val writer = ds.write.format("doris") + if (StringUtils.isNotBlank(config.getSaveMode)) { + writer.mode(config.getSaveMode) + } + + logger.info( + s"Save data from doris url: ${config.getUrl}, targetDatabase: ${config.getTargetDatabase}, targetTable: ${config.getTargetTable}" + ) + writer.options(options).save() + } + +} diff --git a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/datacalc/sink/ElasticsearchSink.scala b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/datacalc/sink/ElasticsearchSink.scala new file mode 100644 index 0000000000..2e448c3e1d --- /dev/null +++ b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/datacalc/sink/ElasticsearchSink.scala @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineplugin.spark.datacalc.sink + +import org.apache.linkis.common.utils.Logging +import org.apache.linkis.engineplugin.spark.datacalc.api.DataCalcSink +import org.apache.linkis.engineplugin.spark.datacalc.exception.ElasticsearchSinkException +import org.apache.linkis.engineplugin.spark.errorcode.SparkErrorCodeSummary + +import org.apache.commons.lang3.StringUtils +import org.apache.spark.sql.{Dataset, Row, SparkSession} + +import scala.collection.JavaConverters._ + +class ElasticsearchSink extends DataCalcSink[ElasticsearchSinkConfig] with Logging { + + def output(spark: SparkSession, ds: Dataset[Row]): Unit = { + var options = Map( + "es.index.auto.create" -> "true", + "es.nodes.wan.only" -> "true", + "es.nodes" -> config.getNode, + "es.port" -> config.getPort, + "es.net.http.auth.user" -> config.getUser, + "es.net.http.auth.pass" -> config.getPassword + ) + + if (config.getOptions != null && !config.getOptions.isEmpty) { + options = config.getOptions.asScala.toMap ++ options + } + + if (config.getSaveMode.equalsIgnoreCase("upsert")) { + if (StringUtils.isBlank(config.getPrimaryKey)) { + throw new ElasticsearchSinkException( + SparkErrorCodeSummary.DATA_CALC_VARIABLE_NOT_EXIST.getErrorCode, + "saveMode is upsert, please set elasticsearch mapping [primaryKey] in variables" + ) + } + options = + options ++ Map("es.write.operation" -> "upsert", "es.mapping.id" -> config.getPrimaryKey) + config.setSaveMode("append") + } + + val writer = ds.write.format("org.elasticsearch.spark.sql") + if (StringUtils.isNotBlank(config.getSaveMode)) { + writer.mode(config.getSaveMode) + } + + logger.info( + s"Load data to elasticsearch nodes: ${config.getNode}, port: ${config.getPort}, index: ${config.getIndex}" + ) + writer.options(options).save(s"${config.getIndex}/${config.getType}") + } + +} diff --git a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/datacalc/sink/FileSink.scala b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/datacalc/sink/FileSink.scala index 1464375e30..3519a9f7f9 100644 --- a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/datacalc/sink/FileSink.scala +++ b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/datacalc/sink/FileSink.scala @@ -48,6 +48,7 @@ class FileSink extends DataCalcSink[FileSinkConfig] with Logging { case "parquet" => writer.parquet(path) case "text" => writer.text(path) case "orc" => writer.orc(path) + case "excel" => writer.format("excel").save(path) case _ => writer.format(config.getSerializer).save(path) } } diff --git a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/datacalc/sink/HiveSink.scala b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/datacalc/sink/HiveSink.scala index 8ba618776b..bef4c39694 100644 --- a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/datacalc/sink/HiveSink.scala +++ b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/datacalc/sink/HiveSink.scala @@ -24,14 +24,11 @@ import org.apache.linkis.engineplugin.spark.errorcode.SparkErrorCodeSummary import org.apache.commons.lang3.StringUtils import org.apache.spark.sql._ -import org.apache.spark.sql.catalyst.catalog.HiveTableRelation import org.apache.spark.sql.execution.datasources.{HadoopFsRelation, LogicalRelation} import org.apache.spark.sql.functions.col import org.apache.spark.sql.sources.DataSourceRegister import org.apache.spark.sql.types.StructField -import org.slf4j.{Logger, LoggerFactory} - class HiveSink extends DataCalcSink[HiveSinkConfig] with Logging { def output(spark: SparkSession, ds: Dataset[Row]): Unit = { @@ -122,7 +119,9 @@ class HiveSink extends DataCalcSink[HiveSinkConfig] with Logging { logFields(sourceFields, targetFields) throw new HiveSinkException( SparkErrorCodeSummary.DATA_CALC_COLUMN_NUM_NOT_MATCH.getErrorCode, - s"$targetTable requires that the data to be inserted have the same number of columns as the target table: target table has ${targetFields.length} column(s) but the inserted data has ${sourceFields.length} column(s)" + s"$targetTable requires that the data to be inserted have the same number of columns " + + s"as the target table: target table has ${targetFields.length} column(s) " + + s"but the inserted data has ${sourceFields.length} column(s)" ) } @@ -184,17 +183,19 @@ class HiveSink extends DataCalcSink[HiveSinkConfig] with Logging { logicalRelation.relation match { case hadoopFsRelation: HadoopFsRelation => hadoopFsRelation.fileFormat match { - case _: org.apache.spark.sql.execution.datasources.orc.OrcFileFormat => - fileFormat = FileFormat.ORC case _: org.apache.spark.sql.execution.datasources.parquet.ParquetFileFormat => fileFormat = FileFormat.PARQUET case dataSourceRegister: DataSourceRegister => fileFormat = FileFormat.withName(dataSourceRegister.shortName.toUpperCase) case _ => + if (hadoopFsRelation.fileFormat.getClass.getSimpleName.equals("OrcFileFormat")) { + fileFormat = FileFormat.ORC + } } } - case hiveTableRelation: HiveTableRelation => - // todo + // case hiveTableRelation: HiveTableRelation => + // todo please note `HiveTableRelation` was added after spark 2.2.1 + case _ => } fileFormat } catch { diff --git a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/datacalc/sink/JdbcSink.scala b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/datacalc/sink/JdbcSink.scala index ab8a21c3f7..e9d60bd2b3 100644 --- a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/datacalc/sink/JdbcSink.scala +++ b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/datacalc/sink/JdbcSink.scala @@ -17,14 +17,16 @@ package org.apache.linkis.engineplugin.spark.datacalc.sink +import org.apache.linkis.common.utils.ClassUtils.getFieldVal import org.apache.linkis.common.utils.Logging import org.apache.linkis.engineplugin.spark.datacalc.api.DataCalcSink import org.apache.commons.lang3.StringUtils +import org.apache.spark.SPARK_VERSION import org.apache.spark.sql.{Dataset, Row, SparkSession} -import org.apache.spark.sql.execution.datasources.jdbc.{JDBCOptions, JdbcUtils} +import org.apache.spark.sql.execution.datasources.jdbc.JDBCOptions -import java.sql.Connection +import java.sql.{Connection, DriverManager} import scala.collection.JavaConverters._ @@ -58,7 +60,8 @@ class JdbcSink extends DataCalcSink[JdbcSinkConfig] with Logging { .repartition(1) .foreachPartition((_: Iterator[Row]) => { val jdbcOptions = new JDBCOptions(options) - val conn: Connection = JdbcUtils.createConnectionFactory(jdbcOptions)() + val conn: Connection = + DriverManager.getConnection(config.getUrl, config.getUser, config.getPassword) try { config.getPreQueries.asScala.foreach(query => { logger.info(s"Execute pre query: $query") @@ -86,7 +89,12 @@ class JdbcSink extends DataCalcSink[JdbcSinkConfig] with Logging { logger.info("Execute query: {}", query) val statement = conn.prepareStatement(query) try { - statement.setQueryTimeout(jdbcOptions.queryTimeout) + // `queryTimeout` was added after spark2.4.0, more details please check SPARK-23856 + if (SPARK_VERSION >= "2.4") { + val queryTimeout = getFieldVal(jdbcOptions, "queryTimeout").asInstanceOf[Int] + statement.setQueryTimeout(queryTimeout) + } + val rows = statement.executeUpdate() logger.info("{} rows affected", rows) } catch { diff --git a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/datacalc/sink/KafkaSink.scala b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/datacalc/sink/KafkaSink.scala new file mode 100644 index 0000000000..7c495e1d5a --- /dev/null +++ b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/datacalc/sink/KafkaSink.scala @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineplugin.spark.datacalc.sink + +import org.apache.linkis.common.utils.Logging +import org.apache.linkis.engineplugin.spark.datacalc.api.DataCalcSink + +import org.apache.spark.sql.{Dataset, Row, SparkSession} + +import scala.collection.JavaConverters._ + +class KafkaSink extends DataCalcSink[KafkaSinkConfig] with Logging { + + def output(spark: SparkSession, ds: Dataset[Row]): Unit = { + var options = Map("kafka.bootstrap.servers" -> config.getServers, "topic" -> config.getTopic) + + if (config.getOptions != null && !config.getOptions.isEmpty) { + options = config.getOptions.asScala.toMap ++ options + } + logger.info(s"Load data to kafka servers: ${config.getServers}, topic: ${config.getTopic}") + + config.getMode match { + case "batch" => + ds.selectExpr("to_json(struct(*)) AS value").write.format("kafka").options(options).save() + case "stream" => + options = Map("checkpointLocation" -> config.getCheckpointLocation) ++ options + ds.writeStream.format("kafka").options(options).start().awaitTermination() + case _ => + } + } + +} diff --git a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/datacalc/sink/MongoSink.scala b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/datacalc/sink/MongoSink.scala new file mode 100644 index 0000000000..d26df410c9 --- /dev/null +++ b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/datacalc/sink/MongoSink.scala @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineplugin.spark.datacalc.sink + +import org.apache.linkis.common.utils.Logging +import org.apache.linkis.engineplugin.spark.datacalc.api.DataCalcSink + +import org.apache.commons.lang3.StringUtils +import org.apache.spark.sql.{Dataset, Row, SparkSession} + +import scala.collection.JavaConverters._ + +class MongoSink extends DataCalcSink[MongoSinkConfig] with Logging { + + def output(spark: SparkSession, ds: Dataset[Row]): Unit = { + var options = Map( + "spark.mongodb.output.database" -> config.getDatabase, + "spark.mongodb.output.collection" -> config.getCollection, + "spark.mongodb.output.uri" -> config.getUri + ) + + if (config.getOptions != null && !config.getOptions.isEmpty) { + options = config.getOptions.asScala.toMap ++ options + } + + val writer = ds.write.format("mongo") + if (StringUtils.isNotBlank(config.getSaveMode)) { + writer.mode(config.getSaveMode) + } + + logger.info( + s"Load data to mongo uri: ${config.getUri}, database: ${config.getDatabase}, collection: ${config.getCollection}" + ) + writer.options(options).save() + } + +} diff --git a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/datacalc/sink/RedisSink.scala b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/datacalc/sink/RedisSink.scala new file mode 100644 index 0000000000..be03cac74b --- /dev/null +++ b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/datacalc/sink/RedisSink.scala @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineplugin.spark.datacalc.sink + +import org.apache.linkis.common.utils.Logging +import org.apache.linkis.engineplugin.spark.datacalc.api.DataCalcSink + +import org.apache.commons.lang3.StringUtils +import org.apache.spark.sql.{Dataset, Row, SparkSession} + +import scala.collection.JavaConverters._ + +class RedisSink extends DataCalcSink[RedisSinkConfig] with Logging { + + def output(spark: SparkSession, ds: Dataset[Row]): Unit = { + var options = Map( + "host" -> config.getHost, + "port" -> config.getPort, + "dbNum" -> config.getDbNum, + "auth" -> config.getAuth, + "table" -> config.getTargetTable + ) + + if (config.getOptions != null && !config.getOptions.isEmpty) { + options = config.getOptions.asScala.toMap ++ options + } + + val writer = ds.write.format("org.apache.spark.sql.redis") + if (StringUtils.isNotBlank(config.getSaveMode)) { + writer.mode(config.getSaveMode) + } + logger.info( + s"Save data to reids host: ${config.getHost}, port: ${config.getPort}, dbNum: ${config.getDbNum}, table: ${config.getTargetTable}" + ) + writer.options(options).save() + } + +} diff --git a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/datacalc/sink/RocketmqSink.scala b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/datacalc/sink/RocketmqSink.scala new file mode 100644 index 0000000000..b2a69f249e --- /dev/null +++ b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/datacalc/sink/RocketmqSink.scala @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineplugin.spark.datacalc.sink + +import org.apache.linkis.common.utils.Logging +import org.apache.linkis.engineplugin.spark.datacalc.api.DataCalcSink + +import org.apache.spark.sql.{Dataset, Row, SparkSession} +import org.apache.spark.sql.functions.lit + +import scala.collection.JavaConverters._ + +class RocketmqSink extends DataCalcSink[RocketmqSinkConfig] with Logging { + + def output(spark: SparkSession, ds: Dataset[Row]): Unit = { + var options = Map("nameServer" -> config.getNameServer, "topic" -> config.getTopic) + + if (config.getOptions != null && !config.getOptions.isEmpty) { + options = config.getOptions.asScala.toMap ++ options + } + + logger.info( + s"Load data to rocketmq nameServer: ${config.getNameServer}, topic: ${config.getTopic}" + ) + + ds.show(false) + config.getMode match { + case "batch" => + ds.selectExpr("to_json(struct(*)) AS body") + .write + .format("org.apache.spark.sql.rocketmq.RocketMQSourceProvider") + .options(options) + .save() + case "stream" => + options = Map("checkpointLocation" -> config.getCheckpointLocation) ++ options + ds.writeStream + .format("org.apache.spark.sql.rocketmq.RocketMQSourceProvider") + .options(options) + .start() + .awaitTermination() + case _ => + } + } + +} diff --git a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/datacalc/sink/SolrSink.scala b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/datacalc/sink/SolrSink.scala new file mode 100644 index 0000000000..af6e9be173 --- /dev/null +++ b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/datacalc/sink/SolrSink.scala @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineplugin.spark.datacalc.sink + +import org.apache.linkis.common.utils.Logging +import org.apache.linkis.engineplugin.spark.datacalc.api.DataCalcSink + +import org.apache.commons.lang3.StringUtils +import org.apache.spark.sql.{Dataset, Row, SparkSession} + +import scala.collection.JavaConverters._ + +class SolrSink extends DataCalcSink[SolrSinkConfig] with Logging { + + def output(spark: SparkSession, ds: Dataset[Row]): Unit = { + var options = Map( + "zkhost" -> config.getZkhost, + "collection" -> config.getCollection, + "soft_commit_secs" -> config.getSoftCommitSecs, + "batch_size" -> config.getBatchSize + ) + + if (config.getOptions != null && !config.getOptions.isEmpty) { + options = config.getOptions.asScala.toMap ++ options + } + + val writer = ds.write.format("solr") + if (StringUtils.isNotBlank(config.getSaveMode)) { + writer.mode(config.getSaveMode) + } + logger.info( + s"Save data to solr zkhost: ${config.getZkhost}, collection: ${config.getCollection}" + ) + writer.options(options).save() + } + +} diff --git a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/datacalc/sink/StarrocksSink.scala b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/datacalc/sink/StarrocksSink.scala new file mode 100644 index 0000000000..bda047a9f2 --- /dev/null +++ b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/datacalc/sink/StarrocksSink.scala @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineplugin.spark.datacalc.sink + +import org.apache.linkis.common.utils.Logging +import org.apache.linkis.engineplugin.spark.datacalc.api.DataCalcSink + +import org.apache.spark.sql.{Dataset, Row, SparkSession} + +import scala.collection.JavaConverters._ + +class StarrocksSink extends DataCalcSink[StarrocksSinkConfig] with Logging { + + def output(spark: SparkSession, ds: Dataset[Row]): Unit = { + var options = Map( + "spark.starrocks.conf" -> "write", + "spark.starrocks.write.fe.urls.http" -> config.getUrl, + "spark.starrocks.write.fe.urls.jdbc" -> config.getJdbcUrl, + "spark.starrocks.write.username" -> config.getUser, + "spark.starrocks.write.password" -> config.getPassword, + "spark.starrocks.write.properties.ignore_json_size" -> "true", + "spark.starrocks.write.database" -> config.getTargetDatabase, + "spark.starrocks.write.table" -> config.getTargetTable + ) + + if (config.getOptions != null && !config.getOptions.isEmpty) { + options = config.getOptions.asScala.toMap ++ options + } + + // todo The starrocks connector currently only supports the 'append' mode, using the starrocks 'Primary Key table' to do 'upsert' + val writer = ds.write.format("starrocks_writer").mode("append") + + logger.info( + s"Save data from starrocks url: ${config.getUrl}, targetDatabase: ${config.getTargetDatabase}, targetTable: ${config.getTargetTable}" + ) + writer.options(options).save() + } + +} diff --git a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/datacalc/source/DataLakeSource.scala b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/datacalc/source/DataLakeSource.scala new file mode 100644 index 0000000000..d1a07211c8 --- /dev/null +++ b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/datacalc/source/DataLakeSource.scala @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineplugin.spark.datacalc.source + +import org.apache.linkis.common.utils.Logging +import org.apache.linkis.engineplugin.spark.datacalc.api.DataCalcSource + +import org.apache.commons.text.StringSubstitutor +import org.apache.spark.sql.{Dataset, Row, SparkSession} + +class DataLakeSource extends DataCalcSource[DataLakeSourceConfig] with Logging { + + override def getData(spark: SparkSession): Dataset[Row] = { + val reader = spark.read + + if (config.getOptions != null && !config.getOptions.isEmpty) { + reader.options(config.getOptions) + } + val substitutor = new StringSubstitutor(config.getVariables) + val path = substitutor.replace(config.getPath) + logger.info(s"Load data to ${config.getTableFormat} tablePath: ${config.getPath}") + + reader.format(config.getTableFormat).load(path) + } + +} diff --git a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/datacalc/source/DorisSource.scala b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/datacalc/source/DorisSource.scala new file mode 100644 index 0000000000..a4819f2181 --- /dev/null +++ b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/datacalc/source/DorisSource.scala @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineplugin.spark.datacalc.source + +import org.apache.linkis.common.utils.Logging +import org.apache.linkis.engineplugin.spark.datacalc.api.DataCalcSource + +import org.apache.spark.sql.{Dataset, Row, SparkSession} + +class DorisSource extends DataCalcSource[DorisSourceConfig] with Logging { + + override def getData(spark: SparkSession): Dataset[Row] = { + val reader = spark.read.format("doris") + + if (config.getOptions != null && !config.getOptions.isEmpty) { + reader.options(config.getOptions) + } + + logger.info( + s"Load data from Doris url: ${config.getUrl}, sourceDatabase: ${config.getSourceDatabase}, sourceTable: ${config.getSourceTable}" + ) + + reader + .option( + "doris.table.identifier", + String.format("%s.%s", config.getSourceDatabase, config.getSourceTable) + ) + .option("doris.fenodes", config.getUrl) + .option("user", config.getUser) + .option("password", config.getPassword) + .load() + } + +} diff --git a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/datacalc/source/ElasticsearchSource.scala b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/datacalc/source/ElasticsearchSource.scala new file mode 100644 index 0000000000..f06a40db96 --- /dev/null +++ b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/datacalc/source/ElasticsearchSource.scala @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineplugin.spark.datacalc.source + +import org.apache.linkis.common.utils.Logging +import org.apache.linkis.engineplugin.spark.datacalc.api.DataCalcSource + +import org.apache.spark.sql.{Dataset, Row, SparkSession} + +class ElasticsearchSource extends DataCalcSource[ElasticsearchSourceConfig] with Logging { + + override def getData(spark: SparkSession): Dataset[Row] = { + val reader = spark.read.format("org.elasticsearch.spark.sql") + + if (config.getOptions != null && !config.getOptions.isEmpty) { + reader.options(config.getOptions) + } + + logger.info( + s"Load data from elasticsearch nodes: ${config.getNode}, port: ${config.getPort}, index: ${config.getIndex}" + ) + + reader + .option("es.nodes", config.getNode) + .option("es.port", config.getPort) + .option("es.net.http.auth.user", config.getUser) + .option("es.net.http.auth.pass", config.getPassword) + .option("es.query", config.getQuery) + .option("es.resource", s"${config.getIndex}/${config.getType}") + .load() + } + +} diff --git a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/datacalc/source/FileSource.scala b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/datacalc/source/FileSource.scala index 8b579a0b65..ae53d6b02c 100644 --- a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/datacalc/source/FileSource.scala +++ b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/datacalc/source/FileSource.scala @@ -41,6 +41,7 @@ class FileSource extends DataCalcSource[FileSourceConfig] with Logging { case "parquet" => reader.parquet(path) case "text" => reader.text(path) case "orc" => reader.orc(path) + case "excel" => reader.format("excel").load(path) case _ => reader.format(config.getSerializer).load(path) } if (config.getColumnNames != null && config.getColumnNames.length > 0) { diff --git a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/datacalc/source/KafkaSource.scala b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/datacalc/source/KafkaSource.scala new file mode 100644 index 0000000000..519b1bfddc --- /dev/null +++ b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/datacalc/source/KafkaSource.scala @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineplugin.spark.datacalc.source + +import org.apache.linkis.common.utils.Logging +import org.apache.linkis.engineplugin.spark.datacalc.api.DataCalcSource + +import org.apache.spark.sql.{Dataset, Row, SparkSession} + +class KafkaSource extends DataCalcSource[KafkaSourceConfig] with Logging { + + override def getData(spark: SparkSession): Dataset[Row] = { + val reader = spark.readStream.format("kafka") + if (config.getOptions != null && !config.getOptions.isEmpty) { + reader.options(config.getOptions) + } + + logger.info(s"Load data from kafka servers: ${config.getServers}, topic: ${config.getTopic}") + + reader + .option("kafka.bootstrap.servers", config.getServers) + .option("subscribe", config.getTopic) + .option("startingOffsets", config.getConsumeMode) + .load() + } + +} diff --git a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/datacalc/source/MongoSource.scala b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/datacalc/source/MongoSource.scala new file mode 100644 index 0000000000..ad26cd92df --- /dev/null +++ b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/datacalc/source/MongoSource.scala @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineplugin.spark.datacalc.source + +import org.apache.linkis.common.utils.Logging +import org.apache.linkis.engineplugin.spark.datacalc.api.DataCalcSource + +import org.apache.spark.sql.{Dataset, Row, SparkSession} + +class MongoSource extends DataCalcSource[MongoSourceConfig] with Logging { + + override def getData(spark: SparkSession): Dataset[Row] = { + val reader = spark.read.format("mongo") + + if (config.getOptions != null && !config.getOptions.isEmpty) { + reader.options(config.getOptions) + } + + logger.info( + s"Load data from mongo uri: ${config.getUri}, database: ${config.getDatabase}, collection: ${config.getCollection}" + ) + + reader + .option("spark.mongodb.input.database", config.getDatabase) + .option("spark.mongodb.input.collection", config.getCollection) + .option("spark.mongodb.input.uri", config.getUri) + .load() + } + +} diff --git a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/datacalc/source/RedisSource.scala b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/datacalc/source/RedisSource.scala new file mode 100644 index 0000000000..67b3f2e0a5 --- /dev/null +++ b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/datacalc/source/RedisSource.scala @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineplugin.spark.datacalc.source + +import org.apache.linkis.common.utils.Logging +import org.apache.linkis.engineplugin.spark.datacalc.api.DataCalcSource + +import org.apache.spark.sql.{Dataset, Row, SparkSession} + +class RedisSource extends DataCalcSource[RedisSourceConfig] with Logging { + + override def getData(spark: SparkSession): Dataset[Row] = { + val reader = spark.read.format("org.apache.spark.sql.redis") + if (config.getOptions != null && !config.getOptions.isEmpty) { + reader.options(config.getOptions) + } + + logger.info( + s"Load data from reids host: ${config.getHost}, port: ${config.getPort}, dbNum: ${config.getDbNum}" + ) + + config.getSerializer match { + case "table" => reader.option("table", config.getSourceTable) + case "keysPattern" => + reader.option("keys.pattern", config.getKeysPattern).option("infer.schema", "true") + } + reader + .option("host", config.getHost) + .option("port", config.getPort) + .option("dbNum", config.getDbNum) + .option("auth", config.getAuth) + .load() + } + +} diff --git a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/datacalc/source/RocketmqSource.scala b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/datacalc/source/RocketmqSource.scala new file mode 100644 index 0000000000..b24ba2ac93 --- /dev/null +++ b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/datacalc/source/RocketmqSource.scala @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineplugin.spark.datacalc.source + +import org.apache.linkis.common.utils.Logging +import org.apache.linkis.engineplugin.spark.datacalc.api.DataCalcSource + +import org.apache.spark.sql.{Dataset, Row, SparkSession} + +class RocketmqSource extends DataCalcSource[RocketmqSourceConfig] with Logging { + + override def getData(spark: SparkSession): Dataset[Row] = { + val reader = spark.readStream.format("org.apache.spark.sql.rocketmq.RocketMQSourceProvider") + if (config.getOptions != null && !config.getOptions.isEmpty) { + reader.options(config.getOptions) + } + + logger.info( + s"Load data from rocketmq nameServer: ${config.getNameServer}, topic: ${config.getTopic}" + ) + + reader + .option("nameServer", config.getNameServer) + .option("topic", config.getTopic) + .option("startingOffsets", config.getConsumeMode) + .load() + } + +} diff --git a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/datacalc/source/SolrSource.scala b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/datacalc/source/SolrSource.scala new file mode 100644 index 0000000000..950eae61da --- /dev/null +++ b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/datacalc/source/SolrSource.scala @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineplugin.spark.datacalc.source + +import org.apache.linkis.common.utils.Logging +import org.apache.linkis.engineplugin.spark.datacalc.api.DataCalcSource + +import org.apache.spark.sql.{Dataset, Row, SparkSession} + +class SolrSource extends DataCalcSource[SolrSourceConfig] with Logging { + + override def getData(spark: SparkSession): Dataset[Row] = { + val reader = spark.read.format("solr") + if (config.getOptions != null && !config.getOptions.isEmpty) { + reader.options(config.getOptions) + } + + logger.info( + s"Save data from solr zkhost: ${config.getZkhost}, collection: ${config.getCollection}" + ) + + reader + .option("zkhost", config.getZkhost) + .option("collection", config.getCollection) + .option("query", "*:*") + .load() + } + +} diff --git a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/datacalc/source/StarrocksSource.scala b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/datacalc/source/StarrocksSource.scala new file mode 100644 index 0000000000..ea2a9fa042 --- /dev/null +++ b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/datacalc/source/StarrocksSource.scala @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineplugin.spark.datacalc.source + +import org.apache.linkis.common.utils.Logging +import org.apache.linkis.engineplugin.spark.datacalc.api.DataCalcSource + +import org.apache.spark.sql.{Dataset, Row, SparkSession} + +class StarrocksSource extends DataCalcSource[StarrocksSourceConfig] with Logging { + + override def getData(spark: SparkSession): Dataset[Row] = { + val reader = spark.read.format("starrocks") + + if (config.getOptions != null && !config.getOptions.isEmpty) { + reader.options(config.getOptions) + } + + logger.info( + s"Load data from starrocks url: ${config.getUrl}, sourceDatabase: ${config.getSourceDatabase}, sourceTable: ${config.getSourceTable}" + ) + + reader + .option( + "starrocks.table.identifier", + String.format("%s.%s", config.getSourceDatabase, config.getSourceTable) + ) + .option("starrocks.fenodes", config.getUrl) + .option("user", config.getUser) + .option("password", config.getPassword) + .load() + } + +} diff --git a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/executor/SparkEngineConnExecutor.scala b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/executor/SparkEngineConnExecutor.scala index 6b6365bccf..7fed0f436d 100644 --- a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/executor/SparkEngineConnExecutor.scala +++ b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/executor/SparkEngineConnExecutor.scala @@ -30,7 +30,7 @@ import org.apache.linkis.engineconn.computation.executor.utlis.{ } import org.apache.linkis.engineconn.core.EngineConnObject import org.apache.linkis.engineconn.core.exception.ExecutorHookFatalException -import org.apache.linkis.engineconn.executor.entity.ResourceFetchExecutor +import org.apache.linkis.engineconn.executor.entity.{ResourceFetchExecutor, YarnExecutor} import org.apache.linkis.engineplugin.spark.common.{Kind, SparkDataCalc} import org.apache.linkis.engineplugin.spark.config.SparkConfiguration import org.apache.linkis.engineplugin.spark.cs.CSSparkHelper @@ -63,6 +63,7 @@ import scala.collection.mutable.ArrayBuffer abstract class SparkEngineConnExecutor(val sc: SparkContext, id: Long) extends ComputationExecutor with Logging + with YarnExecutor with ResourceFetchExecutor { private var initialized: Boolean = false @@ -80,9 +81,16 @@ abstract class SparkEngineConnExecutor(val sc: SparkContext, id: Long) private var thread: Thread = _ + private var applicationId: String = sc.applicationId + + override def getApplicationId: String = applicationId + + override def getApplicationURL: String = "" + override def getYarnMode: String = "" + override def getQueue: String = "" + override def init(): Unit = { logger.info(s"Ready to change engine state!") -// setCodeParser() // todo check super.init() } diff --git a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/executor/SparkOnKubernetesSubmitOnceExecutor.scala b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/executor/SparkOnKubernetesSubmitOnceExecutor.scala new file mode 100644 index 0000000000..e8b7dfb489 --- /dev/null +++ b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/executor/SparkOnKubernetesSubmitOnceExecutor.scala @@ -0,0 +1,164 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineplugin.spark.executor + +import org.apache.linkis.common.utils.{ByteTimeUtils, Utils} +import org.apache.linkis.engineconn.once.executor.{ + OnceExecutorExecutionContext, + OperableOnceExecutor +} +import org.apache.linkis.engineplugin.spark.client.deployment.{ + KubernetesApplicationClusterDescriptorAdapter, + YarnApplicationClusterDescriptorAdapter +} +import org.apache.linkis.engineplugin.spark.config.SparkConfiguration.{ + SPARK_APP_CONF, + SPARK_APPLICATION_ARGS, + SPARK_APPLICATION_MAIN_CLASS +} +import org.apache.linkis.engineplugin.spark.context.SparkEngineConnContext +import org.apache.linkis.engineplugin.spark.utils.SparkJobProgressUtil +import org.apache.linkis.manager.common.entity.resource._ +import org.apache.linkis.manager.common.utils.ResourceUtils +import org.apache.linkis.protocol.engine.JobProgressInfo + +import org.apache.commons.lang3.StringUtils + +import java.util + +import scala.concurrent.duration.Duration + +import io.fabric8.kubernetes.api.model.Quantity + +class SparkOnKubernetesSubmitOnceExecutor( + override val id: Long, + override protected val sparkEngineConnContext: SparkEngineConnContext +) extends SparkOnceExecutor[KubernetesApplicationClusterDescriptorAdapter] + with OperableOnceExecutor { + + private var oldProgress: Float = 0f + + override def doSubmit( + onceExecutorExecutionContext: OnceExecutorExecutionContext, + options: Map[String, String] + ): Unit = { + val args = SPARK_APPLICATION_ARGS.getValue(options) + val mainClass = SPARK_APPLICATION_MAIN_CLASS.getValue(options) + val extConf = SPARK_APP_CONF.getValue(options) + val confMap = new util.HashMap[String, String]() + if (StringUtils.isNotBlank(extConf)) { + for (conf <- extConf.split("\n")) { + if (StringUtils.isNotBlank(conf)) { + val pair = conf.trim.split("=") + if (pair.length == 2) { + confMap.put(pair(0), pair(1)) + } else { + logger.warn(s"ignore spark conf: $conf") + } + } + } + } + logger.info( + s"Ready to submit spark application to kubernetes, mainClass: $mainClass, args: $args." + ) + clusterDescriptorAdapter.deployCluster(mainClass, args, confMap) + } + + override protected def waitToRunning(): Unit = { + // Wait until the task return applicationId (等待返回applicationId) + Utils.waitUntil(() => clusterDescriptorAdapter.initJobId(), Duration.Inf) + // Synchronize applicationId to EC SparkOnceExecutor to facilitate user operations, + // such as obtaining progress and killing jobs(将applicationId同步给EC执行器，方便用户操作，如获取进度，kill任务等) + setApplicationId(clusterDescriptorAdapter.getApplicationId) + super.waitToRunning() + } + + override def getApplicationURL: String = "" + + override def getCurrentNodeResource(): NodeResource = { + logger.info("Begin to get actual used resources!") + Utils.tryCatch({ + val sparkConf = sparkEngineConnContext.getExecutionContext.getSparkConfig + val sparkNamespace = sparkConf.getK8sNamespace + + val executorNum: Int = sparkConf.getNumExecutors + val executorMem: Long = + ByteTimeUtils.byteStringAsBytes(sparkConf.getExecutorMemory) * executorNum + val driverMem: Long = ByteTimeUtils.byteStringAsBytes(sparkConf.getDriverMemory) + + val executorCoresQuantity = Quantity.parse(sparkConf.getK8sExecutorRequestCores) + val executorCores: Long = + (Quantity.getAmountInBytes(executorCoresQuantity).doubleValue() * 1000).toLong * executorNum + val driverCoresQuantity = Quantity.parse(sparkConf.getK8sDriverRequestCores) + val driverCores: Long = + (Quantity.getAmountInBytes(driverCoresQuantity).doubleValue() * 1000).toLong + + logger.info( + "Current actual used resources is driverMem:" + driverMem + ",driverCores:" + driverCores + ",executorMem:" + executorMem + ",executorCores:" + executorCores + ",namespace:" + sparkNamespace + ) + val usedResource = new DriverAndKubernetesResource( + new LoadInstanceResource(0, 0, 0), + new KubernetesResource(executorMem + driverMem, executorCores + driverCores, sparkNamespace) + ) + val nodeResource = new CommonNodeResource + nodeResource.setUsedResource(usedResource) + nodeResource.setResourceType(ResourceUtils.getResourceTypeByResource(usedResource)) + nodeResource + })(t => { + logger.warn("Get actual used resource exception", t) + null + }) + } + + override def getProgress: Float = { + val jobIsFinal = clusterDescriptorAdapter != null && + clusterDescriptorAdapter.getJobState != null && + clusterDescriptorAdapter.getJobState.isFinal + if (oldProgress >= 1 || jobIsFinal) { + 1 + } else { + val sparkUIUrl = this.clusterDescriptorAdapter.getSparkUIUrl + if (StringUtils.isNotBlank(sparkUIUrl)) { + val newProgress = + SparkJobProgressUtil.getProgress(this.getApplicationId, sparkUIUrl) + if (newProgress > oldProgress) { + oldProgress = newProgress + } + } + oldProgress + } + } + + override def getProgressInfo: Array[JobProgressInfo] = { + val sparkUIUrl = this.clusterDescriptorAdapter.getSparkUIUrl + if (StringUtils.isNotBlank(sparkUIUrl)) { + SparkJobProgressUtil.getSparkJobProgressInfo(this.getApplicationId, sparkUIUrl) + } else { + Array.empty + } + } + + override def getMetrics: util.Map[String, Any] = { + new util.HashMap[String, Any]() + } + + override def getDiagnosis: util.Map[String, Any] = { + new util.HashMap[String, Any]() + } + +} diff --git a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/executor/SparkPythonExecutor.scala b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/executor/SparkPythonExecutor.scala index f79b263da6..5d4305c67c 100644 --- a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/executor/SparkPythonExecutor.scala +++ b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/executor/SparkPythonExecutor.scala @@ -38,7 +38,7 @@ import org.apache.linkis.storage.resultset.ResultSetWriter import org.apache.commons.exec.CommandLine import org.apache.commons.io.IOUtils -import org.apache.commons.lang3.{RandomStringUtils, StringUtils} +import org.apache.commons.lang3.StringUtils import org.apache.spark.SparkConf import org.apache.spark.api.java.JavaSparkContext import org.apache.spark.sql.{DataFrame, SparkSession} @@ -76,7 +76,7 @@ class SparkPythonExecutor(val sparkEngineSession: SparkEngineSession, val id: In private val lineOutputStream = new RsOutputStream val sqlContext = sparkEngineSession.sqlContext val SUCCESS = "success" - private lazy val py4jToken: String = RandomStringUtils.randomAlphanumeric(256) + private lazy val py4jToken: String = SecureRandomStringUtils.randomAlphanumeric(256) private lazy val gwBuilder: GatewayServerBuilder = { val builder = new GatewayServerBuilder() diff --git a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/executor/SparkSqlExecutor.scala b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/executor/SparkSqlExecutor.scala index ada91499c3..33b3dadb38 100644 --- a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/executor/SparkSqlExecutor.scala +++ b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/executor/SparkSqlExecutor.scala @@ -18,14 +18,12 @@ package org.apache.linkis.engineplugin.spark.executor import org.apache.linkis.common.utils.Utils -import org.apache.linkis.engineconn.computation.executor.entity.EngineConnTask import org.apache.linkis.engineconn.computation.executor.execute.EngineExecutionContext import org.apache.linkis.engineplugin.spark.common.{Kind, SparkSQL} import org.apache.linkis.engineplugin.spark.config.SparkConfiguration import org.apache.linkis.engineplugin.spark.entity.SparkEngineSession import org.apache.linkis.engineplugin.spark.utils.EngineUtils import org.apache.linkis.governance.common.constant.job.JobRequestConstants -import org.apache.linkis.governance.common.entity.ExecutionNodeStatus import org.apache.linkis.governance.common.paser.SQLCodeParser import org.apache.linkis.scheduler.executer.{ ErrorExecuteResponse, @@ -91,7 +89,6 @@ class SparkSqlExecutor(sparkEngineSession: SparkEngineSession, id: Long) ) ) ) - transformTaskStatus(lastTask, ExecutionNodeStatus.Succeed) SQLSession.showDF( sparkEngineSession.sparkContext, jobGroup, diff --git a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/executor/SparkSubmitOnceExecutor.scala b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/executor/SparkSubmitOnceExecutor.scala index ee823d5415..e744232e6e 100644 --- a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/executor/SparkSubmitOnceExecutor.scala +++ b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/executor/SparkSubmitOnceExecutor.scala @@ -22,7 +22,7 @@ import org.apache.linkis.engineconn.once.executor.{ OnceExecutorExecutionContext, OperableOnceExecutor } -import org.apache.linkis.engineplugin.spark.client.deployment.YarnApplicationClusterDescriptorAdapter +import org.apache.linkis.engineplugin.spark.client.deployment.ClusterDescriptorAdapter import org.apache.linkis.engineplugin.spark.config.SparkConfiguration.{ SPARK_APP_CONF, SPARK_APPLICATION_ARGS, @@ -43,7 +43,7 @@ import scala.concurrent.duration.Duration class SparkSubmitOnceExecutor( override val id: Long, override protected val sparkEngineConnContext: SparkEngineConnContext -) extends SparkOnceExecutor[YarnApplicationClusterDescriptorAdapter] +) extends SparkOnceExecutor[ClusterDescriptorAdapter] with OperableOnceExecutor { private var oldProgress: Float = 0f diff --git a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/factory/SparkEngineConnFactory.scala b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/factory/SparkEngineConnFactory.scala index 2743b73a97..ad9786dff0 100644 --- a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/factory/SparkEngineConnFactory.scala +++ b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/factory/SparkEngineConnFactory.scala @@ -41,6 +41,7 @@ import org.apache.linkis.manager.engineplugin.common.launch.process.Environment import org.apache.linkis.manager.engineplugin.common.launch.process.Environment.variable import org.apache.linkis.manager.label.entity.engine.EngineType import org.apache.linkis.manager.label.entity.engine.EngineType.EngineType +import org.apache.linkis.manager.label.utils.LabelUtil import org.apache.linkis.server.JMap import org.apache.commons.lang3.StringUtils @@ -84,18 +85,44 @@ class SparkEngineConnFactory extends MultiExecutorEngineConnFactory with Logging val hadoopConfDir = EnvConfiguration.HADOOP_CONF_DIR.getValue(options) val sparkHome = SPARK_HOME.getValue(options) val sparkConfDir = SPARK_CONF_DIR.getValue(options) - val sparkConfig: SparkConfig = getSparkConfig(options) + val sparkConfig: SparkConfig = + getSparkConfig(options, LabelUtil.isYarnClusterMode(engineCreationContext.getLabels())) val context = new EnvironmentContext(sparkConfig, hadoopConfDir, sparkConfDir, sparkHome, null) context } - def getSparkConfig(options: util.Map[String, String]): SparkConfig = { + def getSparkConfig(options: util.Map[String, String], isYarnClusterMode: Boolean): SparkConfig = { logger.info("options: " + JsonUtils.jackson.writeValueAsString(options)) val sparkConfig: SparkConfig = new SparkConfig() sparkConfig.setJavaHome(variable(Environment.JAVA_HOME)) sparkConfig.setSparkHome(SPARK_HOME.getValue(options)) - sparkConfig.setMaster(SPARK_MASTER.getValue(options)) - sparkConfig.setDeployMode(SPARK_DEPLOY_MODE.getValue(options)) + val master = SPARK_MASTER.getValue(options) + sparkConfig.setMaster(master) + if (master.startsWith("k8s")) { + sparkConfig.setK8sConfigFile(SPARK_K8S_CONFIG_FILE.getValue(options)) + sparkConfig.setK8sServiceAccount(SPARK_K8S_SERVICE_ACCOUNT.getValue(options)) + sparkConfig.setK8sMasterUrl(SPARK_K8S_MASTER_URL.getValue(options)) + sparkConfig.setK8sUsername(SPARK_K8S_USERNAME.getValue(options)) + sparkConfig.setK8sPassword(SPARK_K8S_PASSWORD.getValue(options)) + sparkConfig.setK8sImage(SPARK_K8S_IMAGE.getValue(options)) + sparkConfig.setK8sNamespace(SPARK_K8S_NAMESPACE.getValue(options)) + sparkConfig.setK8sFileUploadPath(SPARK_KUBERNETES_FILE_UPLOAD_PATH.getValue(options)) + sparkConfig.setK8sSparkVersion(SPARK_K8S_SPARK_VERSION.getValue(options)) + sparkConfig.setK8sRestartPolicy(SPARK_K8S_RESTART_POLICY.getValue(options)) + sparkConfig.setK8sLanguageType(SPARK_K8S_LANGUAGE_TYPE.getValue(options)) + sparkConfig.setK8sImagePullPolicy(SPARK_K8S_IMAGE_PULL_POLICY.getValue(options)) + sparkConfig.setK8sDriverRequestCores(SPARK_K8S_DRIVER_REQUEST_CORES.getValue(options)) + sparkConfig.setK8sExecutorRequestCores(SPARK_K8S_EXECUTOR_REQUEST_CORES.getValue(options)) + sparkConfig.setK8sSparkUIPort(SPARK_K8S_UI_PORT.getValue(options)) + } + + if (master.startsWith("yarn")) { + if (isYarnClusterMode) { + sparkConfig.setDeployMode(SparkConfiguration.SPARK_YARN_CLUSTER) + } else { + sparkConfig.setDeployMode(SparkConfiguration.SPARK_YARN_CLIENT) + } + } sparkConfig.setAppResource(SPARK_APP_RESOURCE.getValue(options)) sparkConfig.setAppName(SPARK_APP_NAME.getValue(options)) sparkConfig.setJars(SPARK_EXTRA_JARS.getValue(options)) // todo @@ -108,6 +135,7 @@ class SparkEngineConnFactory extends MultiExecutorEngineConnFactory with Logging sparkConfig.setExecutorCores(LINKIS_SPARK_EXECUTOR_CORES.getValue(options)) sparkConfig.setNumExecutors(LINKIS_SPARK_EXECUTOR_INSTANCES.getValue(options)) sparkConfig.setQueue(LINKIS_QUEUE_NAME.getValue(options)) + sparkConfig.setPyFiles(SPARK_PYTHON_FILES.getValue(options)) logger.info(s"spark_info: ${sparkConfig}") sparkConfig @@ -129,19 +157,29 @@ class SparkEngineConnFactory extends MultiExecutorEngineConnFactory with Logging val master = sparkConf.getOption("spark.master").getOrElse(CommonVars("spark.master", "yarn").getValue) logger.info(s"------ Create new SparkContext {$master} -------") - val pysparkBasePath = SparkConfiguration.SPARK_HOME.getValue - val pysparkPath = new File(pysparkBasePath, "python" + File.separator + "lib") - var pythonLibUris = pysparkPath.listFiles().map(_.toURI.toString).filter(_.endsWith(".zip")) - if (pythonLibUris.length == 2) { - val sparkConfValue1 = Utils.tryQuietly(CommonVars("spark.yarn.dist.files", "").getValue) - val sparkConfValue2 = Utils.tryQuietly(sparkConf.get("spark.yarn.dist.files")) - if (StringUtils.isNotBlank(sparkConfValue2)) { - pythonLibUris = sparkConfValue2 +: pythonLibUris - } - if (StringUtils.isNotBlank(sparkConfValue1)) { - pythonLibUris = sparkConfValue1 +: pythonLibUris + + val isYarnClusterMode = LabelUtil.isYarnClusterMode(engineCreationContext.getLabels()) + + if (isYarnClusterMode) { + sparkConf.set("spark.submit.deployMode", "cluster") + } + + // todo yarn cluster暂时不支持pyspark,后期对pyspark进行处理 + if (!isYarnClusterMode) { + val pysparkBasePath = SparkConfiguration.SPARK_HOME.getValue + val pysparkPath = new File(pysparkBasePath, "python" + File.separator + "lib") + var pythonLibUris = pysparkPath.listFiles().map(_.toURI.toString).filter(_.endsWith(".zip")) + if (pythonLibUris.length == 2) { + val sparkConfValue1 = Utils.tryQuietly(CommonVars("spark.yarn.dist.files", "").getValue) + val sparkConfValue2 = Utils.tryQuietly(sparkConf.get("spark.yarn.dist.files")) + if (StringUtils.isNotBlank(sparkConfValue2)) { + pythonLibUris = sparkConfValue2 +: pythonLibUris + } + if (StringUtils.isNotBlank(sparkConfValue1)) { + pythonLibUris = sparkConfValue1 +: pythonLibUris + } + sparkConf.set("spark.yarn.dist.files", pythonLibUris.mkString(",")) } - sparkConf.set("spark.yarn.dist.files", pythonLibUris.mkString(",")) } // Distributes needed libraries to workers // when spark version is greater than or equal to 1.5.0 @@ -212,6 +250,10 @@ class SparkEngineConnFactory extends MultiExecutorEngineConnFactory with Logging if (System.getenv("SPARK_HOME") != null) conf.setSparkHome(System.getenv("SPARK_HOME")) conf.set("spark.scheduler.mode", "FAIR") + if (SparkConfiguration.LINKIS_SPARK_ETL_SUPPORT_HUDI.getValue) { + conf.set("spark.serializer", "org.apache.spark.serializer.KryoSerializer") + } + val builder = SparkSession.builder.config(conf) builder.enableHiveSupport().getOrCreate() } diff --git a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/factory/SparkEngineConnResourceFactory.scala b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/factory/SparkEngineConnResourceFactory.scala index 8b642c7233..640476a589 100644 --- a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/factory/SparkEngineConnResourceFactory.scala +++ b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/factory/SparkEngineConnResourceFactory.scala @@ -18,21 +18,64 @@ package org.apache.linkis.engineplugin.spark.factory import org.apache.linkis.common.utils.{ByteTimeUtils, Logging} +import org.apache.linkis.engineplugin.spark.config.SparkConfiguration._ import org.apache.linkis.engineplugin.spark.config.SparkResourceConfiguration._ +import org.apache.linkis.manager.common.conf.RMConfiguration.DEFAULT_KUBERNETES_TYPE import org.apache.linkis.manager.common.entity.resource.{ + DriverAndKubernetesResource, DriverAndYarnResource, + KubernetesResource, LoadInstanceResource, Resource, YarnResource } -import org.apache.linkis.manager.engineplugin.common.resource.AbstractEngineResourceFactory +import org.apache.linkis.manager.engineplugin.common.resource.{ + AbstractEngineResourceFactory, + EngineResourceRequest +} +import org.apache.linkis.manager.label.entity.cluster.ClusterLabel +import org.apache.linkis.manager.label.utils.LabelUtil import org.apache.commons.lang3.StringUtils import java.util +import io.fabric8.kubernetes.api.model.Quantity + class SparkEngineConnResourceFactory extends AbstractEngineResourceFactory with Logging { + override protected def getMinRequestResource( + engineResourceRequest: EngineResourceRequest + ): Resource = { + val clusterLabel = LabelUtil.getLabelFromList[ClusterLabel](engineResourceRequest.labels) + if ( + clusterLabel != null && StringUtils.equals( + clusterLabel.getClusterType.toUpperCase(), + DEFAULT_KUBERNETES_TYPE.getValue + ) + ) { + getRequestKubernetesResource(engineResourceRequest.properties) + } else { + getRequestResource(engineResourceRequest.properties) + } + } + + override protected def getMaxRequestResource( + engineResourceRequest: EngineResourceRequest + ): Resource = { + val clusterLabel = LabelUtil.getLabelFromList[ClusterLabel](engineResourceRequest.labels) + if ( + clusterLabel != null && StringUtils.equals( + clusterLabel.getClusterType.toUpperCase(), + DEFAULT_KUBERNETES_TYPE.getValue + ) + ) { + getRequestKubernetesResource(engineResourceRequest.properties) + } else { + getRequestResource(engineResourceRequest.properties) + } + } + override protected def getRequestResource(properties: util.Map[String, String]): Resource = { val executorNum = LINKIS_SPARK_EXECUTOR_INSTANCES.getValue(properties) val executorMemory = LINKIS_SPARK_EXECUTOR_MEMORY.getValue(properties) @@ -63,4 +106,48 @@ class SparkEngineConnResourceFactory extends AbstractEngineResourceFactory with ) } + def getRequestKubernetesResource(properties: util.Map[String, String]): Resource = { + val executorNum = LINKIS_SPARK_EXECUTOR_INSTANCES.getValue(properties) + val executorCores = if (properties.containsKey(SPARK_K8S_EXECUTOR_REQUEST_CORES.key)) { + val executorCoresQuantity = + Quantity.parse(SPARK_K8S_EXECUTOR_REQUEST_CORES.getValue(properties)) + (Quantity.getAmountInBytes(executorCoresQuantity).doubleValue() * 1000).toLong + } else { + val sparkDefaultExecutorCores: Int = LINKIS_SPARK_EXECUTOR_CORES.getValue(properties) + properties.put(SPARK_K8S_EXECUTOR_REQUEST_CORES.key, sparkDefaultExecutorCores.toString) + sparkDefaultExecutorCores * 1000L + } + val executorMemory = LINKIS_SPARK_EXECUTOR_MEMORY.getValue(properties) + val executorMemoryWithUnit = if (StringUtils.isNumeric(executorMemory)) { + executorMemory + "g" + } else { + executorMemory + } + val driverCores = if (properties.containsKey(SPARK_K8S_DRIVER_REQUEST_CORES.key)) { + val executorCoresQuantity = + Quantity.parse(SPARK_K8S_DRIVER_REQUEST_CORES.getValue(properties)) + (Quantity.getAmountInBytes(executorCoresQuantity).doubleValue() * 1000).toLong + } else { + val sparkDefaultDriverCores: Int = LINKIS_SPARK_DRIVER_CORES.getValue(properties) + properties.put(SPARK_K8S_DRIVER_REQUEST_CORES.key, sparkDefaultDriverCores.toString) + sparkDefaultDriverCores * 1000L + } + val driverMemory = LINKIS_SPARK_DRIVER_MEMORY.getValue(properties) + val driverMemoryWithUnit = if (StringUtils.isNumeric(driverMemory)) { + driverMemory + "g" + } else { + driverMemory + } + val totalExecutorMemory = ByteTimeUtils.byteStringAsBytes( + executorMemoryWithUnit + ) * executorNum + ByteTimeUtils.byteStringAsBytes(driverMemoryWithUnit) + val totalExecutorCores = executorCores * executorNum + driverCores + val namespace = SPARK_K8S_NAMESPACE.getValue(properties) + + new DriverAndKubernetesResource( + new LoadInstanceResource(0, 0, 0), + new KubernetesResource(totalExecutorMemory, totalExecutorCores, namespace) + ) + } + } diff --git a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/factory/SparkOnceExecutorFactory.scala b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/factory/SparkOnceExecutorFactory.scala index 25e2649441..5802a1c859 100644 --- a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/factory/SparkOnceExecutorFactory.scala +++ b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/factory/SparkOnceExecutorFactory.scala @@ -22,10 +22,16 @@ import org.apache.linkis.engineconn.common.engineconn.EngineConn import org.apache.linkis.engineconn.once.executor.OnceExecutor import org.apache.linkis.engineconn.once.executor.creation.OnceExecutorFactory import org.apache.linkis.engineplugin.spark.context.SparkEngineConnContext -import org.apache.linkis.engineplugin.spark.executor.SparkSubmitOnceExecutor +import org.apache.linkis.engineplugin.spark.executor.{ + SparkOnKubernetesSubmitOnceExecutor, + SparkSubmitOnceExecutor +} +import org.apache.linkis.manager.common.conf.RMConfiguration.DEFAULT_KUBERNETES_TYPE import org.apache.linkis.manager.label.entity.Label +import org.apache.linkis.manager.label.entity.cluster.ClusterLabel import org.apache.linkis.manager.label.entity.engine.RunType import org.apache.linkis.manager.label.entity.engine.RunType.RunType +import org.apache.linkis.manager.label.utils.LabelUtil class SparkOnceExecutorFactory extends OnceExecutorFactory { @@ -34,11 +40,24 @@ class SparkOnceExecutorFactory extends OnceExecutorFactory { engineCreationContext: EngineCreationContext, engineConn: EngineConn, labels: Array[Label[_]] - ): OnceExecutor = + ): OnceExecutor = { + val clusterLabel = LabelUtil.getLabelFromArray[ClusterLabel](labels) engineConn.getEngineConnSession match { case context: SparkEngineConnContext => - new SparkSubmitOnceExecutor(id, context) + if ( + null != clusterLabel && clusterLabel.getClusterType.equalsIgnoreCase( + DEFAULT_KUBERNETES_TYPE.getValue + ) + ) { + new SparkOnKubernetesSubmitOnceExecutor(id, context) + } else { + new SparkSubmitOnceExecutor(id, context) + } } + } + + override protected def getSupportRunTypes: Array[String] = + Array(RunType.JAR.toString, RunType.PYSPARK.toString) override protected def getRunType: RunType = RunType.JAR } diff --git a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/launch/SparkEngineConnLaunchBuilder.scala b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/launch/SparkEngineConnLaunchBuilder.scala index e35859a790..8be047e2ff 100644 --- a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/launch/SparkEngineConnLaunchBuilder.scala +++ b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/launch/SparkEngineConnLaunchBuilder.scala @@ -74,9 +74,8 @@ class SparkEngineConnLaunchBuilder extends JavaProcessEngineConnLaunchBuilder { engineConnBuildRequest: EngineConnBuildRequest ): util.Map[String, String] = { val environment = super.getEnvironment - if (isOnceMode) { + if (isOnceMode) addPathToClassPath(environment, s"$$$SPARK_HOME_ENV/jars/*") - } environment } diff --git a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/launch/SparkSubmitProcessEngineConnLaunchBuilder.scala b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/launch/SparkSubmitProcessEngineConnLaunchBuilder.scala index 35089980f5..1e0263eebf 100644 --- a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/launch/SparkSubmitProcessEngineConnLaunchBuilder.scala +++ b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/launch/SparkSubmitProcessEngineConnLaunchBuilder.scala @@ -19,24 +19,31 @@ package org.apache.linkis.engineplugin.spark.launch import org.apache.linkis.common.conf.CommonVars import org.apache.linkis.common.utils.Logging +import org.apache.linkis.engineplugin.spark.config.SparkConfiguration import org.apache.linkis.engineplugin.spark.config.SparkConfiguration.{ ENGINE_JAR, LINKIS_SPARK_CONF, SPARK_APP_NAME, SPARK_DEFAULT_EXTERNAL_JARS_PATH, + SPARK_DEPLOY_MODE, SPARK_DRIVER_CLASSPATH, SPARK_DRIVER_EXTRA_JAVA_OPTIONS, SPARK_PYTHON_VERSION, - SPARK_SUBMIT_PATH + SPARK_SUBMIT_PATH, + SPARK_YARN_CLUSTER_JARS } import org.apache.linkis.engineplugin.spark.config.SparkResourceConfiguration._ +import org.apache.linkis.engineplugin.spark.errorcode.SparkErrorCodeSummary +import org.apache.linkis.engineplugin.spark.exception.SparkEngineException import org.apache.linkis.hadoop.common.conf.HadoopConf import org.apache.linkis.manager.common.entity.resource.DriverAndYarnResource import org.apache.linkis.manager.engineplugin.common.conf.EnvConfiguration import org.apache.linkis.manager.engineplugin.common.launch.entity.EngineConnBuildRequest import org.apache.linkis.manager.engineplugin.common.launch.process.Environment._ import org.apache.linkis.manager.engineplugin.common.launch.process.JavaProcessEngineConnLaunchBuilder +import org.apache.linkis.manager.label.constant.LabelValueConstant import org.apache.linkis.manager.label.entity.engine.UserCreatorLabel +import org.apache.linkis.manager.label.utils.LabelUtil import org.apache.linkis.protocol.UserWithCreator import org.apache.commons.lang3.StringUtils @@ -81,7 +88,11 @@ class SparkSubmitProcessEngineConnLaunchBuilder(builder: JavaProcessEngineConnLa val userEngineResource = engineConnBuildRequest.engineResource val darResource = userEngineResource.getLockedResource.asInstanceOf[DriverAndYarnResource] - val files = getValueAndRemove(properties, "files", "").split(",").filter(isNotBlankPath) + val files: ArrayBuffer[String] = getValueAndRemove(properties, "files", "") + .split(",") + .filter(isNotBlankPath) + .toBuffer + .asInstanceOf[ArrayBuffer[String]] val jars = new ArrayBuffer[String]() jars ++= getValueAndRemove(properties, "jars", "").split(",").filter(isNotBlankPath) jars ++= getValueAndRemove(properties, SPARK_DEFAULT_EXTERNAL_JARS_PATH) @@ -141,8 +152,34 @@ class SparkSubmitProcessEngineConnLaunchBuilder(builder: JavaProcessEngineConnLa memory } + var deployMode: String = SparkConfiguration.SPARK_YARN_CLIENT + + val label = LabelUtil.getEngingeConnRuntimeModeLabel(engineConnBuildRequest.labels) + val isYarnClusterMode: Boolean = + if (null != label && label.getModeValue.equals(LabelValueConstant.YARN_CLUSTER_VALUE)) true + else false + + if (isYarnClusterMode) { + deployMode = SparkConfiguration.SPARK_YARN_CLUSTER + files ++= Array(s"${variable(PWD)}/conf/linkis-engineconn.properties") + + var clusterJars: String = getValueAndRemove(properties, SPARK_YARN_CLUSTER_JARS) + + if (StringUtils.isBlank(clusterJars)) { + throw new SparkEngineException( + SparkErrorCodeSummary.LINKIS_SPARK_YARN_CLUSTER_JARS_ERROR.getErrorCode, + SparkErrorCodeSummary.LINKIS_SPARK_YARN_CLUSTER_JARS_ERROR.getErrorDesc + ) + } + + if (clusterJars.endsWith("/")) { + clusterJars = clusterJars.dropRight(1) + } + jars += s"$clusterJars/*" + } + addOpt("--master", "yarn") - addOpt("--deploy-mode", "client") + addOpt("--deploy-mode", deployMode) addOpt("--name", appName) addProxyUser() @@ -163,8 +200,9 @@ class SparkSubmitProcessEngineConnLaunchBuilder(builder: JavaProcessEngineConnLa addOpt("--num-executors", numExecutors.toString) addOpt("--queue", queue) - getConf(engineConnBuildRequest, gcLogDir, logDir).foreach { case (key, value) => - addOpt("--conf", s"""$key="$value"""") + getConf(engineConnBuildRequest, gcLogDir, logDir, isYarnClusterMode).foreach { + case (key, value) => + addOpt("--conf", s"""$key="$value"""") } addOpt("--class", className) @@ -178,7 +216,8 @@ class SparkSubmitProcessEngineConnLaunchBuilder(builder: JavaProcessEngineConnLa def getConf( engineConnBuildRequest: EngineConnBuildRequest, gcLogDir: String, - logDir: String + logDir: String, + isYarnClusterMode: Boolean ): ArrayBuffer[(String, String)] = { val driverJavaSet = new StringBuilder(" -server") if (StringUtils.isNotEmpty(EnvConfiguration.ENGINE_CONN_DEFAULT_JAVA_OPTS.getValue)) { @@ -194,7 +233,11 @@ class SparkSubmitProcessEngineConnLaunchBuilder(builder: JavaProcessEngineConnLa .foreach(l => { driverJavaSet.append(" ").append(l) }) - driverJavaSet.append(" -Djava.io.tmpdir=" + variable(TEMP_DIRS)) + if (isYarnClusterMode) { + driverJavaSet.append(" -Djava.io.tmpdir=/tmp") + } else { + driverJavaSet.append(" -Djava.io.tmpdir=" + variable(TEMP_DIRS)) + } if (EnvConfiguration.ENGINE_CONN_DEBUG_ENABLE.getValue) { driverJavaSet.append( s" -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=${variable(RANDOM_PORT)}" @@ -212,6 +255,7 @@ class SparkSubmitProcessEngineConnLaunchBuilder(builder: JavaProcessEngineConnLa val keyValue = iterator.next() if ( !SPARK_PYTHON_VERSION.key.equals(keyValue.getKey) && + !SPARK_DEPLOY_MODE.key.equals(keyValue.getKey) && keyValue.getKey.startsWith("spark.") && StringUtils.isNotBlank(keyValue.getValue) ) { diff --git a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/utils/SparkJobProgressUtil.scala b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/utils/SparkJobProgressUtil.scala index 196414420a..94614f902b 100644 --- a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/utils/SparkJobProgressUtil.scala +++ b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/utils/SparkJobProgressUtil.scala @@ -27,11 +27,16 @@ import org.apache.http.client.methods.HttpGet import org.apache.http.impl.client.HttpClients import org.apache.http.util.EntityUtils +import java.util + object SparkJobProgressUtil extends Logging { - def getProgress(applicationId: String): Float = { + def getProgress(applicationId: String, sparkUIUrl: String = ""): Float = { if (StringUtils.isBlank(applicationId)) return 0f - val sparkJobsResult = getSparkJobInfo(applicationId) + val sparkJobsResult = + if (StringUtils.isBlank(sparkUIUrl)) + getSparkJobInfo(applicationId) + else getKubernetesSparkJobInfo(applicationId, sparkUIUrl) if (sparkJobsResult.isEmpty) return 0f val tuple = sparkJobsResult .filter(sparkJobResult => { @@ -48,8 +53,14 @@ object SparkJobProgressUtil extends Logging { tuple._2.toFloat / tuple._1 } - def getSparkJobProgressInfo(applicationId: String): Array[JobProgressInfo] = { - val sparkJobsResult = getSparkJobInfo(applicationId) + def getSparkJobProgressInfo( + applicationId: String, + sparkUIUrl: String = "" + ): Array[JobProgressInfo] = { + val sparkJobsResult = + if (StringUtils.isBlank(sparkUIUrl)) + getSparkJobInfo(applicationId) + else getKubernetesSparkJobInfo(applicationId, sparkUIUrl) if (sparkJobsResult.isEmpty) { Array.empty } else { @@ -96,6 +107,38 @@ object SparkJobProgressUtil extends Logging { ) } + def getKubernetesSparkJobInfo( + applicationId: String, + sparkUIUrl: String + ): Array[java.util.Map[String, Object]] = + if (StringUtils.isBlank(applicationId) || StringUtils.isBlank(sparkUIUrl)) Array.empty + else { + val getSparkJobsStateUrl = s"http://$sparkUIUrl/api/v1/applications/$applicationId" + logger.info(s"get spark job state from kubernetes spark ui, url: $getSparkJobsStateUrl") + val appStateResult = + JsonUtils.jackson.readValue( + get(getSparkJobsStateUrl), + classOf[java.util.Map[String, Object]] + ) + val appAttemptList = appStateResult.get("attempts").asInstanceOf[java.util.List[Object]] + if (appAttemptList == null || appAttemptList.size() == 0) return Array.empty + val appLastAttempt = + appAttemptList.get(appAttemptList.size() - 1).asInstanceOf[util.Map[String, Object]] + val isLastAttemptCompleted = appLastAttempt.get("completed").asInstanceOf[Boolean] + if (isLastAttemptCompleted) return Array.empty + val getSparkJobsInfoUrl = + s"http://$sparkUIUrl/api/v1/applications/$applicationId/jobs" + logger.info(s"get spark job info from kubernetes spark ui: $getSparkJobsInfoUrl") + val jobs = get(getSparkJobsInfoUrl) + if (StringUtils.isBlank(jobs)) { + return Array.empty + } + JsonUtils.jackson.readValue( + get(getSparkJobsInfoUrl), + classOf[Array[java.util.Map[String, Object]]] + ) + } + def get(url: String): String = { val httpGet = new HttpGet(url) val client = HttpClients.createDefault diff --git a/linkis-hadoop-hdfs-client-shade/pom.xml b/linkis-hadoop-hdfs-client-shade/pom.xml index 560bbaa698..e4990f857e 100644 --- a/linkis-hadoop-hdfs-client-shade/pom.xml +++ b/linkis-hadoop-hdfs-client-shade/pom.xml @@ -207,7 +207,7 @@ org.apache.maven.plugins maven-shade-plugin - ${maven-shade-plugin.version} + 3.3.0 true false diff --git a/linkis-public-enhancements/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/dao/JobHistoryMapper.java b/linkis-public-enhancements/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/dao/JobHistoryMapper.java index ebd7cdf31b..6edbfb3783 100644 --- a/linkis-public-enhancements/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/dao/JobHistoryMapper.java +++ b/linkis-public-enhancements/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/dao/JobHistoryMapper.java @@ -27,6 +27,7 @@ import java.util.Date; import java.util.List; +import java.util.Map; public interface JobHistoryMapper { @@ -129,4 +130,50 @@ List selectJobHistoryByTaskidList( @Param("idList") List idList, @Param("umUser") String username); List selectJobHistoryNoCode(JobHistory jobReq); + + /** + * query wait for failover job + * + *

Sql example: SELECT a.* FROM linkis_ps_job_history_group_history a where (a.instances = '' + * or a.instances is null or a.instances not in ('192.168.1.123:9104','192.168.1.124:9104') or + * EXISTS ( select 1 from ( select '192.168.1.123:9104' as instances, 1697775054098 as + * registryTime union all select '192.168.1.124:9104' as instances, 1666239054098 as registryTime + * ) b where a.instances = b.instances and a.created_time < FROM_UNIXTIME(b.registryTime/1000) ) ) + * and status in ('Inited','Running','Scheduled','WaitForRetry') and a.created_time >= + * FROM_UNIXTIME(1666239054098/1000) limit 10 + * + * @param instancesMap + * @param statusList + * @param startTimestamp + * @param limit + * @return + */ + List selectFailoverJobHistory( + @Param("instancesMap") Map instancesMap, + @Param("statusList") List statusList, + @Param("startTimestamp") Long startTimestamp, + @Param("limit") Integer limit); + + List taskDurationTopN( + @Param("startDate") Date startDate, + @Param("endDate") Date endDate, + @Param("umUser") String username, + @Param("engineType") String engineType); + + List taskDurationTopNWithUserCreator( + @Param("umUser") String username, + @Param("userCreatorKey") String userCreatorKey, + @Param("userCreatorValue") String userCreator, + @Param("startDate") Date startDate, + @Param("endDate") Date endDate, + @Param("engineType") String engineType); + + List taskDurationTopNWithCreatorOnly( + @Param("umUser") String username, + @Param("userCreatorKey") String userCreatorKey, + @Param("creator") String userCreator, + @Param("startDate") Date startDate, + @Param("endDate") Date endDate, + @Param("engineType") String engineType); + } diff --git a/linkis-public-enhancements/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/dao/JobStatisticsMapper.java b/linkis-public-enhancements/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/dao/JobStatisticsMapper.java new file mode 100644 index 0000000000..9c0ee86d9e --- /dev/null +++ b/linkis-public-enhancements/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/dao/JobStatisticsMapper.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.jobhistory.dao; + +import org.apache.linkis.jobhistory.entity.JobStatistics; + +import org.apache.ibatis.annotations.Param; + +import java.util.Date; + +public interface JobStatisticsMapper { + + JobStatistics taskExecutionStatistics( + @Param("umUser") String username, + @Param("startDate") Date startDate, + @Param("endDate") Date endDate, + @Param("engineType") String engineType); + + JobStatistics taskExecutionStatisticsWithUserCreator( + @Param("umUser") String username, + @Param("userCreatorKey") String userCreatorKey, + @Param("userCreatorValue") String userCreator, + @Param("startDate") Date startDate, + @Param("endDate") Date endDate, + @Param("engineType") String engineType); + + JobStatistics taskExecutionStatisticsWithCreatorOnly( + @Param("umUser") String username, + @Param("userCreatorKey") String userCreatorKey, + @Param("creator") String userCreator, + @Param("startDate") Date startDate, + @Param("endDate") Date endDate, + @Param("engineType") String engineType); + + JobStatistics engineExecutionStatisticsWithUserCreator( + @Param("umUser") String username, + @Param("userCreatorValue") String userCreator, + @Param("startDate") Date startDate, + @Param("endDate") Date endDate, + @Param("engineType") String engineType); + + JobStatistics engineExecutionStatistics( + @Param("umUser") String username, + @Param("creator") String userCreator, + @Param("startDate") Date startDate, + @Param("endDate") Date endDate, + @Param("engineType") String engineType); +} diff --git a/linkis-public-enhancements/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/entity/JobStatistics.java b/linkis-public-enhancements/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/entity/JobStatistics.java new file mode 100644 index 0000000000..23621708f5 --- /dev/null +++ b/linkis-public-enhancements/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/entity/JobStatistics.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.jobhistory.entity; + +public class JobStatistics { + + private Long id; + + private Integer sumCount; + + private Integer succeedCount; + + private Integer failedCount; + + private Integer cancelledCount; + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public Integer getSumCount() { + return sumCount; + } + + public void setSumCount(Integer sumCount) { + this.sumCount = sumCount; + } + + public Integer getSucceedCount() { + return succeedCount; + } + + public void setSucceedCount(Integer succeedCount) { + this.succeedCount = succeedCount; + } + + public Integer getFailedCount() { + return failedCount; + } + + public void setFailedCount(Integer failedCount) { + this.failedCount = failedCount; + } + + public Integer getCancelledCount() { + return cancelledCount; + } + + public void setCancelledCount(Integer cancelledCount) { + this.cancelledCount = cancelledCount; + } + + @Override + public String toString() { + return "JobHistory{" + "id=" + id + '}'; + } +} diff --git a/linkis-public-enhancements/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/restful/api/QueryRestfulApi.java b/linkis-public-enhancements/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/restful/api/QueryRestfulApi.java index b4f6362e06..8f46189248 100644 --- a/linkis-public-enhancements/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/restful/api/QueryRestfulApi.java +++ b/linkis-public-enhancements/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/restful/api/QueryRestfulApi.java @@ -554,6 +554,95 @@ public void downloadJobList( } } + /** Method list should not contain subjob, which may cause performance problems. */ + @ApiOperation(value = "listDurationTop", notes = "listDurationTop", response = Message.class) + @ApiImplicitParams({ + @ApiImplicitParam(name = "startDate", dataType = "long", example = "1658937600001"), + @ApiImplicitParam(name = "endDate", dataType = "long", example = "1658937600000"), + @ApiImplicitParam(name = "executeApplicationName", dataType = "String"), + @ApiImplicitParam(name = "creator", required = false, dataType = "String", value = "creator"), + @ApiImplicitParam( + name = "proxyUser", + required = false, + dataType = "String", + value = "proxyUser"), + @ApiImplicitParam(name = "pageNow", required = false, dataType = "Integer", value = "page now"), + @ApiImplicitParam(name = "pageSize", dataType = "Integer"), + }) + @RequestMapping(path = "/listDurationTop", method = RequestMethod.GET) + public Message listDurationTop( + HttpServletRequest req, + @RequestParam(value = "startDate", required = false) Long startDate, + @RequestParam(value = "endDate", required = false) Long endDate, + @RequestParam(value = "executeApplicationName", required = false) + String executeApplicationName, + @RequestParam(value = "creator", required = false) String creator, + @RequestParam(value = "proxyUser", required = false) String proxyUser, + @RequestParam(value = "pageNow", required = false) Integer pageNow, + @RequestParam(value = "pageSize", required = false) Integer pageSize) + throws QueryException { + if (org.springframework.util.StringUtils.isEmpty(pageNow)) { + pageNow = 1; + } + if (org.springframework.util.StringUtils.isEmpty(pageSize)) { + pageSize = 20; + } + if (org.springframework.util.StringUtils.isEmpty(proxyUser)) { + proxyUser = null; + } else { + if (!QueryUtils.checkNameValid(proxyUser)) { + return Message.error("Invalid proxyUser : " + proxyUser); + } + } + if (org.springframework.util.StringUtils.isEmpty(creator)) { + creator = null; + } else { + if (!QueryUtils.checkNameValid(creator)) { + return Message.error("Invalid creator : " + creator); + } + } + if (!org.springframework.util.StringUtils.isEmpty(executeApplicationName)) { + if (!QueryUtils.checkNameValid(executeApplicationName)) { + return Message.error("Invalid applicationName : " + executeApplicationName); + } + } else { + executeApplicationName = null; + } + + if (endDate == null) { + endDate = System.currentTimeMillis(); + } + if (startDate == null) { + startDate = 0L; + } + + Date sDate = new Date(startDate); + Date eDate = new Date(endDate); + if (sDate.getTime() == eDate.getTime()) { + Calendar calendar = Calendar.getInstance(); + calendar.setTimeInMillis(endDate); + calendar.add(Calendar.DAY_OF_MONTH, 1); + eDate = new Date(calendar.getTime().getTime()); // todo check + } + List queryTasks = null; + PageHelper.startPage(pageNow, pageSize); + try { + queryTasks = + jobHistoryQueryService.taskDurationTopN( + sDate, eDate, proxyUser, creator, executeApplicationName); + } finally { + PageHelper.clearPage(); + } + + List vos = new ArrayList<>(); + for (JobHistory jobHistory : queryTasks) { + QueryUtils.exchangeExecutionCode(jobHistory); + QueryTaskVO taskVO = TaskConversions.jobHistory2TaskVO(jobHistory, null); + vos.add(taskVO); + } + return Message.ok().data(TaskConstant.TASKS, vos); + } + private List getJobhistoryList( HttpServletRequest req, Long startDate, diff --git a/linkis-public-enhancements/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/restful/api/StatisticsRestfulApi.java b/linkis-public-enhancements/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/restful/api/StatisticsRestfulApi.java new file mode 100644 index 0000000000..23a12c2b94 --- /dev/null +++ b/linkis-public-enhancements/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/restful/api/StatisticsRestfulApi.java @@ -0,0 +1,195 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.jobhistory.restful.api; + +import org.apache.linkis.governance.common.entity.job.QueryException; +import org.apache.linkis.jobhistory.entity.JobStatistics; +import org.apache.linkis.jobhistory.service.JobStatisticsQueryService; +import org.apache.linkis.jobhistory.util.QueryUtils; +import org.apache.linkis.server.Message; + +import org.apache.commons.lang3.time.DateUtils; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.util.StringUtils; +import org.springframework.web.bind.annotation.*; + +import javax.servlet.http.HttpServletRequest; + +import java.io.IOException; +import java.util.Calendar; +import java.util.Date; + +import io.swagger.annotations.Api; +import io.swagger.annotations.ApiImplicitParam; +import io.swagger.annotations.ApiImplicitParams; +import io.swagger.annotations.ApiOperation; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@Api(tags = "jobstatistics api") +@RestController +@RequestMapping(path = "/jobhistory/jobstatistics") +public class StatisticsRestfulApi { + + private Logger log = LoggerFactory.getLogger(this.getClass()); + + @Autowired private JobStatisticsQueryService jobStatisticsQueryService; + + @ApiOperation(value = "taskCount", notes = "taskCount", response = Message.class) + @ApiImplicitParams({ + @ApiImplicitParam(name = "startDate", dataType = "long"), + @ApiImplicitParam(name = "endDate", required = false, dataType = "long", value = "end date"), + @ApiImplicitParam(name = "executeApplicationName", dataType = "String"), + @ApiImplicitParam(name = "creator", required = false, dataType = "String", value = "creator"), + @ApiImplicitParam( + name = "proxyUser", + required = false, + dataType = "String", + value = "proxyUser"), + }) + @RequestMapping(path = "/taskCount", method = RequestMethod.GET) + public Message taskCount( + HttpServletRequest req, + @RequestParam(value = "startDate", required = false) Long startDate, + @RequestParam(value = "endDate", required = false) Long endDate, + @RequestParam(value = "executeApplicationName", required = false) + String executeApplicationName, + @RequestParam(value = "creator", required = false) String creator, + @RequestParam(value = "proxyUser", required = false) String proxyUser) + throws IOException, QueryException { + if (endDate == null) { + endDate = System.currentTimeMillis(); + } + if (startDate == null) { + startDate = 0L; + } + Date sDate = new Date(startDate); + Date eDate = new Date(endDate); + if (startDate == 0L) { + sDate = DateUtils.addDays(eDate, -1); + } + if (sDate.getTime() == eDate.getTime()) { + Calendar instance = Calendar.getInstance(); + instance.setTimeInMillis(endDate); + instance.add(Calendar.DAY_OF_MONTH, 1); + eDate = new Date(instance.getTime().getTime()); + } + if (StringUtils.isEmpty(proxyUser)) { + proxyUser = null; + } else { + if (!QueryUtils.checkNameValid(proxyUser)) { + return Message.error("Invalid proxyUser : " + proxyUser); + } + } + if (StringUtils.isEmpty(creator)) { + creator = null; + } else { + if (!QueryUtils.checkNameValid(creator)) { + return Message.error("Invalid creator : " + creator); + } + } + if (!StringUtils.isEmpty(executeApplicationName)) { + if (!QueryUtils.checkNameValid(executeApplicationName)) { + return Message.error("Invalid applicationName : " + executeApplicationName); + } + } else { + executeApplicationName = null; + } + JobStatistics jobStatistics = + jobStatisticsQueryService.taskExecutionStatistics( + sDate, eDate, proxyUser, creator, executeApplicationName); + + return Message.ok() + .data("sumCount", jobStatistics.getSumCount()) + .data("succeedCount", jobStatistics.getSucceedCount()) + .data("failedCount", jobStatistics.getFailedCount()) + .data("cancelledCount", jobStatistics.getCancelledCount()); + } + + @ApiOperation(value = "engineCount", notes = "engineCount", response = Message.class) + @ApiImplicitParams({ + @ApiImplicitParam(name = "startDate", dataType = "long"), + @ApiImplicitParam(name = "endDate", required = false, dataType = "long", value = "end date"), + @ApiImplicitParam(name = "executeApplicationName", dataType = "String"), + @ApiImplicitParam(name = "creator", required = false, dataType = "String", value = "creator"), + @ApiImplicitParam( + name = "proxyUser", + required = false, + dataType = "String", + value = "proxyUser"), + }) + @RequestMapping(path = "/engineCount", method = RequestMethod.GET) + public Message engineCount( + HttpServletRequest req, + @RequestParam(value = "startDate", required = false) Long startDate, + @RequestParam(value = "endDate", required = false) Long endDate, + @RequestParam(value = "executeApplicationName", required = false) + String executeApplicationName, + @RequestParam(value = "creator", required = false) String creator, + @RequestParam(value = "proxyUser", required = false) String proxyUser) + throws IOException, QueryException { + if (endDate == null) { + endDate = System.currentTimeMillis(); + } + if (startDate == null) { + startDate = 0L; + } + Date sDate = new Date(startDate); + Date eDate = new Date(endDate); + if (startDate == 0L) { + sDate = DateUtils.addDays(eDate, -1); + } + if (sDate.getTime() == eDate.getTime()) { + Calendar instance = Calendar.getInstance(); + instance.setTimeInMillis(endDate); + instance.add(Calendar.DAY_OF_MONTH, 1); + eDate = new Date(instance.getTime().getTime()); + } + if (StringUtils.isEmpty(proxyUser)) { + proxyUser = null; + } else { + if (!QueryUtils.checkNameValid(proxyUser)) { + return Message.error("Invalid proxyUser : " + proxyUser); + } + } + if (StringUtils.isEmpty(creator)) { + creator = null; + } else { + if (!QueryUtils.checkNameValid(creator)) { + return Message.error("Invalid creator : " + creator); + } + } + if (!StringUtils.isEmpty(executeApplicationName)) { + if (!QueryUtils.checkNameValid(executeApplicationName)) { + return Message.error("Invalid applicationName : " + executeApplicationName); + } + } else { + executeApplicationName = null; + } + JobStatistics jobStatistics = + jobStatisticsQueryService.engineExecutionStatistics( + sDate, eDate, proxyUser, creator, executeApplicationName); + + return Message.ok() + .data("countEngine", jobStatistics.getSumCount()) + .data("countEngineSucceed", jobStatistics.getSucceedCount()) + .data("countEngineFailed", jobStatistics.getFailedCount()) + .data("countEngineShutting", jobStatistics.getCancelledCount()); + } +} diff --git a/linkis-public-enhancements/linkis-jobhistory/src/main/resources/mapper/common/JobStatisticsMapper.xml b/linkis-public-enhancements/linkis-jobhistory/src/main/resources/mapper/common/JobStatisticsMapper.xml new file mode 100644 index 0000000000..809f82e3fc --- /dev/null +++ b/linkis-public-enhancements/linkis-jobhistory/src/main/resources/mapper/common/JobStatisticsMapper.xml @@ -0,0 +1,52 @@ + + + + + + + + + + + + + + + + + + + + + diff --git a/linkis-public-enhancements/linkis-jobhistory/src/main/resources/mapper/common/JobHistoryMapper.xml b/linkis-public-enhancements/linkis-jobhistory/src/main/resources/mapper/mysql/JobHistoryMapper.xml similarity index 82% rename from linkis-public-enhancements/linkis-jobhistory/src/main/resources/mapper/common/JobHistoryMapper.xml rename to linkis-public-enhancements/linkis-jobhistory/src/main/resources/mapper/mysql/JobHistoryMapper.xml index 524f6905f6..74cf9057f5 100644 --- a/linkis-public-enhancements/linkis-jobhistory/src/main/resources/mapper/common/JobHistoryMapper.xml +++ b/linkis-public-enhancements/linkis-jobhistory/src/main/resources/mapper/mysql/JobHistoryMapper.xml @@ -250,7 +250,9 @@ - update linkis_ps_job_history_group_history set observe_info = #{observeInfo} where id = #{taskid} + UPDATE linkis_ps_job_history_group_history + SET observe_info = #{observeInfo} + WHERE id = #{taskid} @@ -270,4 +272,65 @@ ORDER BY linkis_ps_job_history_group_history.created_time DESC + + + + + + + + diff --git a/linkis-public-enhancements/linkis-jobhistory/src/main/resources/mapper/mysql/JobStatisticsMapper.xml b/linkis-public-enhancements/linkis-jobhistory/src/main/resources/mapper/mysql/JobStatisticsMapper.xml new file mode 100644 index 0000000000..d05728faac --- /dev/null +++ b/linkis-public-enhancements/linkis-jobhistory/src/main/resources/mapper/mysql/JobStatisticsMapper.xml @@ -0,0 +1,86 @@ + + + + + + + + + + + + + + + + + + diff --git a/linkis-public-enhancements/linkis-jobhistory/src/main/resources/mapper/postgresql/JobHistoryMapper.xml b/linkis-public-enhancements/linkis-jobhistory/src/main/resources/mapper/postgresql/JobHistoryMapper.xml new file mode 100644 index 0000000000..0289141e89 --- /dev/null +++ b/linkis-public-enhancements/linkis-jobhistory/src/main/resources/mapper/postgresql/JobHistoryMapper.xml @@ -0,0 +1,293 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + "id", "job_req_id", "submit_user", "execute_user", "source", "labels", "params", + "progress", "status", "log_path", "error_code", "error_desc", "created_time", "updated_time", + "instances", "metrics","engine_type", "execution_code", "result_location", "observe_info" + + + + "job_req_id", "submit_user", "execute_user", "source", "labels", "params", + "progress", "status", "log_path", "error_code", "error_desc", "created_time", "updated_time", "instances", "metrics","engine_type", "execution_code", "observe_info" + + + + INSERT INTO linkis_ps_job_history_group_history() + VALUES (#{jobReqId},#{submitUser},#{executeUser},#{source}, + #{labels},#{params},#{progress}, + #{status},#{logPath},#{errorCode},#{errorDesc}, + #{createdTime},#{updateTimeMills},#{instances},#{metrics},#{engineType},#{executionCode},#{observeInfo}) + + + + + + + + + + + + + + + + UPDATE linkis_ps_job_history_group_history + + updated_time = #{updateTimeMills}, + job_req_id = #{jobReqId}, + submit_user = #{submitUser}, + execute_user = #{executeUser}, + source = #{source}, + labels = #{labels}, + params = #{params}, + progress = #{progress}, + status = #{status}, + log_path = #{logPath}, + result_location=#{resultLocation}, + error_code = #{errorCode}, + error_desc = #{errorDesc}, + created_time = #{createdTime}, + instances = #{instances}, + metrics = #{metrics}, + engine_type = #{engineType}, + execution_code = #{executionCode}, + observe_info = #{observeInfo}, + + + WHERE id = #{id} AND updated_time #{updateTimeMills} + + + + + + + + + + + + + update linkis_ps_job_history_group_history set observe_info = #{observeInfo} where id = #{taskid} + + + + update linkis_ps_job_history_group_history set status = 'Cancelled' ,error_desc = #{errorDesc} + WHERE id IN + + #{id} + + + + + + + + + + + diff --git a/linkis-public-enhancements/linkis-jobhistory/src/main/resources/mapper/postgresql/JobStatisticsMapper.xml b/linkis-public-enhancements/linkis-jobhistory/src/main/resources/mapper/postgresql/JobStatisticsMapper.xml new file mode 100644 index 0000000000..f1dd6e0887 --- /dev/null +++ b/linkis-public-enhancements/linkis-jobhistory/src/main/resources/mapper/postgresql/JobStatisticsMapper.xml @@ -0,0 +1,86 @@ + + + + + + + + + + + + + + + + + + diff --git a/linkis-public-enhancements/linkis-jobhistory/src/main/scala/org/apache/linkis/jobhistory/service/JobHistoryQueryService.java b/linkis-public-enhancements/linkis-jobhistory/src/main/scala/org/apache/linkis/jobhistory/service/JobHistoryQueryService.java index 158457d79a..e753cb0cfe 100644 --- a/linkis-public-enhancements/linkis-jobhistory/src/main/scala/org/apache/linkis/jobhistory/service/JobHistoryQueryService.java +++ b/linkis-public-enhancements/linkis-jobhistory/src/main/scala/org/apache/linkis/jobhistory/service/JobHistoryQueryService.java @@ -55,4 +55,6 @@ public interface JobHistoryQueryService { void clearUndoneTasksByEntranceInstance(EntranceInstanceConfRequest request, Sender sender); List searchByTasks(List taskidList, String username); + + } diff --git a/linkis-public-enhancements/linkis-jobhistory/src/main/scala/org/apache/linkis/jobhistory/service/JobStatisticsQueryService.java b/linkis-public-enhancements/linkis-jobhistory/src/main/scala/org/apache/linkis/jobhistory/service/JobStatisticsQueryService.java new file mode 100644 index 0000000000..5d11b623ad --- /dev/null +++ b/linkis-public-enhancements/linkis-jobhistory/src/main/scala/org/apache/linkis/jobhistory/service/JobStatisticsQueryService.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.jobhistory.service; + +import org.apache.linkis.jobhistory.entity.JobStatistics; + +import java.util.Date; + + +public interface JobStatisticsQueryService { + + JobStatistics taskExecutionStatistics(Date startDate, Date endDate, String username, String creator, String engineType); + + JobStatistics engineExecutionStatistics(Date startDate, Date endDate, String username, String creator, String engineType); +} diff --git a/linkis-public-enhancements/linkis-jobhistory/src/main/scala/org/apache/linkis/jobhistory/service/impl/JobHistoryQueryServiceImpl.scala b/linkis-public-enhancements/linkis-jobhistory/src/main/scala/org/apache/linkis/jobhistory/service/impl/JobHistoryQueryServiceImpl.scala index 470c40b83f..23472dd1c8 100644 --- a/linkis-public-enhancements/linkis-jobhistory/src/main/scala/org/apache/linkis/jobhistory/service/impl/JobHistoryQueryServiceImpl.scala +++ b/linkis-public-enhancements/linkis-jobhistory/src/main/scala/org/apache/linkis/jobhistory/service/impl/JobHistoryQueryServiceImpl.scala @@ -500,4 +500,44 @@ class JobHistoryQueryServiceImpl extends JobHistoryQueryService with Logging { jobHistoryMapper.selectJobHistoryByTaskidList(taskidList, username) } + override def taskDurationTopN( + sDate: Date, + eDate: Date, + username: String, + creator: String, + engineType: String + ): util.List[JobHistory] = { + val result = if (StringUtils.isBlank(creator)) { + jobHistoryMapper.taskDurationTopN(sDate, eDate, username, engineType) + } else if (StringUtils.isBlank(username)) { + val fakeLabel = new UserCreatorLabel + jobHistoryMapper.taskDurationTopNWithCreatorOnly( + username, + fakeLabel.getLabelKey, + creator, + sDate, + eDate, + engineType + ) + } else { + val fakeLabel = new UserCreatorLabel + fakeLabel.setUser(username) + fakeLabel.setCreator(creator) + val userCreator = fakeLabel.getStringValue + Utils.tryCatch(fakeLabel.valueCheck(userCreator)) { t => + logger.info("input user or creator is not correct", t) + throw t + } + jobHistoryMapper.taskDurationTopNWithUserCreator( + username, + fakeLabel.getLabelKey, + userCreator, + sDate, + eDate, + engineType + ) + } + result + } + } diff --git a/linkis-public-enhancements/linkis-jobhistory/src/main/scala/org/apache/linkis/jobhistory/service/impl/JobStatisticsQueryServiceImpl.scala b/linkis-public-enhancements/linkis-jobhistory/src/main/scala/org/apache/linkis/jobhistory/service/impl/JobStatisticsQueryServiceImpl.scala new file mode 100644 index 0000000000..27b8173d05 --- /dev/null +++ b/linkis-public-enhancements/linkis-jobhistory/src/main/scala/org/apache/linkis/jobhistory/service/impl/JobStatisticsQueryServiceImpl.scala @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.jobhistory.service.impl + +import org.apache.linkis.common.utils.{Logging, Utils} +import org.apache.linkis.jobhistory.dao.JobStatisticsMapper +import org.apache.linkis.jobhistory.entity.JobStatistics +import org.apache.linkis.jobhistory.service.JobStatisticsQueryService +import org.apache.linkis.manager.label.entity.engine.UserCreatorLabel + +import org.apache.commons.lang3.StringUtils + +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.stereotype.Service + +import java.util.Date + +@Service +class JobStatisticsQueryServiceImpl extends JobStatisticsQueryService with Logging { + + @Autowired + private var jobStatisticsMapper: JobStatisticsMapper = _ + + override def taskExecutionStatistics( + sDate: Date, + eDate: Date, + username: String, + creator: String, + engineType: String + ): JobStatistics = { + val result = if (StringUtils.isBlank(creator)) { + jobStatisticsMapper.taskExecutionStatistics(username, sDate, eDate, engineType) + } else if (StringUtils.isBlank(username)) { + val fakeLabel = new UserCreatorLabel + jobStatisticsMapper.taskExecutionStatisticsWithCreatorOnly( + username, + fakeLabel.getLabelKey, + creator, + sDate, + eDate, + engineType + ) + } else { + val fakeLabel = new UserCreatorLabel + fakeLabel.setUser(username) + fakeLabel.setCreator(creator) + val userCreator = fakeLabel.getStringValue + Utils.tryCatch(fakeLabel.valueCheck(userCreator)) { t => + logger.info("input user or creator is not correct", t) + throw t + } + jobStatisticsMapper.taskExecutionStatisticsWithUserCreator( + username, + fakeLabel.getLabelKey, + userCreator, + sDate, + eDate, + engineType + ) + } + result + } + + override def engineExecutionStatistics( + sDate: Date, + eDate: Date, + username: String, + creator: String, + engineType: String + ): JobStatistics = { + val result = if (StringUtils.isBlank(username) || StringUtils.isBlank(creator)) { + jobStatisticsMapper.engineExecutionStatistics(username, creator, sDate, eDate, engineType) + } else { + val fakeLabel = new UserCreatorLabel + fakeLabel.setUser(username) + fakeLabel.setCreator(creator) + val userCreator = fakeLabel.getStringValue + Utils.tryCatch(fakeLabel.valueCheck(userCreator)) { t => + logger.info("input user or creator is not correct", t) + throw t + } + jobStatisticsMapper.engineExecutionStatisticsWithUserCreator( + username, + userCreator, + sDate, + eDate, + engineType + ) + } + result + } + +} diff --git a/linkis-public-enhancements/linkis-pes-common/src/main/java/org/apache/linkis/metadata/query/common/cache/CacheConfiguration.java b/linkis-public-enhancements/linkis-pes-common/src/main/java/org/apache/linkis/metadata/query/common/cache/CacheConfiguration.java index cc17a28bbf..bfff0fe92e 100644 --- a/linkis-public-enhancements/linkis-pes-common/src/main/java/org/apache/linkis/metadata/query/common/cache/CacheConfiguration.java +++ b/linkis-public-enhancements/linkis-pes-common/src/main/java/org/apache/linkis/metadata/query/common/cache/CacheConfiguration.java @@ -34,5 +34,10 @@ public class CacheConfiguration { public static final CommonVars MYSQL_RELATIONSHIP_LIST = CommonVars.apply( "wds.linkis.server.mdq.mysql.relationship", - "mysql,oracle,kingbase,postgresql,sqlserver,db2,greenplum,dm,mysql,doris,clickhouse,starrocks"); + "mysql,oracle,kingbase,postgresql,sqlserver,db2,greenplum,dm,doris,clickhouse,tidb,starrocks,gaussdb,oceanbase"); + + public static final CommonVars QUERY_DATABASE_RELATIONSHIP = + CommonVars.apply( + "linkis.server.mdq.query.database.relationship", + "{\"tidb\":\"mysql\",\"doris\":\"mysql\",\"starrocks\":\"mysql\",\"oceanbase\":\"mysql\",\"gaussdb\":\"postgresql\"}"); } diff --git a/linkis-public-enhancements/linkis-pes-common/src/main/java/org/apache/linkis/metadata/query/common/domain/GenerateSqlInfo.java b/linkis-public-enhancements/linkis-pes-common/src/main/java/org/apache/linkis/metadata/query/common/domain/GenerateSqlInfo.java new file mode 100644 index 0000000000..d5a7e0250a --- /dev/null +++ b/linkis-public-enhancements/linkis-pes-common/src/main/java/org/apache/linkis/metadata/query/common/domain/GenerateSqlInfo.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.metadata.query.common.domain; + +import java.io.Serializable; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; + +/** The meta information of field */ +@JsonSerialize(include = JsonSerialize.Inclusion.NON_EMPTY) +@JsonIgnoreProperties(ignoreUnknown = true) +public class GenerateSqlInfo implements Serializable { + + private String ddl; + private String dml; + private String dql; + + public String getDdl() { + return ddl; + } + + public void setDdl(String ddl) { + this.ddl = ddl; + } + + public String getDml() { + return dml; + } + + public void setDml(String dml) { + this.dml = dml; + } + + public String getDql() { + return dql; + } + + public void setDql(String dql) { + this.dql = dql; + } +} diff --git a/linkis-public-enhancements/linkis-pes-common/src/main/java/org/apache/linkis/metadata/query/common/service/AbstractDbMetaService.java b/linkis-public-enhancements/linkis-pes-common/src/main/java/org/apache/linkis/metadata/query/common/service/AbstractDbMetaService.java index cb3ed521c6..f2108eafd2 100644 --- a/linkis-public-enhancements/linkis-pes-common/src/main/java/org/apache/linkis/metadata/query/common/service/AbstractDbMetaService.java +++ b/linkis-public-enhancements/linkis-pes-common/src/main/java/org/apache/linkis/metadata/query/common/service/AbstractDbMetaService.java @@ -18,6 +18,7 @@ package org.apache.linkis.metadata.query.common.service; import org.apache.linkis.common.exception.WarnException; +import org.apache.linkis.metadata.query.common.domain.GenerateSqlInfo; import org.apache.linkis.metadata.query.common.domain.MetaColumnInfo; import org.apache.linkis.metadata.query.common.domain.MetaPartitionInfo; @@ -38,6 +39,17 @@ public List getDatabases(String operator, Map params) { return this.getConnAndRun(operator, params, this::queryDatabases); } + @Override + public String getSqlConnectUrl(String operator, Map params) { + return this.getConnAndRun(operator, params, this::querySqlConnectUrl); + } + + @Override + public GenerateSqlInfo getJdbcSql( + String operator, Map params, String database, String table) { + return this.getConnAndRun(operator, params, conn -> this.queryJdbcSql(conn, database, table)); + } + @Override public List getTables(String operator, Map params, String database) { return this.getConnAndRun(operator, params, conn -> this.queryTables(conn, database)); @@ -88,6 +100,28 @@ public List queryDatabases(C connection) { throw new WarnException(-1, "This method is no supported"); } + /** + * Get sql connect url + * + * @param connection metadata connection + * @return + */ + public String querySqlConnectUrl(C connection) { + throw new WarnException(-1, "This method is no supported"); + } + + /** + * Get jdbc sql + * + * @param connection metadata connection + * @param database database + * @param table table + * @return + */ + public GenerateSqlInfo queryJdbcSql(C connection, String database, String table) { + throw new WarnException(-1, "This method is no supported"); + } + /** * Get table list by connection and database * diff --git a/linkis-public-enhancements/linkis-pes-common/src/main/java/org/apache/linkis/metadata/query/common/service/GenerateSqlTemplate.java b/linkis-public-enhancements/linkis-pes-common/src/main/java/org/apache/linkis/metadata/query/common/service/GenerateSqlTemplate.java new file mode 100644 index 0000000000..c2fb4f218b --- /dev/null +++ b/linkis-public-enhancements/linkis-pes-common/src/main/java/org/apache/linkis/metadata/query/common/service/GenerateSqlTemplate.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.metadata.query.common.service; + +public class GenerateSqlTemplate { + + public static final String ES_DDL_SQL_TEMPLATE = + "CREATE TEMPORARY TABLE %s " + + "USING org.elasticsearch.spark.sql " + + "OPTIONS (" + + " 'es.nodes' '%s'," + + " 'es.port' '%s'," + + " 'es.resource' '%s/_doc'" + + ")"; + + public static final String JDBC_DDL_SQL_TEMPLATE = + "CREATE TEMPORARY TABLE %s " + + "USING org.apache.spark.sql.jdbc " + + "OPTIONS (" + + " url '%s'," + + " dbtable '%s'," + + " user '%s'," + + " password '%s'" + + ")"; + + public static final String KAFKA_DDL_SQL_TEMPLATE = + "CREATE TEMPORARY TABLE %s " + + "USING kafka " + + "OPTIONS (" + + " 'kafka.bootstrap.servers' '%s'," + + " 'subscribe' '%s'" + + ")"; + + public static final String MONGO_DDL_SQL_TEMPLATE = + "CREATE TEMPORARY TABLE %s " + + "USING mongo " + + "OPTIONS (" + + " 'spark.mongodb.input.uri' '%s'," + + " 'spark.mongodb.input.database' '%s'," + + " 'spark.mongodb.input.collection' '%s'" + + ")"; + public static final String DML_SQL_TEMPLATE = "INSERT INTO %s SELECT * FROM ${resultTable}"; + + public static final String DQL_SQL_TEMPLATE = "SELECT %s FROM %s"; + + public static String generateDqlSql(String columns, String table) { + return String.format(GenerateSqlTemplate.DQL_SQL_TEMPLATE, columns, table); + } + + public static String generateDmlSql(String table) { + return String.format(GenerateSqlTemplate.DML_SQL_TEMPLATE, table); + } +} diff --git a/linkis-public-enhancements/linkis-pes-common/src/main/java/org/apache/linkis/metadata/query/common/service/MetadataDbService.java b/linkis-public-enhancements/linkis-pes-common/src/main/java/org/apache/linkis/metadata/query/common/service/MetadataDbService.java index 4ef371c242..58ef7984c4 100644 --- a/linkis-public-enhancements/linkis-pes-common/src/main/java/org/apache/linkis/metadata/query/common/service/MetadataDbService.java +++ b/linkis-public-enhancements/linkis-pes-common/src/main/java/org/apache/linkis/metadata/query/common/service/MetadataDbService.java @@ -17,6 +17,7 @@ package org.apache.linkis.metadata.query.common.service; +import org.apache.linkis.metadata.query.common.domain.GenerateSqlInfo; import org.apache.linkis.metadata.query.common.domain.MetaColumnInfo; import org.apache.linkis.metadata.query.common.domain.MetaPartitionInfo; @@ -84,4 +85,23 @@ Map getPartitionProps( */ List getColumns( String operator, Map params, String database, String table); + + /** + * Get sql connect url + * + * @param params connect params + * @return + */ + public String getSqlConnectUrl(String operator, Map params); + + /** + * Get jdbc ddl sql + * + * @param params + * @param database + * @param table + * @return + */ + GenerateSqlInfo getJdbcSql( + String operator, Map params, String database, String table); } diff --git a/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/basedatamanager/server/restful/ConfigurationTemplateRestfulApi.java b/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/basedatamanager/server/restful/ConfigurationTemplateRestfulApi.java index d6db4f7571..657d4531f3 100644 --- a/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/basedatamanager/server/restful/ConfigurationTemplateRestfulApi.java +++ b/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/basedatamanager/server/restful/ConfigurationTemplateRestfulApi.java @@ -21,6 +21,7 @@ import org.apache.linkis.basedatamanager.server.request.ConfigurationTemplateSaveRequest; import org.apache.linkis.basedatamanager.server.response.EngineLabelResponse; import org.apache.linkis.basedatamanager.server.service.ConfigurationTemplateService; +import org.apache.linkis.common.conf.Configuration; import org.apache.linkis.server.Message; import org.apache.linkis.server.utils.ModuleUserUtils; @@ -49,7 +50,11 @@ public class ConfigurationTemplateRestfulApi { @RequestMapping(path = "/save", method = RequestMethod.POST) public Message add( HttpServletRequest httpRequest, @RequestBody ConfigurationTemplateSaveRequest request) { - ModuleUserUtils.getOperationUser(httpRequest, "save a configuration template"); + String username = + ModuleUserUtils.getOperationUser(httpRequest, "save a configuration template"); + if (!Configuration.isAdmin(username)) { + return Message.error("User '" + username + "' is not admin user[非管理员用户]"); + } if (Objects.isNull(request) || StringUtils.isEmpty(request.getEngineLabelId()) || StringUtils.isEmpty(request.getKey()) @@ -67,8 +72,12 @@ public Message add( @ApiOperation(value = "delete", notes = "delete a configuration template", httpMethod = "DELETE") @RequestMapping(path = "/{keyId}", method = RequestMethod.DELETE) public Message delete(HttpServletRequest httpRequest, @PathVariable("keyId") Long keyId) { - ModuleUserUtils.getOperationUser( - httpRequest, "delete a configuration template, keyId: " + keyId); + String username = + ModuleUserUtils.getOperationUser( + httpRequest, "delete a configuration template, keyId: " + keyId); + if (!Configuration.isAdmin(username)) { + return Message.error("User '" + username + "' is not admin user[非管理员用户]"); + } Boolean flag = configurationTemplateService.deleteConfigurationTemplate(keyId); return Message.ok("").data("success: ", flag); } diff --git a/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/basedatamanager/server/restful/DatasourceAccessRestfulApi.java b/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/basedatamanager/server/restful/DatasourceAccessRestfulApi.java index 7d39d493bd..4b0df0051d 100644 --- a/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/basedatamanager/server/restful/DatasourceAccessRestfulApi.java +++ b/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/basedatamanager/server/restful/DatasourceAccessRestfulApi.java @@ -107,8 +107,14 @@ public Message add( httpMethod = "DELETE") @RequestMapping(path = "/{id}", method = RequestMethod.DELETE) public Message remove(HttpServletRequest request, @PathVariable("id") Long id) { - ModuleUserUtils.getOperationUser( - request, "Remove a Datasource Access Record,id:" + id.toString()); + String username = + ModuleUserUtils.getOperationUser( + request, "Remove a Datasource Access Record,id:" + id.toString()); + + if (!Configuration.isAdmin(username)) { + return Message.error("User '" + username + "' is not admin user[非管理员用户]"); + } + boolean result = datasourceAccessService.removeById(id); return Message.ok("").data("result", result); } diff --git a/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/basedatamanager/server/restful/DatasourceEnvRestfulApi.java b/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/basedatamanager/server/restful/DatasourceEnvRestfulApi.java index cf2953b0e4..e6029c761c 100644 --- a/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/basedatamanager/server/restful/DatasourceEnvRestfulApi.java +++ b/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/basedatamanager/server/restful/DatasourceEnvRestfulApi.java @@ -97,7 +97,12 @@ public Message add(HttpServletRequest request, @RequestBody DatasourceEnvEntity httpMethod = "DELETE") @RequestMapping(path = "/{id}", method = RequestMethod.DELETE) public Message remove(HttpServletRequest request, @PathVariable("id") Long id) { - ModuleUserUtils.getOperationUser(request, "Remove a Datasource Env Record,id:" + id.toString()); + String username = + ModuleUserUtils.getOperationUser( + request, "Remove a Datasource Env Record,id:" + id.toString()); + if (!Configuration.isAdmin(username)) { + return Message.error("User '" + username + "' is not admin user[非管理员用户]"); + } boolean result = datasourceEnvService.removeById(id); return Message.ok("").data("result", result); } diff --git a/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/basedatamanager/server/restful/DatasourceTypeKeyRestfulApi.java b/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/basedatamanager/server/restful/DatasourceTypeKeyRestfulApi.java index 557c753227..b8ef65df5b 100644 --- a/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/basedatamanager/server/restful/DatasourceTypeKeyRestfulApi.java +++ b/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/basedatamanager/server/restful/DatasourceTypeKeyRestfulApi.java @@ -99,8 +99,12 @@ public Message add( httpMethod = "DELETE") @RequestMapping(path = "/{id}", method = RequestMethod.DELETE) public Message remove(HttpServletRequest request, @PathVariable("id") Long id) { - ModuleUserUtils.getOperationUser( - request, "Remove a Datasource Type Key Record,id:" + id.toString()); + String username = + ModuleUserUtils.getOperationUser( + request, "Remove a Datasource Type Key Record,id:" + id.toString()); + if (!Configuration.isAdmin(username)) { + return Message.error("User '" + username + "' is not admin user[非管理员用户]"); + } boolean result = datasourceTypeKeyService.removeById(id); return Message.ok("").data("result", result); } diff --git a/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/basedatamanager/server/restful/DatasourceTypeRestfulApi.java b/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/basedatamanager/server/restful/DatasourceTypeRestfulApi.java index 9fc8ea9d73..3c47d9385f 100644 --- a/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/basedatamanager/server/restful/DatasourceTypeRestfulApi.java +++ b/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/basedatamanager/server/restful/DatasourceTypeRestfulApi.java @@ -95,8 +95,13 @@ public Message add(HttpServletRequest request, @RequestBody DatasourceTypeEntity httpMethod = "DELETE") @RequestMapping(path = "/{id}", method = RequestMethod.DELETE) public Message remove(HttpServletRequest request, @PathVariable("id") Long id) { - ModuleUserUtils.getOperationUser( - request, "Remove a Datasource Type Record,id:" + id.toString()); + String username = + ModuleUserUtils.getOperationUser( + request, "Remove a Datasource Type Record,id:" + id.toString()); + if (!Configuration.isAdmin(username)) { + return Message.error("User '" + username + "' is not admin user[非管理员用户]"); + } + boolean result = datasourceTypeService.removeById(id); return Message.ok("").data("result", result); } diff --git a/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/basedatamanager/server/restful/GatewayAuthTokenRestfulApi.java b/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/basedatamanager/server/restful/GatewayAuthTokenRestfulApi.java index 93151ff771..364ca6b516 100644 --- a/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/basedatamanager/server/restful/GatewayAuthTokenRestfulApi.java +++ b/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/basedatamanager/server/restful/GatewayAuthTokenRestfulApi.java @@ -57,9 +57,17 @@ public class GatewayAuthTokenRestfulApi { @RequestMapping(path = "", method = RequestMethod.GET) public Message list( HttpServletRequest request, String searchName, Integer currentPage, Integer pageSize) { - ModuleUserUtils.getOperationUser( - request, "Query list data of Gateway Auth Token,search name:" + searchName); + + String username = + ModuleUserUtils.getOperationUser( + request, "Query list data of Gateway Auth Token,search name:" + searchName); + + if (!Configuration.isAdmin(username)) { + return Message.error("User '" + username + "' is not admin user[非管理员用户]"); + } + PageInfo pageList = gatewayAuthTokenService.getListByPage(searchName, currentPage, pageSize); + return Message.ok("").data("list", pageList); } @@ -67,8 +75,14 @@ public Message list( @ApiOperation(value = "get", notes = "Get a Gateway Auth Token Record by id", httpMethod = "GET") @RequestMapping(path = "/{id}", method = RequestMethod.GET) public Message get(HttpServletRequest request, @PathVariable("id") Long id) { - ModuleUserUtils.getOperationUser( - request, "Get a Gateway Auth Token Record,id:" + id.toString()); + + String username = + ModuleUserUtils.getOperationUser( + request, "Get a Gateway Auth Token Record,id:" + id.toString()); + + if (!Configuration.isAdmin(username)) { + return Message.error("User '" + username + "' is not admin user[非管理员用户]"); + } GatewayAuthTokenEntity gatewayAuthToken = gatewayAuthTokenService.getById(id); return Message.ok("").data("item", gatewayAuthToken); } diff --git a/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/service/impl/UDFServiceImpl.java b/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/service/impl/UDFServiceImpl.java index 4e41ea28b3..e5155c8ddb 100644 --- a/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/service/impl/UDFServiceImpl.java +++ b/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/service/impl/UDFServiceImpl.java @@ -127,7 +127,14 @@ public long addUDF(UDFAddVo udfVo, String userName) throws Exception { throw new UDFException("分类名不能为空！"); } // 支持hdfs path - FsPath fsPath = new FsPath(udfVo.getPath()); + String path = udfVo.getPath(); + if (StringUtils.isBlank(path) || path.contains("../")) { + throw new UDFException( + "The path: " + + path + + " of udf is error. Please rename it and rebuild it.(udf的路径错误，请修改后重建)"); + } + FsPath fsPath = new FsPath(path); // FileSystem fileSystem = (FileSystem) FSFactory.getFs(fsPath.getFsType()); FileSystem fileSystem = (FileSystem) FSFactory.getFsByProxyUser(fsPath, userName); if (udfVo.getUdfType() == UDF_JAR && StringUtils.isNotBlank(udfVo.getPath())) { diff --git a/pom.xml b/pom.xml index 83c627de85..3051aa3dca 100644 --- a/pom.xml +++ b/pom.xml @@ -41,7 +41,6 @@ Apache 2.0 License http://www.apache.org/licenses/LICENSE-2.0.html - repo @@ -119,6 +118,8 @@ 30.0-jre 4.1.100.Final + + 5.4.1 2.8.9 2.13.4.20221013 @@ -162,9 +163,9 @@ 5.7.2 - 3.9.0 + 4.3.1 3.17.2 - 1.4.200 + 2.2.220 5.3.34 @@ -172,6 +173,8 @@ 2.7.11 3.1.7 2021.0.8 + 2021.0.6.0 + 3.1.7 UTF-8 @@ -201,15 +204,35 @@ ${java.version} ${java.version} 3.3.0 + 3.3.0 + 3.2.0 4.7.1 2.24.1 - 0.8.7 + 0.8.8 false + + -XX:+IgnoreUnrecognizedVMOptions + --add-opens=java.base/java.lang=ALL-UNNAMED + --add-opens=java.base/java.lang.invoke=ALL-UNNAMED + --add-opens=java.base/java.lang.reflect=ALL-UNNAMED + --add-opens=java.base/java.io=ALL-UNNAMED + --add-opens=java.base/java.net=ALL-UNNAMED + --add-opens=java.base/java.nio=ALL-UNNAMED + --add-opens=java.base/java.util=ALL-UNNAMED + --add-opens=java.base/java.util.concurrent=ALL-UNNAMED + --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED + --add-opens=java.base/sun.nio.ch=ALL-UNNAMED + --add-opens=java.base/sun.nio.cs=ALL-UNNAMED + --add-opens=java.base/sun.security.action=ALL-UNNAMED + --add-opens=java.base/sun.util.calendar=ALL-UNNAMED + -Djdk.reflect.useDirectMethodHandle=false + + org.apache.linkis linkis-module @@ -225,13 +248,11 @@ - org.apache.linkis linkis-common ${project.version} - org.apache.linkis linkis-mybatis From 4a6b417b3498301050c42c720658bae25d3de40d Mon Sep 17 00:00:00 2001 From: peacewong Date: Fri, 30 Aug 2024 18:52:13 +0800 Subject: [PATCH 23/90] =?UTF-8?q?=E5=AE=8C=E6=88=90=E6=89=80=E6=9C=89?= =?UTF-8?q?=E6=A8=A1=E5=9D=97=E7=9A=84=E5=90=88=E5=B9=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 3051aa3dca..eec5768c64 100644 --- a/pom.xml +++ b/pom.xml @@ -163,7 +163,7 @@ 5.7.2 - 4.3.1 + 3.9.0 3.17.2 2.2.220 From 82474e8651c46a000261a7e87533dff131d93a58 Mon Sep 17 00:00:00 2001 From: peacewong Date: Fri, 30 Aug 2024 19:54:09 +0800 Subject: [PATCH 24/90] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E7=BC=96=E8=AF=91?= =?UTF-8?q?=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../linkis/jobhistory/service/JobHistoryQueryService.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/linkis-public-enhancements/linkis-jobhistory/src/main/scala/org/apache/linkis/jobhistory/service/JobHistoryQueryService.java b/linkis-public-enhancements/linkis-jobhistory/src/main/scala/org/apache/linkis/jobhistory/service/JobHistoryQueryService.java index e753cb0cfe..e348b24b93 100644 --- a/linkis-public-enhancements/linkis-jobhistory/src/main/scala/org/apache/linkis/jobhistory/service/JobHistoryQueryService.java +++ b/linkis-public-enhancements/linkis-jobhistory/src/main/scala/org/apache/linkis/jobhistory/service/JobHistoryQueryService.java @@ -56,5 +56,7 @@ public interface JobHistoryQueryService { List searchByTasks(List taskidList, String username); + List taskDurationTopN(Date sDate, Date eDate, String username, String creator, String engineType); + } From 9ad70483ae4e45272b6047e10c7acd34116603c9 Mon Sep 17 00:00:00 2001 From: peacewong Date: Mon, 2 Sep 2024 12:07:59 +0800 Subject: [PATCH 25/90] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E7=BC=96=E8=AF=91?= =?UTF-8?q?=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../simple/SimpleExecuteBusContext.scala | 28 +++++++++++++++++++ .../parser/CommonEntranceParser.scala | 2 +- 2 files changed, 29 insertions(+), 1 deletion(-) create mode 100644 linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/execute/simple/SimpleExecuteBusContext.scala diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/execute/simple/SimpleExecuteBusContext.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/execute/simple/SimpleExecuteBusContext.scala new file mode 100644 index 0000000000..6f2798a52c --- /dev/null +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/execute/simple/SimpleExecuteBusContext.scala @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.entrance.execute.simple +import org.apache.linkis.orchestrator.listener.OrchestratorListenerBusContext + +object SimpleExecuteBusContext { + + private lazy val orchestratorListenerBusContext = OrchestratorListenerBusContext.createBusContext + + def getOrchestratorListenerBusContext(): OrchestratorListenerBusContext = + orchestratorListenerBusContext + +} diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/parser/CommonEntranceParser.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/parser/CommonEntranceParser.scala index afc18bdc19..58fc1f45c3 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/parser/CommonEntranceParser.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/parser/CommonEntranceParser.scala @@ -195,7 +195,7 @@ class CommonEntranceParser(val persistenceManager: PersistenceManager) } private def generateAndVerifyClusterLabel(labels: util.Map[String, Label[_]]): Unit = { - if (!Configuration.IS_MULTIPLE_YARN_CLUSTER.getValue.asInstanceOf[Boolean]) { + if (!Configuration.IS_MULTIPLE_YARN_CLUSTER) { return } var clusterLabel = labels From aa544f1f5e1b661d9375575397b210bce0dc08e2 Mon Sep 17 00:00:00 2001 From: peacewong Date: Mon, 2 Sep 2024 12:26:17 +0800 Subject: [PATCH 26/90] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E7=BC=96=E8=AF=91?= =?UTF-8?q?=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../apache/linkis/entrance/server/DefaultEntranceServer.java | 2 +- .../org/apache/linkis/entrance/conf/EntranceConfiguration.scala | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/server/DefaultEntranceServer.java b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/server/DefaultEntranceServer.java index 7c38d27947..a1ec6fbd13 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/server/DefaultEntranceServer.java +++ b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/server/DefaultEntranceServer.java @@ -100,7 +100,7 @@ private void shutdownEntrance(ContextClosedEvent event) { } else { if (EntranceConfiguration.ENTRANCE_SHUTDOWN_FAILOVER_CONSUME_QUEUE_ENABLED()) { logger.warn("Entrance exit to update and clean all ConsumeQueue task instances"); - updateAllNotExecutionTaskInstances(false); + //updateAllNotExecutionTaskInstances(false); } logger.warn("Entrance exit to stop all job"); diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/conf/EntranceConfiguration.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/conf/EntranceConfiguration.scala index a4082f7e77..8b5cae9703 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/conf/EntranceConfiguration.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/conf/EntranceConfiguration.scala @@ -259,7 +259,7 @@ object EntranceConfiguration { // if true, the job in ConsumeQueue can be failover val ENTRANCE_SHUTDOWN_FAILOVER_CONSUME_QUEUE_ENABLED = - CommonVars("linkis.entrance.shutdown.failover.consume.queue.enable", true).getValue + CommonVars("linkis.entrance.shutdown.failover.consume.queue.enable", false).getValue val ENTRANCE_GROUP_SCAN_ENABLED = CommonVars("linkis.entrance.group.scan.enable", false) From 8a50cecb9038e128640817a87c26e07a9d01ac23 Mon Sep 17 00:00:00 2001 From: peacewong Date: Mon, 2 Sep 2024 17:53:53 +0800 Subject: [PATCH 27/90] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E7=BC=96=E8=AF=91?= =?UTF-8?q?=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../spark/datacalc/source/SolrSource.scala | 44 ------------------- 1 file changed, 44 deletions(-) delete mode 100644 linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/datacalc/source/SolrSource.scala diff --git a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/datacalc/source/SolrSource.scala b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/datacalc/source/SolrSource.scala deleted file mode 100644 index 950eae61da..0000000000 --- a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/datacalc/source/SolrSource.scala +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.engineplugin.spark.datacalc.source - -import org.apache.linkis.common.utils.Logging -import org.apache.linkis.engineplugin.spark.datacalc.api.DataCalcSource - -import org.apache.spark.sql.{Dataset, Row, SparkSession} - -class SolrSource extends DataCalcSource[SolrSourceConfig] with Logging { - - override def getData(spark: SparkSession): Dataset[Row] = { - val reader = spark.read.format("solr") - if (config.getOptions != null && !config.getOptions.isEmpty) { - reader.options(config.getOptions) - } - - logger.info( - s"Save data from solr zkhost: ${config.getZkhost}, collection: ${config.getCollection}" - ) - - reader - .option("zkhost", config.getZkhost) - .option("collection", config.getCollection) - .option("query", "*:*") - .load() - } - -} From 98683d73be65ef6238398b40c04f1ecc07695a50 Mon Sep 17 00:00:00 2001 From: peacewong Date: Mon, 2 Sep 2024 21:43:34 +0800 Subject: [PATCH 28/90] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E7=BC=96=E8=AF=91?= =?UTF-8?q?=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../linkis/engineplugin/spark/datacalc/util/PluginUtil.java | 1 - 1 file changed, 1 deletion(-) diff --git a/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/datacalc/util/PluginUtil.java b/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/datacalc/util/PluginUtil.java index 2d29c1b551..b3827c7afb 100644 --- a/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/datacalc/util/PluginUtil.java +++ b/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/datacalc/util/PluginUtil.java @@ -49,7 +49,6 @@ private static Map> getSourcePlugins() { classMap.put("rocketmq", RocketmqSource.class); classMap.put("mongo", MongoSource.class); classMap.put("elasticsearch", ElasticsearchSource.class); - classMap.put("solr", SolrSource.class); classMap.put("kafka", KafkaSource.class); classMap.put("starrocks", StarrocksSource.class); classMap.put("doris", DorisSource.class); From 7d82cf4ec919d4a9632ed85cf70a4d00165c1a8d Mon Sep 17 00:00:00 2001 From: peacewong Date: Mon, 2 Sep 2024 22:52:00 +0800 Subject: [PATCH 29/90] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E7=BC=96=E8=AF=91?= =?UTF-8?q?=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../main/scala/org/apache/linkis/common/conf/Configuration.scala | 1 - 1 file changed, 1 deletion(-) diff --git a/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/conf/Configuration.scala b/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/conf/Configuration.scala index d4754b3b55..d4654ca8e7 100644 --- a/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/conf/Configuration.scala +++ b/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/conf/Configuration.scala @@ -75,7 +75,6 @@ object Configuration extends Logging { val IS_VIEW_FS_ENV = CommonVars("wds.linkis.env.is.viewfs", true) - val IS_VIEW_FS_ENV = CommonVars("wds.linkis.env.is.viewfs", true) val ERROR_MSG_TIP = CommonVars( From 0a88fab8812929946d3c2ca9b2760a2baf7b0354 Mon Sep 17 00:00:00 2001 From: peacewong Date: Tue, 3 Sep 2024 10:56:46 +0800 Subject: [PATCH 30/90] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E7=BC=96=E8=AF=91?= =?UTF-8?q?=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../PythonModuleLoadEngineConnHookTest.scala | 82 ------------------- .../executor/hook/PythonModuleLoadTest.scala | 63 -------------- .../hook/PythonSparkEngineHookTest.scala | 61 -------------- 3 files changed, 206 deletions(-) delete mode 100644 linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/test/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonModuleLoadEngineConnHookTest.scala delete mode 100644 linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/test/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonModuleLoadTest.scala delete mode 100644 linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/test/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonSparkEngineHookTest.scala diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/test/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonModuleLoadEngineConnHookTest.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/test/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonModuleLoadEngineConnHookTest.scala deleted file mode 100644 index e507a7b22f..0000000000 --- a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/test/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonModuleLoadEngineConnHookTest.scala +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.engineconn.computation.executor.hook - -import org.apache.linkis.engineconn.common.creation.{DefaultEngineCreationContext, EngineCreationContext} -import org.apache.linkis.engineconn.common.engineconn.DefaultEngineConn -import org.apache.linkis.manager.label.entity.engine.CodeLanguageLabel -import org.junit.jupiter.api.Test -import org.mockito.Mockito.{mock, verify, when} - - -// 单元测试案例 -class PythonModuleLoadEngineConnHookTest { - - @Test - def testAfterExecutionExecute(): Unit = { - // 创建模拟对象 - val mockEngineCreationContext = new DefaultEngineCreationContext - val mockEngineConn = mock[DefaultEngineConn] - val hook = new PythonSparkEngineHook - - // 设置模拟行为 - var labels = new CodeLanguageLabel - labels.setCodeType("spark") - - // 执行测试方法 - hook.afterExecutionExecute(mockEngineCreationContext, mockEngineConn) - - } - - @Test - def testAfterEngineServerStartFailed(): Unit = { - // 创建模拟对象 - val mockEngineCreationContext = mock[EngineCreationContext] - val mockThrowable = mock[Throwable] - val hook = new PythonSparkEngineHook - - // 设置模拟行为 - var labels = new CodeLanguageLabel - labels.setCodeType("spark") - - // 执行测试方法 - hook.afterEngineServerStartFailed(mockEngineCreationContext, mockThrowable) - - } - - @Test - def testBeforeCreateEngineConn(): Unit = { - // 创建模拟对象 - - // 验证调用 - - } - - @Test - def testBeforeExecutionExecute(): Unit = { - // 创建模拟对象 - val mockEngineCreationContext = mock[EngineCreationContext] - val mockEngineConn = mock[DefaultEngineConn] - val hook = new PythonSparkEngineHook - - // 执行测试方法 - hook.beforeExecutionExecute(mockEngineCreationContext, mockEngineConn) - - - } -} \ No newline at end of file diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/test/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonModuleLoadTest.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/test/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonModuleLoadTest.scala deleted file mode 100644 index 18970a593b..0000000000 --- a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/test/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonModuleLoadTest.scala +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.engineconn.computation.executor.hook - -import org.apache.linkis.udf.entity.PythonModuleInfoVO -import org.junit.jupiter.api.Test - -import java.util - -class PythonModuleLoadTest { - /** - * 测试getEngineType方法，确保返回正确的引擎类型。 - */ - @Test def testGetEngineType(): Unit = { - val pythonModuleLoad: PythonModuleLoad = new PythonSparkEngineHook() { - override protected def getEngineType = "Spark" - } - } - - /** - * 测试constructCode方法，确保构建的代码字符串正确。 - */ - @Test def testConstructCode(): Unit = { - val pythonModuleLoad: PythonModuleLoad = new PythonSparkEngineHook() { - protected def constructCode(pythonModuleInfo: Nothing): String = "import " - } - val moduleInfo = new Nothing("numpy", "/path/to/numpy") - val expectedCode = "import numpy" - } - - /** - * 测试loadPythonModules方法，确保模块加载逻辑正确。 - */ - @Test def testLoadPythonModules(): Unit = { - val pythonModuleLoad: PythonModuleLoad = new PythonSparkEngineHook() { - override protected def getEngineType = "Spark" - - protected def constructCode(pythonModuleInfo: Nothing): String = "import " - } - val moduleInfoList = new util.ArrayList[PythonModuleInfoVO]() - moduleInfoList.add(new Nothing("numpy", "/path/to/numpy")) - moduleInfoList.add(new Nothing("pandas", "/path/to/pandas")) - // val labels = new Array[Label[_]] - // pythonModuleLoad.loadPythonModules(labels) - // 如果loadPythonModules方法有副作用，例如修改外部状态或调用其他方法， - // 那么这里应该添加相应的断言或验证。 - } -} \ No newline at end of file diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/test/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonSparkEngineHookTest.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/test/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonSparkEngineHookTest.scala deleted file mode 100644 index d3ff351070..0000000000 --- a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/test/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonSparkEngineHookTest.scala +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * This test suite contains unit tests for the PythonSparkEngineHook class. - * It ensures that the hook constructs the correct code for loading Python modules - * and logs the appropriate information. - */ -import org.apache.linkis.engineconn.computation.executor.hook.PythonSparkEngineHook -import org.apache.linkis.udf.entity.PythonModuleInfoVO -import org.junit.jupiter.api.Test -import org.mockito.Mockito - -class PythonSparkEngineHookTest { - - /** - * Test to verify that the constructCode method returns the correct code for loading a Python module. - */ - @Test - def testConstructCode(): Unit = { - val pythonModuleInfo = new PythonModuleInfoVO - pythonModuleInfo.setPath("file:///path/to/module.py") - - val hook = new PythonSparkEngineHook - // val result = hook.constructCode(pythonModuleInfo) - - // assert(result == "sc.addPyFile('file:///path/to/module.py')") - } - - /** - * Test to verify that the constructCode method logs the correct information when constructing the code. - */ - @Test - def testConstructCodeReturn(): Unit = { - val pythonModuleInfo = new PythonModuleInfoVO - pythonModuleInfo.setPath("file:///path/to/module.py") - - val hook = new PythonSparkEngineHook - val logger = Mockito.mock(classOf[org.slf4j.Logger]) - // hook.logger = logger - - // hook.constructCode(pythonModuleInfo) - - val expectedLog = "pythonLoadCode: sc.addPyFile('file:///path/to/module.py')" - Mockito.verify(logger).info(expectedLog) - } -} \ No newline at end of file From 7b170354c551898c9cf8a112fb04056e780d889f Mon Sep 17 00:00:00 2001 From: peacewong Date: Tue, 3 Sep 2024 11:36:09 +0800 Subject: [PATCH 31/90] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E7=BC=96=E8=AF=91?= =?UTF-8?q?=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../executor/JDBCEngineConnExecutor.scala | 32 ++++--------------- 1 file changed, 6 insertions(+), 26 deletions(-) diff --git a/linkis-engineconn-plugins/jdbc/src/main/scala/org/apache/linkis/manager/engineplugin/jdbc/executor/JDBCEngineConnExecutor.scala b/linkis-engineconn-plugins/jdbc/src/main/scala/org/apache/linkis/manager/engineplugin/jdbc/executor/JDBCEngineConnExecutor.scala index 195d4fa74c..f5f3e9fc24 100644 --- a/linkis-engineconn-plugins/jdbc/src/main/scala/org/apache/linkis/manager/engineplugin/jdbc/executor/JDBCEngineConnExecutor.scala +++ b/linkis-engineconn-plugins/jdbc/src/main/scala/org/apache/linkis/manager/engineplugin/jdbc/executor/JDBCEngineConnExecutor.scala @@ -19,57 +19,37 @@ package org.apache.linkis.manager.engineplugin.jdbc.executor import org.apache.linkis.common.conf.Configuration import org.apache.linkis.common.utils.{OverloadUtils, Utils} -import org.apache.linkis.engineconn.computation.executor.execute.{ - ConcurrentComputationExecutor, - EngineExecutionContext -} +import org.apache.linkis.engineconn.computation.executor.execute.{ConcurrentComputationExecutor, EngineExecutionContext} import org.apache.linkis.engineconn.core.EngineConnObject import org.apache.linkis.governance.common.paser.SQLCodeParser -import org.apache.linkis.governance.common.protocol.conf.{ - RequestQueryEngineConfig, - ResponseQueryConfig -} -import org.apache.linkis.manager.common.entity.resource.{ - CommonNodeResource, - LoadResource, - NodeResource -} +import org.apache.linkis.governance.common.protocol.conf.{RequestQueryEngineConfig, ResponseQueryConfig} +import org.apache.linkis.manager.common.entity.resource.{CommonNodeResource, LoadResource, NodeResource} import org.apache.linkis.manager.engineplugin.common.conf.EngineConnPluginConf import org.apache.linkis.manager.engineplugin.common.util.NodeResourceUtils import org.apache.linkis.manager.engineplugin.jdbc.ConnectionManager import org.apache.linkis.manager.engineplugin.jdbc.conf.JDBCConfiguration import org.apache.linkis.manager.engineplugin.jdbc.constant.JDBCEngineConnConstant import org.apache.linkis.manager.engineplugin.jdbc.errorcode.JDBCErrorCodeSummary.JDBC_GET_DATASOURCEINFO_ERROR -import org.apache.linkis.manager.engineplugin.jdbc.exception.{ - JDBCGetDatasourceInfoException, - JDBCParamsIllegalException -} +import org.apache.linkis.manager.engineplugin.jdbc.exception.{JDBCGetDatasourceInfoException, JDBCParamsIllegalException} import org.apache.linkis.manager.engineplugin.jdbc.monitor.ProgressMonitor import org.apache.linkis.manager.label.entity.Label import org.apache.linkis.manager.label.entity.engine.{EngineTypeLabel, UserCreatorLabel} import org.apache.linkis.protocol.CacheableProtocol import org.apache.linkis.protocol.engine.JobProgressInfo import org.apache.linkis.rpc.{RPCMapCache, Sender} -import org.apache.linkis.scheduler.executer.{ - AliasOutputExecuteResponse, - ErrorExecuteResponse, - ExecuteResponse, - SuccessExecuteResponse -} +import org.apache.linkis.scheduler.executer.{AliasOutputExecuteResponse, ErrorExecuteResponse, ExecuteResponse, SuccessExecuteResponse} import org.apache.linkis.storage.domain.{Column, DataType} import org.apache.linkis.storage.resultset.ResultSetFactory import org.apache.linkis.storage.resultset.table.{TableMetaData, TableRecord} - import org.apache.commons.io.IOUtils import org.apache.commons.lang3.StringUtils - +import org.apache.linkis.engineconn.computation.executor.entity.EngineConnTask import org.springframework.util.CollectionUtils import java.sql.{Connection, ResultSet, SQLException, Statement} import java.util import java.util.Collections import java.util.concurrent.ConcurrentHashMap - import scala.collection.mutable.ArrayBuffer class JDBCEngineConnExecutor(override val outputPrintLimit: Int, val id: Int) From 9bea35de17b36c8277883a5ef22dfdaab51bfe90 Mon Sep 17 00:00:00 2001 From: peacewong Date: Tue, 3 Sep 2024 15:13:19 +0800 Subject: [PATCH 32/90] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E7=BC=96=E8=AF=91?= =?UTF-8?q?=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../linkis/common/utils/SecurityUtils.java | 36 +++++++++++++++++++ pom.xml | 2 +- 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/linkis-commons/linkis-common/src/main/java/org/apache/linkis/common/utils/SecurityUtils.java b/linkis-commons/linkis-common/src/main/java/org/apache/linkis/common/utils/SecurityUtils.java index 0278b3337e..af163a6494 100644 --- a/linkis-commons/linkis-common/src/main/java/org/apache/linkis/common/utils/SecurityUtils.java +++ b/linkis-commons/linkis-common/src/main/java/org/apache/linkis/common/utils/SecurityUtils.java @@ -320,4 +320,40 @@ private static boolean isNotSecurity(String key, String value, String param) { return key.toLowerCase().contains(param.toLowerCase()) || value.toLowerCase().contains(param.toLowerCase()); } + + /** + * allowLoadLocalInfile=false&autoDeserialize=false&allowLocalInfile=false&allowUrlInLocalInfile=false + * + * @return + */ + public static Properties getMysqlSecurityParams() { + Properties properties = new Properties(); + properties.setProperty("allowLoadLocalInfile", "false"); + properties.setProperty("autoDeserialize", "false"); + properties.setProperty("allowLocalInfile", "false"); + properties.setProperty("allowUrlInLocalInfile", "false"); + return properties; + } + + /** + * Check if the path has a relative path + * + * @param path + * @return + */ + public static boolean containsRelativePath(String path) { + if (path.startsWith("./") + || path.contains("/./") + || path.startsWith("../") + || path.contains("/../")) { + return true; + } + if (path.startsWith(".\\") + || path.contains("\\.\\") + || path.startsWith("..\\") + || path.contains("\\..\\")) { + return true; + } + return false; + } } diff --git a/pom.xml b/pom.xml index eec5768c64..07281ac844 100644 --- a/pom.xml +++ b/pom.xml @@ -136,7 +136,7 @@ 0.9.12 3.5.3.1 8.0.28 - 1.1.22 + 1.1.23 3.27.0-GA 3.2.2 2.6 From 54e6b7901b1b2b5dd0bf7bcc951dae0c4eefd659 Mon Sep 17 00:00:00 2001 From: peacewong Date: Tue, 3 Sep 2024 17:36:39 +0800 Subject: [PATCH 33/90] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E7=BC=96=E8=AF=91?= =?UTF-8?q?=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../linkis/manager/engineplugin/jdbc/ConnectionManager.java | 6 ------ 1 file changed, 6 deletions(-) diff --git a/linkis-engineconn-plugins/jdbc/src/main/java/org/apache/linkis/manager/engineplugin/jdbc/ConnectionManager.java b/linkis-engineconn-plugins/jdbc/src/main/java/org/apache/linkis/manager/engineplugin/jdbc/ConnectionManager.java index f7598d113d..a5679f1cf5 100644 --- a/linkis-engineconn-plugins/jdbc/src/main/java/org/apache/linkis/manager/engineplugin/jdbc/ConnectionManager.java +++ b/linkis-engineconn-plugins/jdbc/src/main/java/org/apache/linkis/manager/engineplugin/jdbc/ConnectionManager.java @@ -216,12 +216,6 @@ protected DataSource buildDataSource(String dbUrl, Map propertie datasource.setPoolPreparedStatements(poolPreparedStatements); datasource.setRemoveAbandoned(removeAbandoned); datasource.setRemoveAbandonedTimeout(removeAbandonedTimeout); - if (connectionTimeout > 0) { - datasource.setConnectTimeout(connectionTimeout); - } - if (socketTimeout > 0) { - datasource.setSocketTimeout(socketTimeout); - } if (queryTimeout > 0) { datasource.setQueryTimeout(queryTimeout); } From 0281290c646f0c6a017277cfdbfa4e8bfe6e9053 Mon Sep 17 00:00:00 2001 From: aiceflower Date: Wed, 25 Sep 2024 14:06:13 +0800 Subject: [PATCH 34/90] update version to 1.9.0 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 719941a424..c72deceee6 100644 --- a/pom.xml +++ b/pom.xml @@ -103,7 +103,7 @@ - 1.8.0-wds + 1.9.0-wds 2.9.2 2.4.3 2.7.2 From 8744f9c44bebd300c72e53b3cb505db6c2f12f0e Mon Sep 17 00:00:00 2001 From: peacewong Date: Tue, 1 Oct 2024 19:17:12 +0800 Subject: [PATCH 35/90] =?UTF-8?q?=E4=BB=A3=E7=A0=81=E5=90=88=E5=B9=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../linkis/common/conf/Configuration.scala | 1 - .../hadoop/common/utils/HDFSUtils.scala | 6 ++- .../conf/EntranceSpringConfiguration.java | 3 -- .../server/DefaultEntranceServer.java | 2 +- .../execute/EntranceExecutorManager.scala | 1 + .../entrance/utils/JobHistoryHelper.scala | 8 ++-- .../manager/am/conf/AMConfiguration.java | 2 +- .../am/manager/DefaultEngineNodeManager.java | 14 ++++++ .../engine/DefaultEngineReuseService.scala | 6 ++- .../executor/JDBCEngineConnExecutor.scala | 33 +++++++++++--- .../jobhistory/dao/JobHistoryMapper.java | 41 +++++++++-------- .../restful/api/QueryRestfulApi.java | 44 +++++++++---------- .../impl/JobHistoryQueryServiceImpl.scala | 12 ++--- 13 files changed, 104 insertions(+), 69 deletions(-) diff --git a/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/conf/Configuration.scala b/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/conf/Configuration.scala index 2b34980077..16cac1d204 100644 --- a/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/conf/Configuration.scala +++ b/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/conf/Configuration.scala @@ -75,7 +75,6 @@ object Configuration extends Logging { val IS_VIEW_FS_ENV = CommonVars("wds.linkis.env.is.viewfs", true) - val ERROR_MSG_TIP = CommonVars( "linkis.jobhistory.error.msg.tip", diff --git a/linkis-commons/linkis-hadoop-common/src/main/scala/org/apache/linkis/hadoop/common/utils/HDFSUtils.scala b/linkis-commons/linkis-hadoop-common/src/main/scala/org/apache/linkis/hadoop/common/utils/HDFSUtils.scala index 3bd48b4461..aa8a933081 100644 --- a/linkis-commons/linkis-hadoop-common/src/main/scala/org/apache/linkis/hadoop/common/utils/HDFSUtils.scala +++ b/linkis-commons/linkis-hadoop-common/src/main/scala/org/apache/linkis/hadoop/common/utils/HDFSUtils.scala @@ -68,8 +68,10 @@ object HDFSUtils extends Logging { .foreach { hdfsFileSystemContainer => val locker = hdfsFileSystemContainer.getUser + LOCKER_SUFFIX locker.intern() synchronized { - if (hdfsFileSystemContainer.canRemove() && !HadoopConf.HDFS_ENABLE_NOT_CLOSE_USERS - .contains(hdfsFileSystemContainer.getUser) { + if ( + hdfsFileSystemContainer.canRemove() && !HadoopConf.HDFS_ENABLE_NOT_CLOSE_USERS + .contains(hdfsFileSystemContainer.getUser) + ) { fileSystemCache.remove( hdfsFileSystemContainer.getUser + JOINT + hdfsFileSystemContainer.getLabel ) diff --git a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/conf/EntranceSpringConfiguration.java b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/conf/EntranceSpringConfiguration.java index f5f61604f0..f896ae9a77 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/conf/EntranceSpringConfiguration.java +++ b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/conf/EntranceSpringConfiguration.java @@ -34,9 +34,6 @@ import org.apache.linkis.entrance.scheduler.EntranceGroupFactory; import org.apache.linkis.entrance.scheduler.EntranceParallelConsumerManager; import org.apache.linkis.entrance.scheduler.EntranceSchedulerContext; -import org.apache.linkis.orchestrator.ecm.EngineConnManagerBuilder; -import org.apache.linkis.orchestrator.ecm.EngineConnManagerBuilder$; -import org.apache.linkis.orchestrator.ecm.entity.Policy; import org.apache.linkis.scheduler.Scheduler; import org.apache.linkis.scheduler.SchedulerContext; import org.apache.linkis.scheduler.executer.ExecutorManager; diff --git a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/server/DefaultEntranceServer.java b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/server/DefaultEntranceServer.java index a1ec6fbd13..0130c5d680 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/server/DefaultEntranceServer.java +++ b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/server/DefaultEntranceServer.java @@ -100,7 +100,7 @@ private void shutdownEntrance(ContextClosedEvent event) { } else { if (EntranceConfiguration.ENTRANCE_SHUTDOWN_FAILOVER_CONSUME_QUEUE_ENABLED()) { logger.warn("Entrance exit to update and clean all ConsumeQueue task instances"); - //updateAllNotExecutionTaskInstances(false); + // updateAllNotExecutionTaskInstances(false); } logger.warn("Entrance exit to stop all job"); diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/execute/EntranceExecutorManager.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/execute/EntranceExecutorManager.scala index e5248b056e..f4e8f51539 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/execute/EntranceExecutorManager.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/execute/EntranceExecutorManager.scala @@ -29,6 +29,7 @@ import org.apache.linkis.scheduler.queue.{GroupFactory, Job, SchedulerEvent} import java.util.Date import java.util.concurrent.atomic.AtomicLong + import scala.concurrent.duration.Duration abstract class EntranceExecutorManager(groupFactory: GroupFactory) diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/utils/JobHistoryHelper.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/utils/JobHistoryHelper.scala index 376a369ff1..88badd1bf9 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/utils/JobHistoryHelper.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/utils/JobHistoryHelper.scala @@ -135,16 +135,16 @@ object JobHistoryHelper extends Logging { * @param retryWhenUpdateFail */ def updateAllConsumeQueueTask( - taskIdList: util.List[Long], - retryWhenUpdateFail: Boolean = false - ): Unit = { + taskIdList: util.List[Long], + retryWhenUpdateFail: Boolean = false + ): Unit = { if (taskIdList.isEmpty) return val updateTaskIds = new util.ArrayList[Long]() if ( - EntranceConfiguration.ENTRANCE_UPDATE_BATCH_SIZE.getValue > 0 && + EntranceConfiguration.ENTRANCE_UPDATE_BATCH_SIZE.getValue > 0 && taskIdList.size() > EntranceConfiguration.ENTRANCE_UPDATE_BATCH_SIZE.getValue ) { for (i <- 0 until EntranceConfiguration.ENTRANCE_UPDATE_BATCH_SIZE.getValue) { diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/conf/AMConfiguration.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/conf/AMConfiguration.java index 5164542445..051b53e3a5 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/conf/AMConfiguration.java +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/conf/AMConfiguration.java @@ -101,7 +101,7 @@ public class AMConfiguration { CommonVars.apply("wds.linkis.allow.batch.kill.engine.types", "spark,hive,python"); public static final CommonVars UNALLOW_BATCH_KILL_ENGINE_TYPES = - CommonVars.apply("wds.linkis.allow.batch.kill.engine.types", "trino,appconn,io_file"); + CommonVars.apply("wds.linkis.allow.batch.kill.engine.types", "trino,appconn,io_file,jdbc"); public static final CommonVars MULTI_USER_ENGINE_USER = CommonVars.apply("wds.linkis.multi.user.engine.user", getDefaultMultiEngineUser()); diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/manager/DefaultEngineNodeManager.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/manager/DefaultEngineNodeManager.java index 02b143d5cd..0b34bc7bf1 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/manager/DefaultEngineNodeManager.java +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/manager/DefaultEngineNodeManager.java @@ -31,9 +31,11 @@ import org.apache.linkis.manager.common.entity.metrics.NodeMetrics; import org.apache.linkis.manager.common.entity.node.*; import org.apache.linkis.manager.common.entity.persistence.PersistenceLabel; +import org.apache.linkis.manager.common.entity.persistence.PersistenceNode; import org.apache.linkis.manager.common.protocol.engine.EngineOperateRequest; import org.apache.linkis.manager.common.protocol.engine.EngineOperateResponse; import org.apache.linkis.manager.common.protocol.node.NodeHeartbeatMsg; +import org.apache.linkis.manager.dao.NodeManagerMapper; import org.apache.linkis.manager.label.builder.factory.LabelBuilderFactory; import org.apache.linkis.manager.label.builder.factory.LabelBuilderFactoryContext; import org.apache.linkis.manager.label.entity.engine.EngineInstanceLabel; @@ -73,6 +75,8 @@ public class DefaultEngineNodeManager implements EngineNodeManager { @Autowired private ResourceManager resourceManager; + @Autowired private NodeManagerMapper nodeManagerMapper; + @Autowired private LabelManagerPersistence labelManagerPersistence; private final LabelBuilderFactory labelBuilderFactory = @@ -223,6 +227,7 @@ public EngineNode[] getEngineNodes(ScoreServiceInstance[] scoreServiceInstances) if (scoreServiceInstances == null || scoreServiceInstances.length == 0) { return null; } + List instances = new ArrayList(); List scoreServiceInstancesList = Arrays.asList(scoreServiceInstances); EngineNode[] engineNodes = scoreServiceInstancesList.stream() @@ -231,6 +236,7 @@ public EngineNode[] getEngineNodes(ScoreServiceInstance[] scoreServiceInstances) AMEngineNode engineNode = new AMEngineNode(); engineNode.setScore(scoreServiceInstance.getScore()); engineNode.setServiceInstance(scoreServiceInstance.getServiceInstance()); + instances.add(scoreServiceInstance.getServiceInstance().getInstance()); return engineNode; }) .toArray(EngineNode[]::new); @@ -252,6 +258,8 @@ public EngineNode[] getEngineNodes(ScoreServiceInstance[] scoreServiceInstances) List nodeMetrics = nodeMetricManagerPersistence.getNodeMetrics(Arrays.asList(engineNodes)); + List persistenceNodes = nodeManagerMapper.getNodesByInstances(instances); + for (EngineNode engineNode : engineNodes) { Optional optionMetrics = nodeMetrics.stream() @@ -269,6 +277,12 @@ public EngineNode[] getEngineNodes(ScoreServiceInstance[] scoreServiceInstances) optionMetrics.ifPresent(metrics -> metricsConverter.fillMetricsToNode(engineNode, metrics)); optionRMNode.ifPresent(rmNode -> engineNode.setNodeResource(rmNode.getNodeResource())); + + persistenceNodes.stream() + .filter( + node -> node.getInstance().equals(engineNode.getServiceInstance().getInstance())) + .findFirst() + .ifPresent(persistenceNode -> engineNode.setParams(persistenceNode.getParams())); } } catch (Exception e) { LinkisRetryException linkisRetryException = diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/DefaultEngineReuseService.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/DefaultEngineReuseService.scala index 84ed877b31..11d6aa35ad 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/DefaultEngineReuseService.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/DefaultEngineReuseService.scala @@ -210,10 +210,12 @@ class DefaultEngineReuseService extends AbstractEngineService with EngineReuseSe } if (engine.getNodeResource.getUsedResource != null) { // 引擎资源只有满足需要的资源才复用 - pythonVersionMatch && engine.getNodeResource.getUsedResource.notLess(resource.getMaxResource) + pythonVersionMatch && engine.getNodeResource.getUsedResource + .notLess(resource.getMaxResource) } else { // 引擎正在启动中，比较锁住的资源，最终是否复用沿用之前复用逻辑 - pythonVersionMatch && engine.getNodeResource.getLockedResource.notLess(resource.getMaxResource) + pythonVersionMatch && engine.getNodeResource.getLockedResource + .notLess(resource.getMaxResource) } }) } diff --git a/linkis-engineconn-plugins/jdbc/src/main/scala/org/apache/linkis/manager/engineplugin/jdbc/executor/JDBCEngineConnExecutor.scala b/linkis-engineconn-plugins/jdbc/src/main/scala/org/apache/linkis/manager/engineplugin/jdbc/executor/JDBCEngineConnExecutor.scala index a85aa36235..6c4585c8f7 100644 --- a/linkis-engineconn-plugins/jdbc/src/main/scala/org/apache/linkis/manager/engineplugin/jdbc/executor/JDBCEngineConnExecutor.scala +++ b/linkis-engineconn-plugins/jdbc/src/main/scala/org/apache/linkis/manager/engineplugin/jdbc/executor/JDBCEngineConnExecutor.scala @@ -19,37 +19,58 @@ package org.apache.linkis.manager.engineplugin.jdbc.executor import org.apache.linkis.common.conf.Configuration import org.apache.linkis.common.utils.{OverloadUtils, Utils} -import org.apache.linkis.engineconn.computation.executor.execute.{ConcurrentComputationExecutor, EngineExecutionContext} +import org.apache.linkis.engineconn.computation.executor.entity.EngineConnTask +import org.apache.linkis.engineconn.computation.executor.execute.{ + ConcurrentComputationExecutor, + EngineExecutionContext +} import org.apache.linkis.engineconn.core.EngineConnObject import org.apache.linkis.governance.common.paser.SQLCodeParser -import org.apache.linkis.governance.common.protocol.conf.{RequestQueryEngineConfig, ResponseQueryConfig} -import org.apache.linkis.manager.common.entity.resource.{CommonNodeResource, LoadResource, NodeResource} +import org.apache.linkis.governance.common.protocol.conf.{ + RequestQueryEngineConfig, + ResponseQueryConfig +} +import org.apache.linkis.manager.common.entity.resource.{ + CommonNodeResource, + LoadResource, + NodeResource +} import org.apache.linkis.manager.engineplugin.common.conf.EngineConnPluginConf import org.apache.linkis.manager.engineplugin.common.util.NodeResourceUtils import org.apache.linkis.manager.engineplugin.jdbc.ConnectionManager import org.apache.linkis.manager.engineplugin.jdbc.conf.JDBCConfiguration import org.apache.linkis.manager.engineplugin.jdbc.constant.JDBCEngineConnConstant import org.apache.linkis.manager.engineplugin.jdbc.errorcode.JDBCErrorCodeSummary.JDBC_GET_DATASOURCEINFO_ERROR -import org.apache.linkis.manager.engineplugin.jdbc.exception.{JDBCGetDatasourceInfoException, JDBCParamsIllegalException} +import org.apache.linkis.manager.engineplugin.jdbc.exception.{ + JDBCGetDatasourceInfoException, + JDBCParamsIllegalException +} import org.apache.linkis.manager.engineplugin.jdbc.monitor.ProgressMonitor import org.apache.linkis.manager.label.entity.Label import org.apache.linkis.manager.label.entity.engine.{EngineTypeLabel, UserCreatorLabel} import org.apache.linkis.protocol.CacheableProtocol import org.apache.linkis.protocol.engine.JobProgressInfo import org.apache.linkis.rpc.{RPCMapCache, Sender} -import org.apache.linkis.scheduler.executer.{AliasOutputExecuteResponse, ErrorExecuteResponse, ExecuteResponse, SuccessExecuteResponse} +import org.apache.linkis.scheduler.executer.{ + AliasOutputExecuteResponse, + ErrorExecuteResponse, + ExecuteResponse, + SuccessExecuteResponse +} import org.apache.linkis.storage.domain.{Column, DataType} import org.apache.linkis.storage.resultset.ResultSetFactory import org.apache.linkis.storage.resultset.table.{TableMetaData, TableRecord} + import org.apache.commons.io.IOUtils import org.apache.commons.lang3.StringUtils -import org.apache.linkis.engineconn.computation.executor.entity.EngineConnTask + import org.springframework.util.CollectionUtils import java.sql.{Connection, ResultSet, SQLException, Statement} import java.util import java.util.Collections import java.util.concurrent.ConcurrentHashMap + import scala.collection.mutable.ArrayBuffer class JDBCEngineConnExecutor(override val outputPrintLimit: Int, val id: Int) diff --git a/linkis-public-enhancements/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/dao/JobHistoryMapper.java b/linkis-public-enhancements/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/dao/JobHistoryMapper.java index 681015ce7a..5783e86a91 100644 --- a/linkis-public-enhancements/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/dao/JobHistoryMapper.java +++ b/linkis-public-enhancements/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/dao/JobHistoryMapper.java @@ -154,31 +154,30 @@ List selectJobHistoryByTaskidList( * @return */ List selectFailoverJobHistory( - @Param("instancesMap") Map instancesMap, - @Param("statusList") List statusList, - @Param("startTimestamp") Long startTimestamp, - @Param("limit") Integer limit); + @Param("instancesMap") Map instancesMap, + @Param("statusList") List statusList, + @Param("startTimestamp") Long startTimestamp, + @Param("limit") Integer limit); List taskDurationTopN( - @Param("startDate") Date startDate, - @Param("endDate") Date endDate, - @Param("umUser") String username, - @Param("engineType") String engineType); + @Param("startDate") Date startDate, + @Param("endDate") Date endDate, + @Param("umUser") String username, + @Param("engineType") String engineType); List taskDurationTopNWithUserCreator( - @Param("umUser") String username, - @Param("userCreatorKey") String userCreatorKey, - @Param("userCreatorValue") String userCreator, - @Param("startDate") Date startDate, - @Param("endDate") Date endDate, - @Param("engineType") String engineType); + @Param("umUser") String username, + @Param("userCreatorKey") String userCreatorKey, + @Param("userCreatorValue") String userCreator, + @Param("startDate") Date startDate, + @Param("endDate") Date endDate, + @Param("engineType") String engineType); List taskDurationTopNWithCreatorOnly( - @Param("umUser") String username, - @Param("userCreatorKey") String userCreatorKey, - @Param("creator") String userCreator, - @Param("startDate") Date startDate, - @Param("endDate") Date endDate, - @Param("engineType") String engineType); - + @Param("umUser") String username, + @Param("userCreatorKey") String userCreatorKey, + @Param("creator") String userCreator, + @Param("startDate") Date startDate, + @Param("endDate") Date endDate, + @Param("engineType") String engineType); } diff --git a/linkis-public-enhancements/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/restful/api/QueryRestfulApi.java b/linkis-public-enhancements/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/restful/api/QueryRestfulApi.java index 8f46189248..9aaba5a06b 100644 --- a/linkis-public-enhancements/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/restful/api/QueryRestfulApi.java +++ b/linkis-public-enhancements/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/restful/api/QueryRestfulApi.java @@ -557,30 +557,30 @@ public void downloadJobList( /** Method list should not contain subjob, which may cause performance problems. */ @ApiOperation(value = "listDurationTop", notes = "listDurationTop", response = Message.class) @ApiImplicitParams({ - @ApiImplicitParam(name = "startDate", dataType = "long", example = "1658937600001"), - @ApiImplicitParam(name = "endDate", dataType = "long", example = "1658937600000"), - @ApiImplicitParam(name = "executeApplicationName", dataType = "String"), - @ApiImplicitParam(name = "creator", required = false, dataType = "String", value = "creator"), - @ApiImplicitParam( - name = "proxyUser", - required = false, - dataType = "String", - value = "proxyUser"), - @ApiImplicitParam(name = "pageNow", required = false, dataType = "Integer", value = "page now"), - @ApiImplicitParam(name = "pageSize", dataType = "Integer"), + @ApiImplicitParam(name = "startDate", dataType = "long", example = "1658937600001"), + @ApiImplicitParam(name = "endDate", dataType = "long", example = "1658937600000"), + @ApiImplicitParam(name = "executeApplicationName", dataType = "String"), + @ApiImplicitParam(name = "creator", required = false, dataType = "String", value = "creator"), + @ApiImplicitParam( + name = "proxyUser", + required = false, + dataType = "String", + value = "proxyUser"), + @ApiImplicitParam(name = "pageNow", required = false, dataType = "Integer", value = "page now"), + @ApiImplicitParam(name = "pageSize", dataType = "Integer"), }) @RequestMapping(path = "/listDurationTop", method = RequestMethod.GET) public Message listDurationTop( - HttpServletRequest req, - @RequestParam(value = "startDate", required = false) Long startDate, - @RequestParam(value = "endDate", required = false) Long endDate, - @RequestParam(value = "executeApplicationName", required = false) + HttpServletRequest req, + @RequestParam(value = "startDate", required = false) Long startDate, + @RequestParam(value = "endDate", required = false) Long endDate, + @RequestParam(value = "executeApplicationName", required = false) String executeApplicationName, - @RequestParam(value = "creator", required = false) String creator, - @RequestParam(value = "proxyUser", required = false) String proxyUser, - @RequestParam(value = "pageNow", required = false) Integer pageNow, - @RequestParam(value = "pageSize", required = false) Integer pageSize) - throws QueryException { + @RequestParam(value = "creator", required = false) String creator, + @RequestParam(value = "proxyUser", required = false) String proxyUser, + @RequestParam(value = "pageNow", required = false) Integer pageNow, + @RequestParam(value = "pageSize", required = false) Integer pageSize) + throws QueryException { if (org.springframework.util.StringUtils.isEmpty(pageNow)) { pageNow = 1; } @@ -628,8 +628,8 @@ public Message listDurationTop( PageHelper.startPage(pageNow, pageSize); try { queryTasks = - jobHistoryQueryService.taskDurationTopN( - sDate, eDate, proxyUser, creator, executeApplicationName); + jobHistoryQueryService.taskDurationTopN( + sDate, eDate, proxyUser, creator, executeApplicationName); } finally { PageHelper.clearPage(); } diff --git a/linkis-public-enhancements/linkis-jobhistory/src/main/scala/org/apache/linkis/jobhistory/service/impl/JobHistoryQueryServiceImpl.scala b/linkis-public-enhancements/linkis-jobhistory/src/main/scala/org/apache/linkis/jobhistory/service/impl/JobHistoryQueryServiceImpl.scala index 45efc75bc5..68441b9e85 100644 --- a/linkis-public-enhancements/linkis-jobhistory/src/main/scala/org/apache/linkis/jobhistory/service/impl/JobHistoryQueryServiceImpl.scala +++ b/linkis-public-enhancements/linkis-jobhistory/src/main/scala/org/apache/linkis/jobhistory/service/impl/JobHistoryQueryServiceImpl.scala @@ -507,12 +507,12 @@ class JobHistoryQueryServiceImpl extends JobHistoryQueryService with Logging { } override def taskDurationTopN( - sDate: Date, - eDate: Date, - username: String, - creator: String, - engineType: String - ): util.List[JobHistory] = { + sDate: Date, + eDate: Date, + username: String, + creator: String, + engineType: String + ): util.List[JobHistory] = { val result = if (StringUtils.isBlank(creator)) { jobHistoryMapper.taskDurationTopN(sDate, eDate, username, engineType) } else if (StringUtils.isBlank(username)) { From 06a491eacf3a2331472aa09677562eb69d53665e Mon Sep 17 00:00:00 2001 From: peacewong Date: Tue, 1 Oct 2024 22:32:11 +0800 Subject: [PATCH 36/90] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E7=BC=96=E8=AF=91?= =?UTF-8?q?=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../executor/hook/PythonModuleLoadEngineConnHook.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonModuleLoadEngineConnHook.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonModuleLoadEngineConnHook.scala index 2295ff1086..062f228298 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonModuleLoadEngineConnHook.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonModuleLoadEngineConnHook.scala @@ -249,7 +249,7 @@ class PythonEngineHook extends PythonModuleLoadEngineConnHook { val fileName: String = new java.io.File(path).getName val destPath: String = tmpDir + fileName val config: Configuration = HDFSUtils.getConfiguration(HadoopConf.HADOOP_ROOT_USER.getValue) - val fs: FileSystem = HDFSUtils.getHDFSUserFileSystem(user, config) + val fs: FileSystem = HDFSUtils.getHDFSUserFileSystem(user, null, config) fs.copyToLocalFile(new Path(path), new Path("file://" + destPath)) if (fileName.endsWith("zip")) { tmpDir += fileName From 2ccee09158bcbd0a45fc6369b489d7176a9b416e Mon Sep 17 00:00:00 2001 From: taoran1250 <543121890@qq.com> Date: Thu, 10 Oct 2024 15:41:33 +0800 Subject: [PATCH 37/90] fix: pom update (#613) * fix: pom update * fix: pom update --- linkis-extensions/linkis-et-monitor/pom.xml | 12 ++++++++++++ pom.xml | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/linkis-extensions/linkis-et-monitor/pom.xml b/linkis-extensions/linkis-et-monitor/pom.xml index bed33a17ce..18d7079951 100644 --- a/linkis-extensions/linkis-et-monitor/pom.xml +++ b/linkis-extensions/linkis-et-monitor/pom.xml @@ -79,6 +79,18 @@ de.codecentric spring-boot-admin-starter-server 2.7.16 + + + io.projectreactor.netty + reactor-netty-http + + + + + + io.projectreactor.netty + reactor-netty-http + 1.1.13 diff --git a/pom.xml b/pom.xml index 5edda6a687..521624f069 100644 --- a/pom.xml +++ b/pom.xml @@ -134,7 +134,7 @@ 2.10.5 1.4.20 0.9.12 - 3.5.3.1 + 3.5.7 8.0.28 1.1.23 3.27.0-GA From 23dc3214f65f802b4152e8d86ab2a34f650787c5 Mon Sep 17 00:00:00 2001 From: aiceflower Date: Fri, 18 Oct 2024 14:53:31 +0800 Subject: [PATCH 38/90] add udf check extention for spark (#616) --- .../conf/ComputationExecutorConf.scala | 10 +++++++ .../service/TaskExecutionServiceImpl.scala | 22 ++++++++++---- .../spark/extension/SparkUDFCheckRule.scala | 30 +++++++++++++++++++ .../factory/SparkEngineConnFactory.scala | 10 ++++++- 4 files changed, 66 insertions(+), 6 deletions(-) create mode 100644 linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/extension/SparkUDFCheckRule.scala diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/conf/ComputationExecutorConf.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/conf/ComputationExecutorConf.scala index a3f7bb49e3..292a150e22 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/conf/ComputationExecutorConf.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/conf/ComputationExecutorConf.scala @@ -136,4 +136,14 @@ object ComputationExecutorConf { val CLOSE_RS_OUTPUT_WHEN_RESET_BY_DEFAULT_ENABLED = CommonVars("linkis.ec.rs.close.when.reset.enabled", true).getValue + val SPECIAL_UDF_CHECK_ENABLED = + CommonVars("linkis.ec.spacial.udf.check.enabled", false) + + val SPECIAL_UDF_NAMES = + CommonVars("linkis.ec.spacial.udf.check.names", "") + + val ONLY_SQL_USE_UDF_KEY = "load.only.sql.use.udf" + + val CODE_TYPE = "codeType" + } diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/service/TaskExecutionServiceImpl.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/service/TaskExecutionServiceImpl.scala index 28df29db31..01b737ac83 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/service/TaskExecutionServiceImpl.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/service/TaskExecutionServiceImpl.scala @@ -18,7 +18,7 @@ package org.apache.linkis.engineconn.computation.executor.service import org.apache.linkis.common.listener.Event -import org.apache.linkis.common.utils.{Logging, Utils} +import org.apache.linkis.common.utils.{CodeAndRunTypeUtils, Logging, Utils} import org.apache.linkis.engineconn.acessible.executor.listener.LogListener import org.apache.linkis.engineconn.acessible.executor.listener.event._ import org.apache.linkis.engineconn.acessible.executor.log.LogHelper @@ -34,7 +34,6 @@ import org.apache.linkis.engineconn.computation.executor.execute.{ ComputationExecutor, ConcurrentComputationExecutor } -import org.apache.linkis.engineconn.computation.executor.hook.ExecutorLabelsRestHook import org.apache.linkis.engineconn.computation.executor.listener.{ ResultSetListener, TaskProgressListener, @@ -50,7 +49,6 @@ import org.apache.linkis.engineconn.core.executor.ExecutorManager import org.apache.linkis.engineconn.executor.entity.ResourceFetchExecutor import org.apache.linkis.engineconn.executor.listener.ExecutorListenerBusContext import org.apache.linkis.engineconn.executor.listener.event.EngineConnSyncEvent -import org.apache.linkis.engineconn.launch.EngineConnServer import org.apache.linkis.governance.common.constant.ec.ECConstants import org.apache.linkis.governance.common.entity.ExecutionNodeStatus import org.apache.linkis.governance.common.exception.engineconn.{ @@ -60,13 +58,12 @@ import org.apache.linkis.governance.common.exception.engineconn.{ import org.apache.linkis.governance.common.protocol.task._ import org.apache.linkis.governance.common.utils.{JobUtils, LoggerUtils} import org.apache.linkis.hadoop.common.utils.KerberosUtils -import org.apache.linkis.manager.common.entity.enumeration.NodeStatus import org.apache.linkis.manager.common.protocol.resource.{ ResponseTaskRunningInfo, ResponseTaskYarnResource } -import org.apache.linkis.manager.engineplugin.common.launch.process.LaunchConstants import org.apache.linkis.manager.label.entity.Label +import org.apache.linkis.manager.label.utils.LabelUtil import org.apache.linkis.protocol.constants.TaskConstant import org.apache.linkis.protocol.message.RequestProtocol import org.apache.linkis.rpc.Sender @@ -223,6 +220,21 @@ class TaskExecutionServiceImpl System.getProperties.put(ComputationExecutorConf.JOB_ID_TO_ENV_KEY, jobId) logger.info(s"Received job with id ${jobId}.") } + + // only sql can use udf check + val udfNames: String = ComputationExecutorConf.SPECIAL_UDF_NAMES.getValue + if ( + ComputationExecutorConf.SPECIAL_UDF_CHECK_ENABLED.getValue && StringUtils.isNotBlank( + udfNames + ) + ) { + System.getProperties.put(ComputationExecutorConf.ONLY_SQL_USE_UDF_KEY, udfNames) + val codeType: String = LabelUtil.getCodeType(requestTask.getLabels) + val languageType: String = CodeAndRunTypeUtils.getLanguageTypeByCodeType(codeType) + System.getProperties.put(ComputationExecutorConf.CODE_TYPE, languageType) + logger.info(s"add spacial udf check with job id ${jobId}.") + } + val task = new CommonEngineConnTask(taskId, retryAble) task.setCode(requestTask.getCode) task.setProperties(requestTask.getProperties) diff --git a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/extension/SparkUDFCheckRule.scala b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/extension/SparkUDFCheckRule.scala new file mode 100644 index 0000000000..28f84eee51 --- /dev/null +++ b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/extension/SparkUDFCheckRule.scala @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineplugin.spark.extension + +import org.apache.spark.sql.SparkSession +import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan +import org.apache.spark.sql.catalyst.rules.Rule + +case class SparkUDFCheckRule(sparkSession: SparkSession) extends Rule[LogicalPlan] { + + override def apply(plan: LogicalPlan): LogicalPlan = { + plan + } + +} diff --git a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/factory/SparkEngineConnFactory.scala b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/factory/SparkEngineConnFactory.scala index ad9786dff0..2c948ca2c8 100644 --- a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/factory/SparkEngineConnFactory.scala +++ b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/factory/SparkEngineConnFactory.scala @@ -20,6 +20,7 @@ package org.apache.linkis.engineplugin.spark.factory import org.apache.linkis.common.conf.CommonVars import org.apache.linkis.common.utils.{JsonUtils, Logging, Utils} import org.apache.linkis.engineconn.common.creation.EngineCreationContext +import org.apache.linkis.engineconn.computation.executor.conf.ComputationExecutorConf import org.apache.linkis.engineconn.launch.EngineConnServer import org.apache.linkis.engineplugin.spark.client.context.{ExecutionContext, SparkConfig} import org.apache.linkis.engineplugin.spark.config.SparkConfiguration @@ -32,6 +33,7 @@ import org.apache.linkis.engineplugin.spark.exception.{ SparkCreateFileException, SparkSessionNullException } +import org.apache.linkis.engineplugin.spark.extension.SparkUDFCheckRule import org.apache.linkis.manager.engineplugin.common.conf.EnvConfiguration import org.apache.linkis.manager.engineplugin.common.creation.{ ExecutorFactory, @@ -253,8 +255,14 @@ class SparkEngineConnFactory extends MultiExecutorEngineConnFactory with Logging if (SparkConfiguration.LINKIS_SPARK_ETL_SUPPORT_HUDI.getValue) { conf.set("spark.serializer", "org.apache.spark.serializer.KryoSerializer") } - val builder = SparkSession.builder.config(conf) + + if (ComputationExecutorConf.SPECIAL_UDF_CHECK_ENABLED.getValue) { + builder.withExtensions(extension => { + extension.injectOptimizerRule(SparkUDFCheckRule) + }) + } + builder.enableHiveSupport().getOrCreate() } From 76d3dfb46b3daadb4c26b4370dfb7f89a58cfb98 Mon Sep 17 00:00:00 2001 From: peacewong Date: Fri, 18 Oct 2024 17:53:43 +0800 Subject: [PATCH 39/90] =?UTF-8?q?=E4=BF=AE=E5=A4=8Dnpe=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../org/apache/linkis/hadoop/common/utils/HDFSUtils.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/linkis-commons/linkis-hadoop-common/src/main/scala/org/apache/linkis/hadoop/common/utils/HDFSUtils.scala b/linkis-commons/linkis-hadoop-common/src/main/scala/org/apache/linkis/hadoop/common/utils/HDFSUtils.scala index aa8a933081..d4b6af555a 100644 --- a/linkis-commons/linkis-hadoop-common/src/main/scala/org/apache/linkis/hadoop/common/utils/HDFSUtils.scala +++ b/linkis-commons/linkis-hadoop-common/src/main/scala/org/apache/linkis/hadoop/common/utils/HDFSUtils.scala @@ -155,11 +155,11 @@ object HDFSUtils extends Logging { hdfsFileSystemContainer.updateLastAccessTime hdfsFileSystemContainer.getFileSystem } else { - getHDFSUserFileSystem(userName, label, getConfiguration(userName, label)) + getHDFSUserFileSystem(userName, label, getConfigurationByLabel(userName, label)) } } } else { - getHDFSUserFileSystem(userName, label, getConfiguration(userName, label)) + getHDFSUserFileSystem(userName, label, getConfigurationByLabel(userName, label)) } } From b372c9c64954c4831455f160f08c24609d5b4c8b Mon Sep 17 00:00:00 2001 From: peacewong Date: Sun, 20 Oct 2024 23:06:40 +0800 Subject: [PATCH 40/90] =?UTF-8?q?Hive=20=E6=94=AF=E6=8C=81=E5=AD=97?= =?UTF-8?q?=E7=AC=A6=E4=B8=B2=E5=B8=A6=E5=88=B6=E8=A1=A8=E7=AC=A6=E5=8F=B7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../constant/job/JobRequestConstants.scala | 2 + .../executor/HiveEngineConnExecutor.scala | 58 ++++++++++++------- 2 files changed, 40 insertions(+), 20 deletions(-) diff --git a/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/constant/job/JobRequestConstants.scala b/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/constant/job/JobRequestConstants.scala index fef9120853..73fa68b175 100644 --- a/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/constant/job/JobRequestConstants.scala +++ b/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/constant/job/JobRequestConstants.scala @@ -33,4 +33,6 @@ object JobRequestConstants { val LINKIS_JDBC_DEFAULT_DB = "linkis.jdbc.default.db" + val LINKIS_HIVE_EC_READ_RESULT_BY_OBJECT = "readResByObject" + } diff --git a/linkis-engineconn-plugins/hive/src/main/scala/org/apache/linkis/engineplugin/hive/executor/HiveEngineConnExecutor.scala b/linkis-engineconn-plugins/hive/src/main/scala/org/apache/linkis/engineplugin/hive/executor/HiveEngineConnExecutor.scala index 788d9fbef8..333a0a41ed 100644 --- a/linkis-engineconn-plugins/hive/src/main/scala/org/apache/linkis/engineplugin/hive/executor/HiveEngineConnExecutor.scala +++ b/linkis-engineconn-plugins/hive/src/main/scala/org/apache/linkis/engineplugin/hive/executor/HiveEngineConnExecutor.scala @@ -124,6 +124,8 @@ class HiveEngineConnExecutor( private val splitter = "_" + private var readResByObject = false + override def init(): Unit = { LOG.info(s"Ready to change engine state!") if (HadoopConf.KEYTAB_PROXYUSER_ENABLED.getValue) { @@ -137,6 +139,16 @@ class HiveEngineConnExecutor( engineExecutorContext: EngineExecutionContext, code: String ): ExecuteResponse = { + readResByObject = engineExecutorContext.getProperties.getOrDefault( + JobRequestConstants.LINKIS_HIVE_EC_READ_RESULT_BY_OBJECT, + false + ) + if (readResByObject) { + hiveConf.set( + "list.sink.output.formatter", + "org.apache.hadoop.hive.serde2.thrift.ThriftFormatter" + ) + } this.engineExecutorContext = engineExecutorContext CSHiveHelper.setContextIDInfoToHiveConf(engineExecutorContext, hiveConf) singleSqlProgressMap.clear() @@ -354,30 +366,36 @@ class HiveEngineConnExecutor( val resultSetWriter = engineExecutorContext.createResultSetWriter(ResultSetFactory.TABLE_TYPE) resultSetWriter.addMetaData(metaData) val colLength = metaData.columns.length - val result = new util.ArrayList[String]() + val result = new util.ArrayList[Object]() var rows = 0 while (driver.getResults(result)) { - val scalaResult: mutable.Buffer[String] = result.asScala + val scalaResult: mutable.Buffer[Object] = result.asScala scalaResult foreach { s => - val arr: Array[String] = s.split("\t") - val arrAny: ArrayBuffer[Any] = new ArrayBuffer[Any]() - if (arr.length > colLength) { - logger.error( - s"""There is a \t tab in the result of hive code query, hive cannot cut it, please use spark to execute(查询的结果中有\t制表符，hive不能进行切割,请使用spark执行)""" - ) - throw new ErrorException( - 60078, - """There is a \t tab in the result of your query, hive cannot cut it, please use spark to execute(您查询的结果中有\t制表符，hive不能进行切割,请使用spark执行)""" - ) - } - if (arr.length == colLength) arr foreach arrAny.asJava.add - else if (arr.length == 0) for (i <- 1 to colLength) arrAny.asJava add "" - else { - val i = colLength - arr.length - arr foreach arrAny.asJava.add - for (i <- 1 to i) arrAny.asJava add "" + if (!readResByObject) { + val arr: Array[String] = s.asInstanceOf[String].split("\t") + val arrAny: ArrayBuffer[Any] = new ArrayBuffer[Any]() + if (arr.length > colLength) { + logger.error( + s"""There is a \t tab in the result of hive code query, hive cannot cut it, please use spark to execute(查询的结果中有\t制表符，hive不能进行切割,请使用spark执行)""" + ) + throw new ErrorException( + 60078, + """There is a \t tab in the result of your query, hive cannot cut it, please use spark to execute(您查询的结果中有\t制表符，hive不能进行切割,请使用spark执行)""" + ) + } + if (arr.length == colLength) { + arrAny.appendAll(arr) + } else if (arr.length == 0) for (i <- 1 to colLength) arrAny.asJava add "" + else { + val i = colLength - arr.length + arr foreach arrAny.asJava.add + for (i <- 1 to i) arrAny.asJava add "" + } + resultSetWriter.addRecord(new TableRecord(arrAny.toArray)) + } else { + resultSetWriter.addRecord(new TableRecord(s.asInstanceOf[Array[Any]])) } - resultSetWriter.addRecord(new TableRecord(arrAny.toArray)) + } rows += result.size result.clear() From 002dad1238abf4a8b728481b15e6c416e5437073 Mon Sep 17 00:00:00 2001 From: peacewong Date: Tue, 22 Oct 2024 11:51:16 +0800 Subject: [PATCH 41/90] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E7=BC=96=E8=AF=91?= =?UTF-8?q?=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../engineplugin/hive/executor/HiveEngineConnExecutor.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/linkis-engineconn-plugins/hive/src/main/scala/org/apache/linkis/engineplugin/hive/executor/HiveEngineConnExecutor.scala b/linkis-engineconn-plugins/hive/src/main/scala/org/apache/linkis/engineplugin/hive/executor/HiveEngineConnExecutor.scala index 333a0a41ed..f533d57d6a 100644 --- a/linkis-engineconn-plugins/hive/src/main/scala/org/apache/linkis/engineplugin/hive/executor/HiveEngineConnExecutor.scala +++ b/linkis-engineconn-plugins/hive/src/main/scala/org/apache/linkis/engineplugin/hive/executor/HiveEngineConnExecutor.scala @@ -142,7 +142,7 @@ class HiveEngineConnExecutor( readResByObject = engineExecutorContext.getProperties.getOrDefault( JobRequestConstants.LINKIS_HIVE_EC_READ_RESULT_BY_OBJECT, false - ) + ).asInstanceOf[Boolean] if (readResByObject) { hiveConf.set( "list.sink.output.formatter", From 8a41397cf7308efade3dde87bb232df287b6d53c Mon Sep 17 00:00:00 2001 From: peacewong Date: Wed, 23 Oct 2024 16:14:12 +0800 Subject: [PATCH 42/90] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E5=AE=A2=E6=88=B7?= =?UTF-8?q?=E7=AB=AF=E6=8B=89=E6=97=A5=E5=BF=97=E7=A9=BA=E6=8C=87=E9=92=88?= =?UTF-8?q?=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../interactor/job/common/LogRetriever.java | 49 +++++++------------ .../job/interactive/InteractiveJob.java | 16 +++--- 2 files changed, 26 insertions(+), 39 deletions(-) diff --git a/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/common/LogRetriever.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/common/LogRetriever.java index 33943f1748..a6928a3a46 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/common/LogRetriever.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/common/LogRetriever.java @@ -36,6 +36,13 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +/** + * Log retrieval logic: + * 1. LogRetriever polls to obtain real-time logs, and if the task is completed, it retrieves persistent logs + * 2. Organized by org.apache.inkis.cli.application. interactor.job. com LogRetriever # sendLogFin decides whether to continue polling logs + * 3. getNextLogLine is the FromLine returned by the log interface + * 4. The return of persistent logs is OpenLogResult2 + */ public class LogRetriever { private static final Logger logger = LoggerFactory.getLogger(LogRetriever.class); @@ -98,7 +105,8 @@ public void queryLogLoop(LogData data) { int nextLogIdx; boolean hasNext = true; int retryCnt = 0; - final int MAX_RETRY = 12; // continues fails for 90s, then exit thread + // continues fails for 90s, then exit thread + final int MAX_RETRY = 12; try { while (hasNext) { curLogIdx = data.getNextLogLineIdx() == null ? 0 : data.getNextLogLineIdx(); @@ -116,7 +124,7 @@ public void queryLogLoop(LogData data) { e); break; } - CliUtils.doSleepQuietly(500l + 500l * retryCnt); // maybe server problem. sleep longer + CliUtils.doSleepQuietly(500L + 500L * retryCnt); continue; } retryCnt = 0; @@ -129,7 +137,7 @@ public void queryLogLoop(LogData data) { if (curLogIdx >= nextLogIdx) { String msg = MessageFormat.format( - "Retrieving log, hasNext={0}, nextLogIdx={1}", hasNext, nextLogIdx); + "Retrieving log, curLogIdx={}, hasNext={0}, nextLogIdx={1}", curLogIdx, hasNext, nextLogIdx); logger.info(msg); } CliUtils.doSleepQuietly(CliConstants.JOB_QUERY_SLEEP_MILLS); @@ -144,37 +152,16 @@ public void queryLogLoop(LogData data) { private void queryJobLogFromLine(LogData data, int fromLine) throws LinkisClientRuntimeException { LinkisOperResultAdapter jobInfoResult = - linkisJobOperator.queryJobInfo(data.getUser(), data.getJobID()); + linkisJobOperator.queryJobInfo(data.getUser(), data.getJobID()); data.updateLog(jobInfoResult); if (!jobInfoResult.getJobStatus().isJobFinishedState()) { - try { - data.updateLog( - linkisJobOperator.queryRunTimeLogFromLine( - data.getUser(), data.getJobID(), data.getExecID(), fromLine)); - } catch (Exception e) { - // job is finished while we start query log(but request is not send). - // then probably server cache is gone and we got a exception here. - // however we cannot know if this happens based on the exception message - logger.warn( - "Caught exception when querying runtime-log. Probably server-side has close stream. Will try openLog api if Job is completed.", - e); - if (jobInfoResult.getJobStatus().isJobFinishedState()) { - CliUtils.doSleepQuietly(500l); - data.updateLog( - linkisJobOperator.queryPersistedLogFromLine( - data.getLogPath(), data.getUser(), data.getJobID(), fromLine)); - } - } + data.updateLog( + linkisJobOperator.queryRunTimeLogFromLine( + data.getUser(), data.getJobID(), data.getExecID(), fromLine)); } else { - try { - data.updateLog( - linkisJobOperator.queryPersistedLogFromLine( - data.getLogPath(), data.getUser(), data.getJobID(), fromLine)); - } catch (Exception e) { - logger.error("Cannot get persisted-inc-log:", e); - // and yes sometimes server may not be able to prepare persisted-log - throw e; - } + data.updateLog( + linkisJobOperator.queryPersistedLogFromLine( + data.getLogPath(), data.getUser(), data.getJobID(), fromLine)); } } diff --git a/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/interactive/InteractiveJob.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/interactive/InteractiveJob.java index ef64ffef47..6fbc454b89 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/interactive/InteractiveJob.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/interactive/InteractiveJob.java @@ -126,13 +126,13 @@ public JobResult run() { // get log while running LogRetriever logRetriever = - new LogRetriever( - jobInfoResult.getUser(), - jobInfoResult.getJobID(), - jobInfoResult.getStrongerExecId(), - true, - oper, - new LogPresenter()); + new LogRetriever( + jobInfoResult.getUser(), + jobInfoResult.getJobID(), + submitResult.getStrongerExecId(), + true, + oper, + new LogPresenter()); // async because we need to query job status logRetriever.retrieveLogAsync(); @@ -156,7 +156,7 @@ public JobResult run() { new ResultRetriever( jobInfoResult.getUser(), jobInfoResult.getJobID(), - jobInfoResult.getStrongerExecId(), + submitResult.getStrongerExecId(), oper, presenter); From 4cac70d8a1a1c32e5cd3f0b25fa93399d15404c7 Mon Sep 17 00:00:00 2001 From: peacewong Date: Wed, 23 Oct 2024 17:43:54 +0800 Subject: [PATCH 43/90] =?UTF-8?q?=E9=9C=80=E8=A6=81=E5=8A=A0=E4=B8=8AexecI?= =?UTF-8?q?D=E7=9A=84=E7=A1=AE=E8=AE=A4=E4=BF=9D=E5=AD=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../scala/org/apache/linkis/entrance/EntranceServer.scala | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceServer.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceServer.scala index eed2929c23..261b18921c 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceServer.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceServer.scala @@ -175,6 +175,7 @@ abstract class EntranceServer extends Logging { } getEntranceContext.getOrCreateScheduler().submit(job) + val msg = LogUtils.generateInfo( s"Job with jobId : ${jobRequest.getId} and execID : ${job.getId()} submitted " ) @@ -189,6 +190,10 @@ abstract class EntranceServer extends Logging { entranceJob.getLogListener.foreach(_.onLogUpdate(entranceJob, msg)) case _ => } + getEntranceContext + .getOrCreatePersistenceManager() + .createPersistenceEngine() + .updateIfNeeded(jobRequest) LoggerUtils.removeJobIdMDC() job } { t => From f7b22a681dc43f10c08b87c6b9d29c22e047bcfd Mon Sep 17 00:00:00 2001 From: v-kkhuang <62878639+v-kkhuang@users.noreply.github.com> Date: Wed, 23 Oct 2024 19:00:54 +0800 Subject: [PATCH 44/90] [1.9.0]Added StarRocks task timeout alarm and timeout kill function (#615) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Added StarRocks task timeout alarm and timeout kill function * Added StarRocks task timeout alarm and timeout kill function --------- Co-authored-by: “v_kkhuang” <“420895376@qq.com”> --- .../linkis/monitor/config/ListenerConfig.java | 4 +- .../monitor/entity/ClientSingleton.java | 107 +++++++++++++ .../monitor/scheduled/JobHistoryMonitor.java | 15 +- .../monitor/scheduled/ResourceMonitor.java | 2 +- .../monitor/scheduled/UserModeMonitor.java | 7 +- .../linkis/monitor/until/HttpsUntils.java | 143 +++++++----------- .../mapper/common/JobHistoryMapper.xml | 2 +- .../monitor/client/MonitorHTTPClient.scala | 21 ++- .../client/MonitorHTTPClientClientImpl.scala | 3 +- .../client/MonitorResourceClient.scala | 4 +- .../client/MonitorResourceClientImpl.scala | 5 +- .../StarrocksTimeExceedAlterSender.scala | 84 ++++++++++ .../StarrocksTimeExceedHitEvent.scala} | 10 +- .../jobtime/StarrocksTimeExceedRule.scala | 135 +++++++++++++++++ .../request/DataSourceParamsAction.scala | 92 +++++++++++ .../monitor/request/EmsListAction.scala | 2 +- .../monitor/request/EntranceTaskAction.scala | 2 +- .../monitor/request/KeyvalueAction.scala | 80 ++++++++++ .../monitor/request/KillJobAction.scala | 80 ++++++++++ ...sourceAction.scala => MonitorAction.scala} | 3 +- .../monitor/response/KeyvalueResult.scala | 36 +++++ .../response/KillJobResultAction.scala | 33 ++++ .../utils/alert/ims/MonitorAlertUtils.scala | 21 ++- 23 files changed, 768 insertions(+), 123 deletions(-) create mode 100644 linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/entity/ClientSingleton.java create mode 100644 linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/jobhistory/jobtime/StarrocksTimeExceedAlterSender.scala rename linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/{request/UserAction.scala => jobhistory/jobtime/StarrocksTimeExceedHitEvent.scala} (76%) create mode 100644 linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/jobhistory/jobtime/StarrocksTimeExceedRule.scala create mode 100644 linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/request/DataSourceParamsAction.scala create mode 100644 linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/request/KeyvalueAction.scala create mode 100644 linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/request/KillJobAction.scala rename linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/request/{MonitorResourceAction.scala => MonitorAction.scala} (88%) create mode 100644 linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/response/KeyvalueResult.scala create mode 100644 linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/response/KillJobResultAction.scala diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/config/ListenerConfig.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/config/ListenerConfig.java index eb5c11af87..98aec85f00 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/config/ListenerConfig.java +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/config/ListenerConfig.java @@ -17,7 +17,7 @@ package org.apache.linkis.monitor.config; -import org.apache.linkis.monitor.until.HttpsUntils; +import org.apache.linkis.monitor.entity.ClientSingleton; import org.apache.linkis.monitor.until.ThreadUtils; import org.apache.linkis.monitor.utils.log.LogUtils; @@ -38,7 +38,7 @@ public class ListenerConfig { private void shutdownEntrance(ContextClosedEvent event) { try { ThreadUtils.executors.shutdown(); - HttpsUntils.client.close(); + ClientSingleton.getInstance().close(); } catch (IOException e) { logger.error("ListenerConfig error msg {}", e.getMessage()); } diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/entity/ClientSingleton.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/entity/ClientSingleton.java new file mode 100644 index 0000000000..d55d0b9a27 --- /dev/null +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/entity/ClientSingleton.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.monitor.entity; + +import org.apache.linkis.bml.conf.BmlConfiguration; +import org.apache.linkis.common.conf.Configuration; +import org.apache.linkis.httpclient.dws.authentication.TokenAuthenticationStrategy; +import org.apache.linkis.httpclient.dws.config.DWSClientConfig; +import org.apache.linkis.httpclient.dws.config.DWSClientConfigBuilder; +import org.apache.linkis.monitor.client.MonitorHTTPClient; +import org.apache.linkis.monitor.client.MonitorHTTPClientClientImpl; + +import org.apache.commons.collections.MapUtils; +import org.apache.commons.lang3.StringUtils; + +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +public class ClientSingleton { + private static MonitorHTTPClient instance; + private static DWSClientConfig dwsClientConfig; + + private ClientSingleton() {} + + public static synchronized MonitorHTTPClient getInstance() { + if (instance == null) { + if (dwsClientConfig == null) { + dwsClientConfig = createClientConfig(null, null); + } + instance = new MonitorHTTPClientClientImpl(dwsClientConfig); + } + return instance; + } + + public static DWSClientConfig createClientConfig(String url, Map properties) { + String realUrl = ""; + if (StringUtils.isBlank(url)) { + realUrl = Configuration.getGateWayURL(); + } else { + realUrl = url; + } + Map parms = new HashMap<>(); + if (MapUtils.isNotEmpty(properties)) { + parms = properties; + } + int maxConnection = + (int) + parms.getOrDefault( + BmlConfiguration.CONNECTION_MAX_SIZE_SHORT_NAME(), + BmlConfiguration.CONNECTION_MAX_SIZE().getValue()); + int connectTimeout = + (int) + parms.getOrDefault( + BmlConfiguration.CONNECTION_TIMEOUT_SHORT_NAME(), + BmlConfiguration.CONNECTION_TIMEOUT().getValue()); + int readTimeout = + (int) + parms.getOrDefault( + BmlConfiguration.CONNECTION_READ_TIMEOUT_SHORT_NAME(), + BmlConfiguration.CONNECTION_READ_TIMEOUT().getValue()); + String tokenKey = + (String) + parms.getOrDefault( + BmlConfiguration.AUTH_TOKEN_KEY_SHORT_NAME(), + BmlConfiguration.AUTH_TOKEN_KEY().getValue()); + String tokenValue = + (String) + parms.getOrDefault( + BmlConfiguration.AUTH_TOKEN_VALUE_SHORT_NAME(), + BmlConfiguration.AUTH_TOKEN_VALUE().getValue()); + + DWSClientConfig clientConfig = + ((DWSClientConfigBuilder) + (DWSClientConfigBuilder.newBuilder() + .addServerUrl(realUrl) + .connectionTimeout(connectTimeout) + .discoveryEnabled(false) + .discoveryFrequency(1, TimeUnit.MINUTES) + .loadbalancerEnabled(false) + .maxConnectionSize(maxConnection) + .retryEnabled(false) + .readTimeout(readTimeout) + .setAuthenticationStrategy(new TokenAuthenticationStrategy()) + .setAuthTokenKey(tokenKey) + .setAuthTokenValue(tokenValue))) + .setDWSVersion("v1") + .build(); + + return clientConfig; + } +} diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scheduled/JobHistoryMonitor.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scheduled/JobHistoryMonitor.java index be7758f7a0..cd66bd9795 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scheduled/JobHistoryMonitor.java +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scheduled/JobHistoryMonitor.java @@ -29,6 +29,8 @@ import org.apache.linkis.monitor.jobhistory.index.JobIndexSender; import org.apache.linkis.monitor.jobhistory.jobtime.JobTimeExceedAlertSender; import org.apache.linkis.monitor.jobhistory.jobtime.JobTimeExceedRule; +import org.apache.linkis.monitor.jobhistory.jobtime.StarrocksTimeExceedAlterSender; +import org.apache.linkis.monitor.jobhistory.jobtime.StarrocksTimeExceedRule; import org.apache.linkis.monitor.jobhistory.labels.JobHistoryLabelsAlertSender; import org.apache.linkis.monitor.jobhistory.labels.JobHistoryLabelsRule; import org.apache.linkis.monitor.jobhistory.runtime.CommonJobRunTimeRule; @@ -204,15 +206,18 @@ public void jobHistoryUnfinishedScan() { logger.info("[INFO] Loaded 0 alerts jobtime alert-rule from alert properties file."); } else { logger.info( - "[INFO] Loaded {} alerts jobtime alert-rules from alert properties file.", - jobTimeAlerts.size()); + "[INFO] Loaded {} alerts jobtime alert-rules from alert properties file.", + jobTimeAlerts.size()); shouldStart = true; JobMonitorUtils.addIntervalToImsAlerts(jobTimeAlerts, realIntervals); JobTimeExceedRule jobTimeExceedRule = - new JobTimeExceedRule( - jobTimeAlerts.keySet(), new JobTimeExceedAlertSender(jobTimeAlerts)); - scanner.addScanRule(jobTimeExceedRule); + new JobTimeExceedRule( + jobTimeAlerts.keySet(), new JobTimeExceedAlertSender(jobTimeAlerts)); + scanner.addScanRule(jobTimeExceedRule); } + StarrocksTimeExceedRule starrocksTimeExceedRule = + new StarrocksTimeExceedRule(new StarrocksTimeExceedAlterSender()); + scanner.addScanRule(starrocksTimeExceedRule); JobMonitorUtils.run(scanner, fetchers, shouldStart); } } diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scheduled/ResourceMonitor.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scheduled/ResourceMonitor.java index e8658050e0..7937829842 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scheduled/ResourceMonitor.java +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scheduled/ResourceMonitor.java @@ -61,7 +61,7 @@ public void ecmResourceTask() { // 获取emNode资源信息 List> emNodeVoList = new ArrayList<>(); try { - Map resultmap = HttpsUntils.sendHttp(null, null); + Map resultmap = HttpsUntils.getEmsResourceList(null, null); // got interface data Map>> data = MapUtils.getMap(resultmap, "data"); emNodeVoList = data.getOrDefault("EMs", new ArrayList<>()); diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scheduled/UserModeMonitor.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scheduled/UserModeMonitor.java index ad6f861479..e55e01352d 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scheduled/UserModeMonitor.java +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scheduled/UserModeMonitor.java @@ -24,7 +24,7 @@ import org.apache.linkis.manager.label.constant.LabelKeyConstant; import org.apache.linkis.monitor.config.MonitorConfig; import org.apache.linkis.monitor.constants.Constants; -import org.apache.linkis.monitor.until.HttpsUntils; +import org.apache.linkis.monitor.entity.ClientSingleton; import org.apache.linkis.monitor.utils.alert.AlertDesc; import org.apache.linkis.monitor.utils.alert.ims.MonitorAlertUtils; import org.apache.linkis.monitor.utils.alert.ims.PooledImsAlertUtils; @@ -57,7 +57,8 @@ public class UserModeMonitor { private static final Logger logger = LoggerFactory.getLogger(UserModeMonitor.class); - private static final DWSClientConfig clientConfig = HttpsUntils.dwsClientConfig; + private static final DWSClientConfig clientConfig = + ClientSingleton.createClientConfig(null, null); private static final UJESClient client = new UJESClientImpl(clientConfig); @@ -138,7 +139,7 @@ private static JobExecuteResult toSubmit(LinkedTreeMap engine) { public void dbJob() { Map properties = new HashMap<>(); properties.put("readTimeout", MonitorConfig.USER_MODE_INTERFACE_TIMEOUT.getValue()); - DWSClientConfig clientConfig = HttpsUntils.createClientConfig(null, properties); + DWSClientConfig clientConfig = ClientSingleton.createClientConfig(null, properties); UJESClientImpl ujesClient = new UJESClientImpl(clientConfig); GetTableStatisticInfoAction builder = GetTableStatisticInfoAction.builder() diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/until/HttpsUntils.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/until/HttpsUntils.java index a504a9d41d..1fe1bf940c 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/until/HttpsUntils.java +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/until/HttpsUntils.java @@ -17,24 +17,22 @@ package org.apache.linkis.monitor.until; -import org.apache.linkis.bml.conf.BmlConfiguration; -import org.apache.linkis.common.conf.Configuration; import org.apache.linkis.common.utils.Utils; -import org.apache.linkis.httpclient.dws.authentication.TokenAuthenticationStrategy; -import org.apache.linkis.httpclient.dws.config.DWSClientConfig; -import org.apache.linkis.httpclient.dws.config.DWSClientConfigBuilder; +import org.apache.linkis.datasource.client.response.GetInfoPublishedByDataSourceNameResult; import org.apache.linkis.monitor.client.MonitorHTTPClient; -import org.apache.linkis.monitor.client.MonitorHTTPClientClientImpl; import org.apache.linkis.monitor.config.MonitorConfig; +import org.apache.linkis.monitor.entity.ClientSingleton; import org.apache.linkis.monitor.entity.IndexEntity; -import org.apache.linkis.monitor.request.EmsListAction; -import org.apache.linkis.monitor.request.EntranceTaskAction; +import org.apache.linkis.monitor.jobhistory.entity.JobHistory; +import org.apache.linkis.monitor.request.*; import org.apache.linkis.monitor.response.EntranceTaskResult; +import org.apache.linkis.monitor.response.KeyvalueResult; +import org.apache.linkis.monitor.response.KillJobResultAction; +import org.apache.linkis.protocol.utils.ZuulEntranceUtils; import org.apache.linkis.server.BDPJettyServerHelper; import org.apache.linkis.ujes.client.response.EmsListResult; import org.apache.commons.collections.MapUtils; -import org.apache.commons.lang3.StringUtils; import org.apache.http.client.config.RequestConfig; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpPost; @@ -47,10 +45,7 @@ import org.springframework.util.Assert; import java.io.IOException; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.TimeUnit; +import java.util.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -58,89 +53,19 @@ public class HttpsUntils { private static final Logger logger = LoggerFactory.getLogger(HttpsUntils.class); - public static DWSClientConfig dwsClientConfig = createClientConfig(null, null); - // IOUtils.closeQuietly(client); - public static MonitorHTTPClient client = new MonitorHTTPClientClientImpl(dwsClientConfig); public static final String localHost = Utils.getLocalHostname(); - public static Map sendHttp(String url, Map properties) + public static Map getEmsResourceList(String url, Map properties) throws IOException { - if (null == dwsClientConfig) { - dwsClientConfig = createClientConfig(url, properties); - } - if (null == client) { - client = new MonitorHTTPClientClientImpl(dwsClientConfig); - } + MonitorHTTPClient client = ClientSingleton.getInstance(); EmsListAction build = EmsListAction.newBuilder().setUser("hadoop").build(); EmsListResult result = client.list(build); return result.getResultMap(); } - public static DWSClientConfig createClientConfig(String url, Map properties) { - String realUrl = ""; - if (StringUtils.isBlank(url)) { - realUrl = Configuration.getGateWayURL(); - } else { - realUrl = url; - } - Map parms = new HashMap<>(); - if (MapUtils.isNotEmpty(properties)) { - parms = properties; - } - int maxConnection = - (int) - parms.getOrDefault( - BmlConfiguration.CONNECTION_MAX_SIZE_SHORT_NAME(), - BmlConfiguration.CONNECTION_MAX_SIZE().getValue()); - int connectTimeout = - (int) - parms.getOrDefault( - BmlConfiguration.CONNECTION_TIMEOUT_SHORT_NAME(), - BmlConfiguration.CONNECTION_TIMEOUT().getValue()); - int readTimeout = - (int) - parms.getOrDefault( - BmlConfiguration.CONNECTION_READ_TIMEOUT_SHORT_NAME(), - BmlConfiguration.CONNECTION_READ_TIMEOUT().getValue()); - String tokenKey = - (String) - parms.getOrDefault( - BmlConfiguration.AUTH_TOKEN_KEY_SHORT_NAME(), - BmlConfiguration.AUTH_TOKEN_KEY().getValue()); - String tokenValue = - (String) - parms.getOrDefault( - BmlConfiguration.AUTH_TOKEN_VALUE_SHORT_NAME(), - BmlConfiguration.AUTH_TOKEN_VALUE().getValue()); - - DWSClientConfig clientConfig = - ((DWSClientConfigBuilder) - (DWSClientConfigBuilder.newBuilder() - .addServerUrl(realUrl) - .connectionTimeout(connectTimeout) - .discoveryEnabled(false) - .discoveryFrequency(1, TimeUnit.MINUTES) - .loadbalancerEnabled(false) - .maxConnectionSize(maxConnection) - .retryEnabled(false) - .readTimeout(readTimeout) - .setAuthenticationStrategy(new TokenAuthenticationStrategy()) - .setAuthTokenKey(tokenKey) - .setAuthTokenValue(tokenValue))) - .setDWSVersion("v1") - .build(); - - return clientConfig; - } - public static Map getEntranceTask(String url, String user, String Instance) throws IOException { - if (null == dwsClientConfig) { - dwsClientConfig = createClientConfig(null, null); - } - if (null == client) { - client = new MonitorHTTPClientClientImpl(dwsClientConfig); - } + MonitorHTTPClient client = ClientSingleton.getInstance(); EntranceTaskAction build = EntranceTaskAction.newBuilder().setUser(user).setInstance(Instance).build(); EntranceTaskResult result = client.entranList(build); @@ -170,4 +95,50 @@ public static void sendIndex(List list) throws IOException { logger.info("send index response :{}", map); Assert.isTrue(!"0".equals(map.get("resultCode")), map.get("resultMsg")); } + + public static String getJDBCConf(String user, String conf) { + MonitorHTTPClient client = ClientSingleton.getInstance(); + KeyvalueAction build = + KeyvalueAction.newBuilder() + .setVersion("4") + .setEngineType("jdbc") + .setCreator("IDE") + .setConfigKey(conf) + .setUser(user) + .build(); + KeyvalueResult result = client.getConfKeyValue(build); + Map data = MapUtils.getMap(result.getResultMap(), "data", new HashMap<>()); + Map configValues = MapUtils.getMap(data, "configValues", new HashMap<>()); + return MapUtils.getString(configValues, "configValue", ""); + } + + public static Map getDatasourceConf(String user, String datasourceName) { + MonitorHTTPClient client = ClientSingleton.getInstance(); + DataSourceParamsAction dataSourceParamsAction = + DataSourceParamsAction.builder() + .setSystem("5435") + .setDataSourceName(datasourceName) + .setUser(user) + .build(); + GetInfoPublishedByDataSourceNameResult result = client.getInfoByDataSourceInfo(dataSourceParamsAction); + Map data = MapUtils.getMap(result.getResultMap(), "data", new HashMap<>()); + Map datasourceInfoMap = MapUtils.getMap(data, "info", new HashMap<>()); + return datasourceInfoMap; + } + + public static void killJob(JobHistory jobHistory) { + MonitorHTTPClient client = ClientSingleton.getInstance(); + String[] split = jobHistory.getInstances().split(";"); + String execID = + ZuulEntranceUtils.generateExecID(jobHistory.getJobReqId(), "linkis-cg-entrance", split); + KillJobAction killJobAction = + KillJobAction.builder() + .setIdList(Collections.singletonList(execID)) + .setTaskIDList(Collections.singletonList(jobHistory.getId())) + .setExecID(execID) + .setUser(jobHistory.getSubmitUser()) + .build(); + KillJobResultAction killJobResultAction = client.killJob(killJobAction); + Map data = MapUtils.getMap(killJobResultAction.getResultMap(), "data", new HashMap<>()); + } } diff --git a/linkis-extensions/linkis-et-monitor/src/main/resources/mapper/common/JobHistoryMapper.xml b/linkis-extensions/linkis-et-monitor/src/main/resources/mapper/common/JobHistoryMapper.xml index 1d7bc170c5..f6807b8857 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/resources/mapper/common/JobHistoryMapper.xml +++ b/linkis-extensions/linkis-et-monitor/src/main/resources/mapper/common/JobHistoryMapper.xml @@ -47,7 +47,7 @@ job.`id`,job.`job_req_id`,job.`submit_user`,job.`execute_user`,job.`labels`,job.`params`,job.`status`,job.`error_code`,job.`created_time`, - job.`updated_time`,job.`instances`,job.`observe_info`,org.`org_id`,org.`org_name` + job.`updated_time`,job.`instances`,job.`engine_type`,job.`observe_info`,org.`org_id`,org.`org_name` SELECT ifnull(max(version_id),0 ) FROM `linkis_ps_dm_datasource_version` WHERE datasource_id = #{dataSourceId}; diff --git a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/elasticsearch/src/main/java/org/apache/linkis/metadata/query/service/EsMetaService.java b/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/elasticsearch/src/main/java/org/apache/linkis/metadata/query/service/EsMetaService.java index f862c7a15a..e462d088ee 100644 --- a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/elasticsearch/src/main/java/org/apache/linkis/metadata/query/service/EsMetaService.java +++ b/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/elasticsearch/src/main/java/org/apache/linkis/metadata/query/service/EsMetaService.java @@ -17,6 +17,7 @@ package org.apache.linkis.metadata.query.service; +import org.apache.linkis.common.utils.AESUtils; import org.apache.linkis.datasourcemanager.common.util.json.Json; import org.apache.linkis.metadata.query.common.domain.MetaColumnInfo; import org.apache.linkis.metadata.query.common.service.AbstractDbMetaService; @@ -39,13 +40,14 @@ public MetadataConnection getConnection( } else { endPoints = ((List) urls).toArray(endPoints); } + String password = + String.valueOf(params.getOrDefault(ElasticParamsMapper.PARAM_ES_PASSWORD.getValue(), "")); ElasticConnection conn = new ElasticConnection( endPoints, String.valueOf( params.getOrDefault(ElasticParamsMapper.PARAM_ES_USERNAME.getValue(), "")), - String.valueOf( - params.getOrDefault(ElasticParamsMapper.PARAM_ES_PASSWORD.getValue(), ""))); + AESUtils.isDecryptByConf(password)); return new MetadataConnection<>(conn, false); } diff --git a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/clickhouse/SqlConnection.java b/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/clickhouse/SqlConnection.java index 0ec7ce9a6e..190262dad6 100644 --- a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/clickhouse/SqlConnection.java +++ b/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/clickhouse/SqlConnection.java @@ -18,6 +18,7 @@ package org.apache.linkis.metadata.query.service.clickhouse; import org.apache.linkis.common.conf.CommonVars; +import org.apache.linkis.common.utils.AESUtils; import org.apache.linkis.metadata.query.common.domain.MetaColumnInfo; import java.io.Closeable; @@ -200,7 +201,8 @@ private Connection getDBConnection(ConnectMessage connectMessage, String databas if (!connectMessage.extraParams.isEmpty()) { url += "?" + extraParamString; } - return DriverManager.getConnection(url, connectMessage.username, connectMessage.password); + return DriverManager.getConnection( + url, connectMessage.username, AESUtils.isDecryptByConf(connectMessage.password)); } /** Connect message */ diff --git a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/db2/SqlConnection.java b/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/db2/SqlConnection.java index 561e935d5b..e1b4afd011 100644 --- a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/db2/SqlConnection.java +++ b/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/db2/SqlConnection.java @@ -18,6 +18,7 @@ package org.apache.linkis.metadata.query.service.db2; import org.apache.linkis.common.conf.CommonVars; +import org.apache.linkis.common.utils.AESUtils; import org.apache.linkis.metadata.query.common.domain.MetaColumnInfo; import org.apache.logging.log4j.util.Strings; @@ -212,7 +213,8 @@ private Connection getDBConnection(ConnectMessage connectMessage, String databas if (!connectMessage.extraParams.isEmpty()) { url += "?" + extraParamString; } - return DriverManager.getConnection(url, connectMessage.username, connectMessage.password); + return DriverManager.getConnection( + url, connectMessage.username, AESUtils.isDecryptByConf(connectMessage.password)); } /** Connect message */ diff --git a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/dm/SqlConnection.java b/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/dm/SqlConnection.java index 27e953728f..eacdfafe38 100644 --- a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/dm/SqlConnection.java +++ b/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/dm/SqlConnection.java @@ -18,6 +18,7 @@ package org.apache.linkis.metadata.query.service.dm; import org.apache.linkis.common.conf.CommonVars; +import org.apache.linkis.common.utils.AESUtils; import org.apache.linkis.metadata.query.common.domain.MetaColumnInfo; import org.apache.commons.lang3.StringUtils; @@ -219,7 +220,7 @@ private Connection getDBConnection(ConnectMessage connectMessage, String databas // connectMessage.password); Properties prop = new Properties(); prop.put("user", connectMessage.username); - prop.put("password", connectMessage.password); + prop.put("password", AESUtils.isDecryptByConf(connectMessage.password)); prop.put("remarksReporting", "true"); return DriverManager.getConnection(url, prop); } catch (Exception e) { diff --git a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/greenplum/SqlConnection.java b/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/greenplum/SqlConnection.java index 4ea0910775..938c343d5b 100644 --- a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/greenplum/SqlConnection.java +++ b/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/greenplum/SqlConnection.java @@ -18,6 +18,7 @@ package org.apache.linkis.metadata.query.service.greenplum; import org.apache.linkis.common.conf.CommonVars; +import org.apache.linkis.common.utils.AESUtils; import org.apache.linkis.metadata.query.common.domain.MetaColumnInfo; import org.apache.logging.log4j.util.Strings; @@ -206,7 +207,8 @@ private Connection getDBConnection(ConnectMessage connectMessage, String databas if (!connectMessage.extraParams.isEmpty()) { url += "?" + extraParamString; } - return DriverManager.getConnection(url, connectMessage.username, connectMessage.password); + return DriverManager.getConnection( + url, connectMessage.username, AESUtils.isDecryptByConf(connectMessage.password)); } /** Connect message */ diff --git a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/kingbase/SqlConnection.java b/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/kingbase/SqlConnection.java index 74379ab151..a753f41796 100644 --- a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/kingbase/SqlConnection.java +++ b/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/kingbase/SqlConnection.java @@ -18,6 +18,7 @@ package org.apache.linkis.metadata.query.service.kingbase; import org.apache.linkis.common.conf.CommonVars; +import org.apache.linkis.common.utils.AESUtils; import org.apache.linkis.metadata.query.common.domain.MetaColumnInfo; import java.io.Closeable; @@ -221,7 +222,8 @@ private Connection getDBConnection(ConnectMessage connectMessage, String databas url += "?" + extraParamString; } try { - return DriverManager.getConnection(url, connectMessage.username, connectMessage.password); + return DriverManager.getConnection( + url, connectMessage.username, AESUtils.isDecryptByConf(connectMessage.password)); } catch (Exception e) { e.printStackTrace(); throw e; diff --git a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/mysql/SqlConnection.java b/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/mysql/SqlConnection.java index 02172cfdde..d35ae7f2a9 100644 --- a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/mysql/SqlConnection.java +++ b/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/mysql/SqlConnection.java @@ -18,6 +18,7 @@ package org.apache.linkis.metadata.query.service.mysql; import org.apache.linkis.common.conf.CommonVars; +import org.apache.linkis.common.utils.AESUtils; import org.apache.linkis.common.utils.SecurityUtils; import org.apache.linkis.metadata.query.common.domain.MetaColumnInfo; @@ -224,7 +225,8 @@ private Connection getDBConnection(ConnectMessage connectMessage, String databas if (!connectMessage.extraParams.isEmpty()) { url += "?" + extraParamString; } - return DriverManager.getConnection(url, connectMessage.username, connectMessage.password); + return DriverManager.getConnection( + url, connectMessage.username, AESUtils.isDecryptByConf(connectMessage.password)); } /** Connect message */ diff --git a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/oracle/SqlConnection.java b/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/oracle/SqlConnection.java index 890659adcf..142effeedb 100644 --- a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/oracle/SqlConnection.java +++ b/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/oracle/SqlConnection.java @@ -18,6 +18,7 @@ package org.apache.linkis.metadata.query.service.oracle; import org.apache.linkis.common.conf.CommonVars; +import org.apache.linkis.common.utils.AESUtils; import org.apache.linkis.metadata.query.common.domain.MetaColumnInfo; import org.apache.commons.lang3.StringUtils; @@ -239,7 +240,7 @@ private Connection getDBConnection( } Properties prop = new Properties(); prop.put("user", connectMessage.username); - prop.put("password", connectMessage.password); + prop.put("password", AESUtils.isDecryptByConf(connectMessage.password)); prop.put("remarksReporting", "true"); return DriverManager.getConnection(url, prop); } diff --git a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/postgres/SqlConnection.java b/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/postgres/SqlConnection.java index 659ca741c7..f72f7284d3 100644 --- a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/postgres/SqlConnection.java +++ b/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/postgres/SqlConnection.java @@ -18,6 +18,7 @@ package org.apache.linkis.metadata.query.service.postgres; import org.apache.linkis.common.conf.CommonVars; +import org.apache.linkis.common.utils.AESUtils; import org.apache.linkis.metadata.query.common.domain.MetaColumnInfo; import org.apache.logging.log4j.util.Strings; @@ -202,7 +203,8 @@ private Connection getDBConnection(ConnectMessage connectMessage, String databas if (!connectMessage.extraParams.isEmpty()) { url += "?" + extraParamString; } - return DriverManager.getConnection(url, connectMessage.username, connectMessage.password); + return DriverManager.getConnection( + url, connectMessage.username, AESUtils.isDecryptByConf(connectMessage.password)); } /** Connect message */ diff --git a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/sqlserver/SqlConnection.java b/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/sqlserver/SqlConnection.java index b349cb7e32..cb86ab169b 100644 --- a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/sqlserver/SqlConnection.java +++ b/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/sqlserver/SqlConnection.java @@ -18,6 +18,7 @@ package org.apache.linkis.metadata.query.service.sqlserver; import org.apache.linkis.common.conf.CommonVars; +import org.apache.linkis.common.utils.AESUtils; import org.apache.linkis.metadata.query.common.domain.MetaColumnInfo; import java.io.Closeable; @@ -215,7 +216,8 @@ private Connection getDBConnection(ConnectMessage connectMessage, String databas if (!connectMessage.extraParams.isEmpty()) { url += "?" + extraParamString; } - return DriverManager.getConnection(url, connectMessage.username, connectMessage.password); + return DriverManager.getConnection( + url, connectMessage.username, AESUtils.isDecryptByConf(connectMessage.password)); } /** Connect message */ diff --git a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/starrocks/SqlConnection.java b/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/starrocks/SqlConnection.java index bcef0d4f7f..71f29c0d19 100644 --- a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/starrocks/SqlConnection.java +++ b/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/starrocks/SqlConnection.java @@ -18,6 +18,7 @@ package org.apache.linkis.metadata.query.service.starrocks; import org.apache.linkis.common.conf.CommonVars; +import org.apache.linkis.common.utils.AESUtils; import org.apache.linkis.common.utils.SecurityUtils; import org.apache.linkis.metadata.query.common.domain.MetaColumnInfo; @@ -216,7 +217,8 @@ private Connection getDBConnection(ConnectMessage connectMessage, String databas if (!connectMessage.extraParams.isEmpty()) { url += "?" + extraParamString; } - return DriverManager.getConnection(url, connectMessage.username, connectMessage.password); + return DriverManager.getConnection( + url, connectMessage.username, AESUtils.isDecryptByConf(connectMessage.password)); } /** Connect message */ From f39700f61191e269a031744820b0618490adb68f Mon Sep 17 00:00:00 2001 From: v-kkhuang <62878639+v-kkhuang@users.noreply.github.com> Date: Wed, 23 Oct 2024 19:22:13 +0800 Subject: [PATCH 47/90] Add synchronized department user table scheduled tasks (#593) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: “v_kkhuang” <“420895376@qq.com”> --- .../dao/UserDepartmentInfoMapper.java | 41 +++++ .../department/entity/UserDepartmentInfo.java | 150 ++++++++++++++++++ .../scheduled/UserDepartmentInfoSync.java | 103 ++++++++++++ .../common/UserDepartmentInfoMapper.xml | 57 +++++++ ...kisJobHistoryScanSpringConfiguration.scala | 5 + .../linkis/monitor/constants/Constants.scala | 1 + .../monitor/factory/MapperFactory.scala | 11 ++ 7 files changed, 368 insertions(+) create mode 100644 linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/department/dao/UserDepartmentInfoMapper.java create mode 100644 linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/department/entity/UserDepartmentInfo.java create mode 100644 linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scheduled/UserDepartmentInfoSync.java create mode 100644 linkis-extensions/linkis-et-monitor/src/main/resources/mapper/common/UserDepartmentInfoMapper.xml diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/department/dao/UserDepartmentInfoMapper.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/department/dao/UserDepartmentInfoMapper.java new file mode 100644 index 0000000000..9f263136ea --- /dev/null +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/department/dao/UserDepartmentInfoMapper.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.monitor.department.dao; + +import org.apache.linkis.monitor.department.entity.UserDepartmentInfo; + +import org.apache.ibatis.annotations.Mapper; +import org.apache.ibatis.annotations.Param; + +import java.util.List; + +@Mapper +public interface UserDepartmentInfoMapper { + + void insertUser(UserDepartmentInfo user); + + int batchInsertUsers(@Param("userDepartmentInfos") List userDepartmentInfos); + + void updateUser(UserDepartmentInfo user); + + UserDepartmentInfo selectUser(@Param("userName") String userName); + + void deleteUser(); + + List selectAllUsers(); +} diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/department/entity/UserDepartmentInfo.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/department/entity/UserDepartmentInfo.java new file mode 100644 index 0000000000..c5ea27e7ce --- /dev/null +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/department/entity/UserDepartmentInfo.java @@ -0,0 +1,150 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.monitor.department.entity; + +import java.util.Date; + +public class UserDepartmentInfo { + + private String clusterCode; + + private String userType; + private String userName; + private String orgId; + private String orgName; + private String queueName; + private String dbName; + private String interfaceUser; + private String isUnionAnalyse; + private Date createTime; + private String userItsmNo; + + // 构造函数、getter和setter方法 + public UserDepartmentInfo( + String clusterCode, + String userType, + String userName, + String orgId, + String orgName, + String queueName, + String dbName, + String interfaceUser, + String isUnionAnalyse, + Date createTime, + String userItsmNo) { + this.clusterCode = clusterCode; + this.userType = userType; + this.userName = userName; + this.orgId = orgId; + this.orgName = orgName; + this.queueName = queueName; + this.dbName = dbName; + this.interfaceUser = interfaceUser; + this.isUnionAnalyse = isUnionAnalyse; + this.createTime = createTime; + this.userItsmNo = userItsmNo; + } + + public String getClusterCode() { + return clusterCode; + } + + public void setClusterCode(String clusterCode) { + this.clusterCode = clusterCode; + } + + public String getUserType() { + return userType; + } + + public void setUserType(String userType) { + this.userType = userType; + } + + public String getUserName() { + return userName; + } + + public void setUserName(String userName) { + this.userName = userName; + } + + public String getOrgId() { + return orgId; + } + + public void setOrgId(String orgId) { + this.orgId = orgId; + } + + public String getOrgName() { + return orgName; + } + + public void setOrgName(String orgName) { + this.orgName = orgName; + } + + public String getQueueName() { + return queueName; + } + + public void setQueueName(String queueName) { + this.queueName = queueName; + } + + public String getDbName() { + return dbName; + } + + public void setDbName(String dbName) { + this.dbName = dbName; + } + + public String getInterfaceUser() { + return interfaceUser; + } + + public void setInterfaceUser(String interfaceUser) { + this.interfaceUser = interfaceUser; + } + + public String getIsUnionAnalyse() { + return isUnionAnalyse; + } + + public void setIsUnionAnalyse(String isUnionAnalyse) { + this.isUnionAnalyse = isUnionAnalyse; + } + + public Date getCreateTime() { + return createTime; + } + + public void setCreateTime(Date createTime) { + this.createTime = createTime; + } + + public String getUserItsmNo() { + return userItsmNo; + } + + public void setUserItsmNo(String userItsmNo) { + this.userItsmNo = userItsmNo; + } +} diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scheduled/UserDepartmentInfoSync.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scheduled/UserDepartmentInfoSync.java new file mode 100644 index 0000000000..a869e7607f --- /dev/null +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scheduled/UserDepartmentInfoSync.java @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.monitor.scheduled; + +import org.apache.linkis.monitor.constants.Constants; +import org.apache.linkis.monitor.department.dao.UserDepartmentInfoMapper; +import org.apache.linkis.monitor.department.entity.UserDepartmentInfo; +import org.apache.linkis.monitor.factory.MapperFactory; +import org.apache.linkis.monitor.utils.alert.AlertDesc; +import org.apache.linkis.monitor.utils.alert.ims.MonitorAlertUtils; +import org.apache.linkis.monitor.utils.alert.ims.PooledImsAlertUtils; + +import org.apache.commons.lang3.StringUtils; + +import org.springframework.context.annotation.PropertySource; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; +import org.springframework.util.CollectionUtils; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@Component +@PropertySource(value = "classpath:linkis-et-monitor.properties", encoding = "UTF-8") +public class UserDepartmentInfoSync { + + private static final Logger logger = LoggerFactory.getLogger(ResourceMonitor.class); + + @Scheduled(cron = "${linkis.monitor.org.user.sync.cron:0 0 0 0/7 * ?}") + public static void DepartmentInfoSync() { + + UserDepartmentInfoMapper userDepartmentInfoMapper = MapperFactory.getUserDepartmentInfoMapper(); + // 获取linkis_org_user_sync信息 + List userDepartmentInfos = userDepartmentInfoMapper.selectAllUsers(); + + if (CollectionUtils.isEmpty(userDepartmentInfos)) { + logger.info("No user department info to sync"); + // 并且发送告警通知 + return; + } else { + logger.info("Start to sync user department info"); + + List alterList = + userDepartmentInfos.stream() + .filter( + userDepartmentInfo -> + StringUtils.isNotBlank(userDepartmentInfo.getUserName()) + && (StringUtils.isBlank(userDepartmentInfo.getOrgId()) + || StringUtils.isBlank(userDepartmentInfo.getOrgName()))) + .collect(Collectors.toList()); + List syncList = + userDepartmentInfos.stream() + .filter( + userDepartmentInfo -> + StringUtils.isNotBlank(userDepartmentInfo.getUserName()) + && StringUtils.isNotBlank(userDepartmentInfo.getOrgId()) + && StringUtils.isNotBlank(userDepartmentInfo.getOrgName())) + .collect(Collectors.toList()); + if (!CollectionUtils.isEmpty(alterList)) { + // 统计异常名称，然后发送告警 + String usernames = + alterList.stream() + .filter(s -> StringUtils.isNotBlank(s.getUserName())) + .map(UserDepartmentInfo::getUserName) + .limit(5) + .collect(Collectors.joining(",")); + if (StringUtils.isNotBlank(usernames)) { + HashMap replaceParm = new HashMap<>(); + replaceParm.put("$user", usernames); + replaceParm.put("$count", String.valueOf(alterList.size())); + Map ecmResourceAlerts = + MonitorAlertUtils.getAlerts(Constants.DEPARTMENT_USER_IM(), replaceParm); + PooledImsAlertUtils.addAlert(ecmResourceAlerts.get("12019")); + } + } + if (!CollectionUtils.isEmpty(syncList)) { + // 删除org_user数据，再同步 + userDepartmentInfoMapper.deleteUser(); + userDepartmentInfoMapper.batchInsertUsers(syncList); + } + } + } +} diff --git a/linkis-extensions/linkis-et-monitor/src/main/resources/mapper/common/UserDepartmentInfoMapper.xml b/linkis-extensions/linkis-et-monitor/src/main/resources/mapper/common/UserDepartmentInfoMapper.xml new file mode 100644 index 0000000000..295cee560f --- /dev/null +++ b/linkis-extensions/linkis-et-monitor/src/main/resources/mapper/common/UserDepartmentInfoMapper.xml @@ -0,0 +1,57 @@ + + + + + + + + INSERT INTO linkis_org_user(cluster_code, user_type, user_name, org_id, org_name, queue_name, db_name, interface_user, is_union_analyse, create_time, user_itsm_no) + VALUES (#{clusterCode}, #{userType}, #{userName}, #{orgId}, #{orgName}, #{queueName}, #{dbName}, #{interfaceUser}, #{isUnionAnalyse}, #{createTime}, #{userItsmNo}) + + + + + INSERT INTO linkis_org_user (cluster_code, user_type, user_name, org_id, org_name, queue_name, db_name, interface_user, is_union_analyse, create_time, user_itsm_no) + VALUES + + (#{user.clusterCode}, #{user.userType}, #{user.userName}, #{user.orgId}, #{user.orgName}, #{user.queueName}, #{user.dbName}, #{user.interfaceUser}, #{user.isUnionAnalyse}, #{user.createTime}, #{user.userItsmNo}) + + + + + + UPDATE linkis_org_user + + cluster_code = #{clusterCode}, + user_type = #{userType}, + org_id = #{orgId}, + org_name = #{orgName}, + queue_name = #{queueName}, + db_name = #{dbName}, + interface_user = #{interfaceUser}, + is_union_analyse = #{isUnionAnalyse}, + create_time = #{createTime}, + user_itsm_no = #{userItsmNo}, + + WHERE user_name = #{userName} + + + + + + + + DELETE FROM linkis_org_user + + + + + + + + + diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/LinkisJobHistoryScanSpringConfiguration.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/LinkisJobHistoryScanSpringConfiguration.scala index e3652306c1..c6b2b04d59 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/LinkisJobHistoryScanSpringConfiguration.scala +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/LinkisJobHistoryScanSpringConfiguration.scala @@ -17,6 +17,7 @@ package org.apache.linkis.monitor +import org.apache.linkis.monitor.department.dao.UserDepartmentInfoMapper import org.apache.linkis.monitor.factory.MapperFactory import org.apache.linkis.monitor.instance.dao.InstanceInfoDao import org.apache.linkis.monitor.jobhistory.dao.JobHistoryMapper @@ -39,10 +40,14 @@ class LinkisJobHistoryScanSpringConfiguration { @Autowired private var instanceInfoMapper: InstanceInfoDao = _ + @Autowired + private var userDepartmentInfoMapper: UserDepartmentInfoMapper = _ + @PostConstruct def init(): Unit = { MapperFactory.setJobHistoryMapper(jobHistoryMapper) MapperFactory.setInstanceInfoMapper(instanceInfoMapper) + MapperFactory.setUserDepartmentInfoMapper(userDepartmentInfoMapper) } } diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/constants/Constants.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/constants/Constants.scala index 9da7e35ebd..a418bcbe1e 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/constants/Constants.scala +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/constants/Constants.scala @@ -79,6 +79,7 @@ object Constants { val BML_CLEAR_IM = "bml.clear.monitor.im." val THREAD_TIME_OUT_IM = "thread.monitor.timeout.im." val JOB_RESULT_IM = "jobhistory.result.monitor.im." + val DEPARTMENT_USER_IM = "department.user.sync.im." val BML_VERSION_MAX_NUM: CommonVars[Int] = CommonVars[Int]("linkis.monitor.bml.cleaner.version.max.num", 50) diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/factory/MapperFactory.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/factory/MapperFactory.scala index eb503c52aa..3f81c66514 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/factory/MapperFactory.scala +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/factory/MapperFactory.scala @@ -17,6 +17,7 @@ package org.apache.linkis.monitor.factory +import org.apache.linkis.monitor.department.dao.UserDepartmentInfoMapper import org.apache.linkis.monitor.instance.dao.{ InsLabelRelationDao, InstanceInfoDao, @@ -34,6 +35,8 @@ object MapperFactory { private var instanceLabelRelationMapper: InsLabelRelationDao = _ + private var userDepartmentInfoMapper: UserDepartmentInfoMapper = _ + def getJobHistoryMapper(): JobHistoryMapper = jobHistoryMapper def setJobHistoryMapper(jobHistoryMapper: JobHistoryMapper): Unit = { @@ -58,4 +61,12 @@ object MapperFactory { MapperFactory.instanceLabelRelationMapper = instanceLabelRelationMapper } + // 获取userDepartmentInfoMapper的值 + def getUserDepartmentInfoMapper: UserDepartmentInfoMapper = userDepartmentInfoMapper + + // 设置userDepartmentInfoMapper的值 + def setUserDepartmentInfoMapper(mapper: UserDepartmentInfoMapper): Unit = { + userDepartmentInfoMapper = mapper + } + } From 18feb944c98c0119064c9a01d236dde1bcf2bec4 Mon Sep 17 00:00:00 2001 From: v-kkhuang <62878639+v-kkhuang@users.noreply.github.com> Date: Wed, 23 Oct 2024 19:23:37 +0800 Subject: [PATCH 48/90] [1.9.0]Add department level result set restriction configuration and inspection (#617) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 添加部门级别的结果集限制配置和检查 - 在 Configuration.scala 中添加 JOB_RESULT_DEPARTMENT_LIMIT 变量，用于配置部门结果集限制- 实现 canResultSetByDepartment 方法，根据部门 ID 判断是否有结果集权限 - 在 QueryRestfulApi.java 中调用 canResultSetByDepartment 方法，将结果加入 API响应 * Update Configuration.scala update default department id --------- Co-authored-by: “v_kkhuang” <“420895376@qq.com”> Co-authored-by: Casion --- .../org/apache/linkis/common/conf/Configuration.scala | 8 ++++++++ .../linkis/jobhistory/restful/api/QueryRestfulApi.java | 2 ++ 2 files changed, 10 insertions(+) diff --git a/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/conf/Configuration.scala b/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/conf/Configuration.scala index 16cac1d204..fbcda761ee 100644 --- a/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/conf/Configuration.scala +++ b/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/conf/Configuration.scala @@ -67,6 +67,9 @@ object Configuration extends Logging { val JOB_HISTORY_DEPARTMENT_ADMIN = CommonVars("wds.linkis.jobhistory.department.admin", "hadoop") + val JOB_RESULT_DEPARTMENT_LIMIT = + CommonVars("linkis.jobhistory.result.limit.department", "") + // Only the specified token has permission to call some api val GOVERNANCE_STATION_ADMIN_TOKEN_STARTWITH = "ADMIN-" @@ -159,4 +162,9 @@ object Configuration extends Logging { case _ => creator } + def canResultSetByDepartment(departmentId: String): Boolean = { + val jobResultLimit = JOB_RESULT_DEPARTMENT_LIMIT.getHotValue.split(",") + !jobResultLimit.exists(departmentId.equalsIgnoreCase) + } + } diff --git a/linkis-public-enhancements/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/restful/api/QueryRestfulApi.java b/linkis-public-enhancements/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/restful/api/QueryRestfulApi.java index 9aaba5a06b..e9860af298 100644 --- a/linkis-public-enhancements/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/restful/api/QueryRestfulApi.java +++ b/linkis-public-enhancements/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/restful/api/QueryRestfulApi.java @@ -85,10 +85,12 @@ public class QueryRestfulApi { @RequestMapping(path = "/governanceStationAdmin", method = RequestMethod.GET) public Message governanceStationAdmin(HttpServletRequest req) { String username = ModuleUserUtils.getOperationUser(req, "governanceStationAdmin"); + String departmentId = JobhistoryUtils.getDepartmentByuser(username); return Message.ok() .data("admin", Configuration.isAdmin(username)) .data("historyAdmin", Configuration.isJobHistoryAdmin(username)) .data("deptAdmin", Configuration.isDepartmentAdmin(username)) + .data("canResultSet", Configuration.canResultSetByDepartment(departmentId)) .data("errorMsgTip", Configuration.ERROR_MSG_TIP().getValue()); } From 4605855d2ec2db2a420dea291bb5f5ff2e16b9e1 Mon Sep 17 00:00:00 2001 From: gelxiogong <15927047952@163.com> Date: Wed, 21 Aug 2024 16:01:21 +0800 Subject: [PATCH 49/90] change to yarn proxy url --- .../YarnApplicationClusterDescriptorAdapter.java | 7 ++++++- .../deployment/YarnSessionClusterDescriptorAdapter.scala | 7 ++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/linkis-engineconn-plugins/flink/src/main/java/org/apache/linkis/engineconnplugin/flink/client/deployment/YarnApplicationClusterDescriptorAdapter.java b/linkis-engineconn-plugins/flink/src/main/java/org/apache/linkis/engineconnplugin/flink/client/deployment/YarnApplicationClusterDescriptorAdapter.java index f9756cc513..aa87739f03 100644 --- a/linkis-engineconn-plugins/flink/src/main/java/org/apache/linkis/engineconnplugin/flink/client/deployment/YarnApplicationClusterDescriptorAdapter.java +++ b/linkis-engineconn-plugins/flink/src/main/java/org/apache/linkis/engineconnplugin/flink/client/deployment/YarnApplicationClusterDescriptorAdapter.java @@ -17,6 +17,8 @@ package org.apache.linkis.engineconnplugin.flink.client.deployment; +import org.apache.hadoop.yarn.api.records.ApplicationReport; +import org.apache.hadoop.yarn.client.api.YarnClient; import org.apache.linkis.engineconnplugin.flink.client.context.ExecutionContext; import org.apache.linkis.engineconnplugin.flink.exception.JobExecutionException; @@ -45,13 +47,16 @@ public void deployCluster(String[] programArguments, String applicationClassName .getClusterClientFactory() .getClusterSpecification(this.executionContext.getFlinkConfig()); YarnClusterDescriptor clusterDescriptor = this.executionContext.createClusterDescriptor(); + YarnClient yarnClient = clusterDescriptor.getYarnClient(); try { ClusterClientProvider clusterClientProvider = clusterDescriptor.deployApplicationCluster( clusterSpecification, applicationConfiguration); clusterClient = clusterClientProvider.getClusterClient(); super.clusterID = clusterClient.getClusterId(); - super.webInterfaceUrl = clusterClient.getWebInterfaceURL(); + ApplicationReport appReport = yarnClient.getApplicationReport(clusterClient.getClusterId()); + super.webInterfaceUrl = appReport.getTrackingUrl(); +// super.webInterfaceUrl = clusterClient.getWebInterfaceURL(); } catch (Exception e) { throw new JobExecutionException(ExceptionUtils.getRootCauseMessage(e), e); } diff --git a/linkis-engineconn-plugins/flink/src/main/java/org/apache/linkis/engineconnplugin/flink/client/deployment/YarnSessionClusterDescriptorAdapter.scala b/linkis-engineconn-plugins/flink/src/main/java/org/apache/linkis/engineconnplugin/flink/client/deployment/YarnSessionClusterDescriptorAdapter.scala index a587bb0f71..d720e9865e 100644 --- a/linkis-engineconn-plugins/flink/src/main/java/org/apache/linkis/engineconnplugin/flink/client/deployment/YarnSessionClusterDescriptorAdapter.scala +++ b/linkis-engineconn-plugins/flink/src/main/java/org/apache/linkis/engineconnplugin/flink/client/deployment/YarnSessionClusterDescriptorAdapter.scala @@ -19,6 +19,8 @@ package org.apache.linkis.engineconnplugin.flink.client.deployment import org.apache.linkis.engineconnplugin.flink.client.context.ExecutionContext import org.apache.flink.api.common.JobID +import org.apache.hadoop.yarn.api.records.ApplicationReport +import org.apache.hadoop.yarn.client.api.YarnClient class YarnSessionClusterDescriptorAdapter(executionContext: ExecutionContext) extends YarnPerJobClusterDescriptorAdapter(executionContext) { @@ -27,9 +29,12 @@ class YarnSessionClusterDescriptorAdapter(executionContext: ExecutionContext) ex val clusterSpecification = this.executionContext.getClusterClientFactory.getClusterSpecification(this.executionContext.getFlinkConfig) val clusterDescriptor = this.executionContext.createClusterDescriptor val clusterClientProvider = clusterDescriptor.deploySessionCluster(clusterSpecification) + val yarnClient = clusterDescriptor.getYarnClient clusterClient = clusterClientProvider.getClusterClient clusterID = clusterClient.getClusterId - webInterfaceUrl = clusterClient.getWebInterfaceURL + val appReport = yarnClient.getApplicationReport(clusterClient.getClusterId) + super.webInterfaceUrl = appReport.getTrackingUrl +// webInterfaceUrl = clusterClient.getWebInterfaceURL bindApplicationId() } From 3eaca161b73ddaac76ff1a5a3e9db0b3a55959eb Mon Sep 17 00:00:00 2001 From: Alexkun Date: Wed, 23 Oct 2024 20:50:14 +0800 Subject: [PATCH 50/90] fix compile error --- .../deployment/YarnApplicationClusterDescriptorAdapter.java | 6 +++--- .../deployment/YarnSessionClusterDescriptorAdapter.scala | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/linkis-engineconn-plugins/flink/src/main/java/org/apache/linkis/engineconnplugin/flink/client/deployment/YarnApplicationClusterDescriptorAdapter.java b/linkis-engineconn-plugins/flink/src/main/java/org/apache/linkis/engineconnplugin/flink/client/deployment/YarnApplicationClusterDescriptorAdapter.java index aa87739f03..ae421dc1db 100644 --- a/linkis-engineconn-plugins/flink/src/main/java/org/apache/linkis/engineconnplugin/flink/client/deployment/YarnApplicationClusterDescriptorAdapter.java +++ b/linkis-engineconn-plugins/flink/src/main/java/org/apache/linkis/engineconnplugin/flink/client/deployment/YarnApplicationClusterDescriptorAdapter.java @@ -17,8 +17,6 @@ package org.apache.linkis.engineconnplugin.flink.client.deployment; -import org.apache.hadoop.yarn.api.records.ApplicationReport; -import org.apache.hadoop.yarn.client.api.YarnClient; import org.apache.linkis.engineconnplugin.flink.client.context.ExecutionContext; import org.apache.linkis.engineconnplugin.flink.exception.JobExecutionException; @@ -28,6 +26,8 @@ import org.apache.flink.client.program.ClusterClientProvider; import org.apache.flink.yarn.YarnClusterDescriptor; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ApplicationReport; +import org.apache.hadoop.yarn.client.api.YarnClient; import java.util.Objects; import java.util.concurrent.TimeUnit; @@ -56,7 +56,7 @@ public void deployCluster(String[] programArguments, String applicationClassName super.clusterID = clusterClient.getClusterId(); ApplicationReport appReport = yarnClient.getApplicationReport(clusterClient.getClusterId()); super.webInterfaceUrl = appReport.getTrackingUrl(); -// super.webInterfaceUrl = clusterClient.getWebInterfaceURL(); + // super.webInterfaceUrl = clusterClient.getWebInterfaceURL(); } catch (Exception e) { throw new JobExecutionException(ExceptionUtils.getRootCauseMessage(e), e); } diff --git a/linkis-engineconn-plugins/flink/src/main/java/org/apache/linkis/engineconnplugin/flink/client/deployment/YarnSessionClusterDescriptorAdapter.scala b/linkis-engineconn-plugins/flink/src/main/java/org/apache/linkis/engineconnplugin/flink/client/deployment/YarnSessionClusterDescriptorAdapter.scala index d720e9865e..7fdc49fa2d 100644 --- a/linkis-engineconn-plugins/flink/src/main/java/org/apache/linkis/engineconnplugin/flink/client/deployment/YarnSessionClusterDescriptorAdapter.scala +++ b/linkis-engineconn-plugins/flink/src/main/java/org/apache/linkis/engineconnplugin/flink/client/deployment/YarnSessionClusterDescriptorAdapter.scala @@ -33,7 +33,7 @@ class YarnSessionClusterDescriptorAdapter(executionContext: ExecutionContext) ex clusterClient = clusterClientProvider.getClusterClient clusterID = clusterClient.getClusterId val appReport = yarnClient.getApplicationReport(clusterClient.getClusterId) - super.webInterfaceUrl = appReport.getTrackingUrl + webInterfaceUrl = appReport.getTrackingUrl // webInterfaceUrl = clusterClient.getWebInterfaceURL bindApplicationId() } From af9bf53fb73740853262b9e839bf0ab6fd2eb126 Mon Sep 17 00:00:00 2001 From: peacewong Date: Thu, 24 Oct 2024 16:25:57 +0800 Subject: [PATCH 51/90] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E9=87=8D=E5=A4=8D?= =?UTF-8?q?=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../engine/DefaultEngineCreateService.scala | 14 +++------ .../service/common/label/LabelChecker.scala | 31 ------------------- 2 files changed, 4 insertions(+), 41 deletions(-) delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/service/common/label/LabelChecker.scala diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/DefaultEngineCreateService.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/DefaultEngineCreateService.scala index c7880a56cf..6f72c74e9b 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/DefaultEngineCreateService.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/DefaultEngineCreateService.scala @@ -26,7 +26,7 @@ import org.apache.linkis.governance.common.conf.GovernanceCommonConf.ENGINE_CONN import org.apache.linkis.governance.common.utils.JobUtils import org.apache.linkis.manager.am.conf.{AMConfiguration, EngineConnConfigurationService} import org.apache.linkis.manager.am.exception.AMErrorException -import org.apache.linkis.manager.am.label.EngineReuseLabelChooser +import org.apache.linkis.manager.am.label.{EngineReuseLabelChooser, LabelChecker} import org.apache.linkis.manager.am.selector.{ECAvailableRule, NodeSelector} import org.apache.linkis.manager.am.vo.CanCreateECRes import org.apache.linkis.manager.common.constant.AMConstant @@ -35,10 +35,7 @@ import org.apache.linkis.manager.common.entity.node.{EMNode, EngineNode} import org.apache.linkis.manager.common.entity.resource.NodeResource import org.apache.linkis.manager.common.protocol.engine.{EngineCreateRequest, EngineStopRequest} import org.apache.linkis.manager.common.utils.ManagerUtils -import org.apache.linkis.manager.engineplugin.common.launch.entity.{ - EngineConnBuildRequestImpl, - EngineConnCreationDescImpl -} +import org.apache.linkis.manager.engineplugin.common.launch.entity.{EngineConnBuildRequestImpl, EngineConnCreationDescImpl} import org.apache.linkis.manager.engineplugin.common.resource.TimeoutEngineResourceRequest import org.apache.linkis.manager.label.builder.factory.LabelBuilderFactoryContext import org.apache.linkis.manager.label.entity.{EngineNodeLabel, Label} @@ -49,19 +46,16 @@ import org.apache.linkis.manager.label.utils.LabelUtils import org.apache.linkis.manager.persistence.NodeMetricManagerPersistence import org.apache.linkis.manager.rm.{AvailableResource, NotEnoughResource} import org.apache.linkis.manager.rm.service.ResourceManager -import org.apache.linkis.manager.service.common.label.{LabelChecker, LabelFilter} +import org.apache.linkis.manager.service.common.label.LabelFilter import org.apache.linkis.rpc.Sender import org.apache.linkis.rpc.message.annotation.Receiver import org.apache.linkis.server.BDPJettyServerHelper - import org.apache.commons.lang3.StringUtils - import org.springframework.beans.factory.annotation.Autowired import org.springframework.stereotype.Service import java.util -import java.util.concurrent.{TimeoutException, TimeUnit} - +import java.util.concurrent.{TimeUnit, TimeoutException} import scala.collection.JavaConverters._ import scala.concurrent.duration.Duration diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/service/common/label/LabelChecker.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/service/common/label/LabelChecker.scala deleted file mode 100644 index 779b06963b..0000000000 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/service/common/label/LabelChecker.scala +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.manager.service.common.label - -import org.apache.linkis.manager.label.entity.Label - -import java.util - -trait LabelChecker { - - def checkEngineLabel(labelList: util.List[Label[_]]): Boolean - - def checkEMLabel(labelList: util.List[Label[_]]): Boolean - - def checkCorrespondingLabel(labelList: util.List[Label[_]], clazz: Class[_]*): Boolean -} From b45a4c7267a81985557712aa963d0b311b516d44 Mon Sep 17 00:00:00 2001 From: v-kkhuang <62878639+v-kkhuang@users.noreply.github.com> Date: Fri, 25 Oct 2024 14:35:26 +0800 Subject: [PATCH 52/90] [1.9.0]Fix Hive dependency issues (#618) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix hive engine error * Security work order repair * sync ddl --------- Co-authored-by: “v_kkhuang” <“420895376@qq.com”> --- linkis-dist/package/db/linkis_ddl.sql | 21 ++++++++++++++++++- .../upgrade/1.4.1_schema/mysql/linkis_ddl.sql | 21 ++++++++++++++++++- linkis-engineconn-plugins/flink/pom.xml | 17 ++++++++++++--- linkis-engineconn-plugins/hive/pom.xml | 17 ++++++++++++++- .../executor/HiveEngineConnExecutor.scala | 6 ++++-- 5 files changed, 74 insertions(+), 8 deletions(-) diff --git a/linkis-dist/package/db/linkis_ddl.sql b/linkis-dist/package/db/linkis_ddl.sql index 47ea5ab5b2..e1e35b74f9 100644 --- a/linkis-dist/package/db/linkis_ddl.sql +++ b/linkis-dist/package/db/linkis_ddl.sql @@ -1070,4 +1070,23 @@ CREATE TABLE `linkis_cg_tenant_department_config` ( `is_valid` varchar(1) COLLATE utf8_bin NOT NULL DEFAULT 'Y' COMMENT '是否有效', PRIMARY KEY (`id`), UNIQUE KEY `uniq_creator_department` (`creator`,`department`) -) ENGINE=InnoDB AUTO_INCREMENT=0 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; \ No newline at end of file +) ENGINE=InnoDB AUTO_INCREMENT=0 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; + +-- ---------------------------- +-- Table structure for linkis_org_user_sync +-- ---------------------------- +DROP TABLE IF EXISTS `linkis_org_user_sync`; +CREATE TABLE `linkis_org_user_sync` ( + `cluster_code` varchar(16) COMMENT '集群', + `user_type` varchar(64) COMMENT '用户类型', + `user_name` varchar(128) COMMENT '授权用户', + `org_id` varchar(16) COMMENT '部门ID', + `org_name` varchar(64) COMMENT '部门名字', + `queue_name` varchar(64) COMMENT '默认资源队列', + `db_name` varchar(64) COMMENT '默认操作数据库', + `interface_user` varchar(64) COMMENT '接口人', + `is_union_analyse` varchar(64) COMMENT '是否联合分析人', + `create_time` varchar(64) COMMENT '用户创建时间', + `user_itsm_no` varchar(64) COMMENT '用户创建单号', + PRIMARY KEY (`user_name`) +) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4 COLLATE=utf8mb4_bin COMMENT ='用户部门统计INC表'; \ No newline at end of file diff --git a/linkis-dist/package/db/upgrade/1.4.1_schema/mysql/linkis_ddl.sql b/linkis-dist/package/db/upgrade/1.4.1_schema/mysql/linkis_ddl.sql index f9b29a040f..33f663e598 100644 --- a/linkis-dist/package/db/upgrade/1.4.1_schema/mysql/linkis_ddl.sql +++ b/linkis-dist/package/db/upgrade/1.4.1_schema/mysql/linkis_ddl.sql @@ -74,4 +74,23 @@ ALTER TABLE linkis_cg_manager_service_instance_metrics ADD COLUMN description va ALTER TABLE linkis_ps_bml_resources_task ADD CONSTRAINT uniq_rid_version UNIQUE (`resource_id`, `version`); ALTER TABLE linkis_cg_ec_resource_info_record ADD UNIQUE INDEX uniq_sinstance_status_cuser_ctime (`service_instance`, `status`, `create_user`, `create_time`); -ALTER TABLE linkis_cg_manager_service_instance ADD COLUMN params text COLLATE utf8_bin DEFAULT NULL; \ No newline at end of file +ALTER TABLE linkis_cg_manager_service_instance ADD COLUMN params text COLLATE utf8_bin DEFAULT NULL; + +-- ---------------------------- +-- Table structure for linkis_org_user_sync +-- ---------------------------- +DROP TABLE IF EXISTS `linkis_org_user_sync`; +CREATE TABLE `linkis_org_user_sync` ( + `cluster_code` varchar(16) COMMENT '集群', + `user_type` varchar(64) COMMENT '用户类型', + `user_name` varchar(128) COMMENT '授权用户', + `org_id` varchar(16) COMMENT '部门ID', + `org_name` varchar(64) COMMENT '部门名字', + `queue_name` varchar(64) COMMENT '默认资源队列', + `db_name` varchar(64) COMMENT '默认操作数据库', + `interface_user` varchar(64) COMMENT '接口人', + `is_union_analyse` varchar(64) COMMENT '是否联合分析人', + `create_time` varchar(64) COMMENT '用户创建时间', + `user_itsm_no` varchar(64) COMMENT '用户创建单号', + PRIMARY KEY (`user_name`) +) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4 COLLATE=utf8mb4_bin COMMENT ='用户部门统计INC表'; \ No newline at end of file diff --git a/linkis-engineconn-plugins/flink/pom.xml b/linkis-engineconn-plugins/flink/pom.xml index 6001b41d43..1416185d6a 100644 --- a/linkis-engineconn-plugins/flink/pom.xml +++ b/linkis-engineconn-plugins/flink/pom.xml @@ -496,6 +496,12 @@ org.codehaus.jackson jackson-jaxrs ${jackson.version} + + + org.codehaus.jackson + jackson-mapper-asl + + @@ -508,12 +514,17 @@ org.codehaus.jackson jackson-xc ${jackson.version} + + + org.codehaus.jackson + jackson-mapper-asl + + - org.codehaus.jackson - jackson-mapper-asl - 1.9.13 + com.fasterxml.jackson.core + jackson-databind diff --git a/linkis-engineconn-plugins/hive/pom.xml b/linkis-engineconn-plugins/hive/pom.xml index 5cdf29b3a9..261dca1bad 100644 --- a/linkis-engineconn-plugins/hive/pom.xml +++ b/linkis-engineconn-plugins/hive/pom.xml @@ -337,6 +337,12 @@ org.codehaus.jackson jackson-jaxrs ${jackson.version} + + + org.codehaus.jackson + jackson-mapper-asl + + @@ -349,8 +355,17 @@ org.codehaus.jackson jackson-xc ${jackson.version} + + + org.codehaus.jackson + jackson-mapper-asl + + + + + com.fasterxml.jackson.core + jackson-databind - diff --git a/linkis-engineconn-plugins/hive/src/main/scala/org/apache/linkis/engineplugin/hive/executor/HiveEngineConnExecutor.scala b/linkis-engineconn-plugins/hive/src/main/scala/org/apache/linkis/engineplugin/hive/executor/HiveEngineConnExecutor.scala index f533d57d6a..5499cb3d62 100644 --- a/linkis-engineconn-plugins/hive/src/main/scala/org/apache/linkis/engineplugin/hive/executor/HiveEngineConnExecutor.scala +++ b/linkis-engineconn-plugins/hive/src/main/scala/org/apache/linkis/engineplugin/hive/executor/HiveEngineConnExecutor.scala @@ -52,6 +52,7 @@ import org.apache.linkis.storage.domain.{Column, DataType} import org.apache.linkis.storage.resultset.ResultSetFactory import org.apache.linkis.storage.resultset.table.{TableMetaData, TableRecord} +import org.apache.commons.collections.MapUtils import org.apache.commons.lang3.StringUtils import org.apache.hadoop.hive.common.HiveInterruptUtils import org.apache.hadoop.hive.conf.HiveConf @@ -139,10 +140,11 @@ class HiveEngineConnExecutor( engineExecutorContext: EngineExecutionContext, code: String ): ExecuteResponse = { - readResByObject = engineExecutorContext.getProperties.getOrDefault( + readResByObject = MapUtils.getBoolean( + engineExecutorContext.getProperties, JobRequestConstants.LINKIS_HIVE_EC_READ_RESULT_BY_OBJECT, false - ).asInstanceOf[Boolean] + ) if (readResByObject) { hiveConf.set( "list.sink.output.formatter", From 4eef5d47e5c602ea21d508f0ce016daf2ab694ea Mon Sep 17 00:00:00 2001 From: peacewong Date: Fri, 25 Oct 2024 17:55:07 +0800 Subject: [PATCH 53/90] Fix get tables only get table bug --- .../org/apache/linkis/ujes/jdbc/UJESSQLDatabaseMetaData.scala | 2 ++ 1 file changed, 2 insertions(+) diff --git a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLDatabaseMetaData.scala b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLDatabaseMetaData.scala index 25e94c5370..e7e1df7498 100644 --- a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLDatabaseMetaData.scala +++ b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLDatabaseMetaData.scala @@ -392,6 +392,8 @@ class UJESSQLDatabaseMetaData(ujesSQLConnection: LinkisSQLConnection) StringUtils.isNotBlank(tableNamePattern) && tableNamePattern.equalsIgnoreCase(tableName) ) { resultTables.add(resultTable) + } else { + resultTables.add(resultTable) } } } From c934f0b93d49ba0b9d08aae22553ce27b4ba6dcc Mon Sep 17 00:00:00 2001 From: v-kkhuang <62878639+v-kkhuang@users.noreply.github.com> Date: Mon, 28 Oct 2024 11:19:34 +0800 Subject: [PATCH 54/90] fix datasource error (#620) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: “v_kkhuang” <“420895376@qq.com”> --- .../datasources/csv/DolphinToSpark.scala | 2 +- .../core/restful/DataSourceCoreRestfulApi.java | 15 +++++++++++++++ .../core/restful/RestfulApiHelper.java | 1 + 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/spark/sql/execution/datasources/csv/DolphinToSpark.scala b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/spark/sql/execution/datasources/csv/DolphinToSpark.scala index 1a0eecc794..e9411e45c3 100644 --- a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/spark/sql/execution/datasources/csv/DolphinToSpark.scala +++ b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/spark/sql/execution/datasources/csv/DolphinToSpark.scala @@ -77,7 +77,7 @@ object DolphinToSpark { case wds.BigIntType => LongType case wds.FloatType => FloatType case wds.DoubleType => DoubleType - case wds.DecimalType => DecimalType(bigDecimalPrecision, bigDecimalScale) + case wds.DecimalType(dataType, 3) => DecimalType(bigDecimalPrecision, bigDecimalScale) case wds.DateType => DateType // case wds.TimestampType => TimestampType case wds.BinaryType => BinaryType diff --git a/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/server/src/main/java/org/apache/linkis/datasourcemanager/core/restful/DataSourceCoreRestfulApi.java b/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/server/src/main/java/org/apache/linkis/datasourcemanager/core/restful/DataSourceCoreRestfulApi.java index 33f6e7ef34..350fffe010 100644 --- a/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/server/src/main/java/org/apache/linkis/datasourcemanager/core/restful/DataSourceCoreRestfulApi.java +++ b/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/server/src/main/java/org/apache/linkis/datasourcemanager/core/restful/DataSourceCoreRestfulApi.java @@ -190,6 +190,8 @@ public Message insertJsonInfo(@RequestBody DataSource dataSource, HttpServletReq AESUtils.encrypt( connectParams.get("password").toString(), AESUtils.LINKIS_DATASOURCE_AES_KEY.getValue())); + // 标记密码已经加密 + dataSource.getConnectParams().put("isEncrypt", "1"); } insertDataSource(dataSource); return Message.ok().data("insertId", dataSource.getId()); @@ -270,6 +272,19 @@ public Message updateDataSourceInJson( dataSource.setKeyDefinitions(keyDefinitionList); Map connectParams = dataSource.getConnectParams(); + + if (AESUtils.LINKIS_DATASOURCE_AES_SWITCH.getValue()) { + dataSource + .getConnectParams() + .replace( + "password", + AESUtils.encrypt( + connectParams.get("password").toString(), + AESUtils.LINKIS_DATASOURCE_AES_KEY.getValue())); + // 标记密码已经加密 + dataSource.getConnectParams().put("isEncrypt", "1"); + } + // add default value filed keyDefinitionList.forEach( keyDefinition -> { diff --git a/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/server/src/main/java/org/apache/linkis/datasourcemanager/core/restful/RestfulApiHelper.java b/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/server/src/main/java/org/apache/linkis/datasourcemanager/core/restful/RestfulApiHelper.java index a57141e167..5e1b3d99ba 100644 --- a/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/server/src/main/java/org/apache/linkis/datasourcemanager/core/restful/RestfulApiHelper.java +++ b/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/server/src/main/java/org/apache/linkis/datasourcemanager/core/restful/RestfulApiHelper.java @@ -77,6 +77,7 @@ public static void encryptPasswordKey( if (AESUtils.LINKIS_DATASOURCE_AES_SWITCH.getValue()) { passwordStr = AESUtils.encrypt(passwordStr, AESUtils.LINKIS_DATASOURCE_AES_KEY.getValue()); + connectParams.put("isEncrypt", "1"); } else { passwordStr = CryptoUtils.object2String(passwordStr); } From 70c2cabfe9dd77d96b8199038e3f89044cf04226 Mon Sep 17 00:00:00 2001 From: v-kkhuang <62878639+v-kkhuang@users.noreply.github.com> Date: Mon, 28 Oct 2024 11:20:34 +0800 Subject: [PATCH 55/90] Python module optimization upload (#619) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: “v_kkhuang” <“420895376@qq.com”> --- .../linkis/udf/entity/PythonModuleInfoVO.java | 11 + .../api/rpc/RequestPythonModuleProtocol.scala | 5 + .../rpc/ResponsePythonModuleProtocol.scala | 2 + .../exception/WorkspaceExceptionManager.java | 4 + .../filesystem/restful/api/FsRestfulApi.java | 90 +++--- .../filesystem/util/FilesystemUtils.java | 281 ++++++++++++++++++ .../apache/linkis/udf/api/UDFRestfulApi.java | 12 +- .../linkis/udf/entity/PythonModuleInfo.java | 11 + .../mapper/common/PythonModuleInfoMapper.xml | 6 +- .../linkis/udf/api/rpc/UdfReceiver.scala | 15 + 10 files changed, 388 insertions(+), 49 deletions(-) create mode 100644 linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/filesystem/util/FilesystemUtils.java diff --git a/linkis-public-enhancements/linkis-pes-common/src/main/java/org/apache/linkis/udf/entity/PythonModuleInfoVO.java b/linkis-public-enhancements/linkis-pes-common/src/main/java/org/apache/linkis/udf/entity/PythonModuleInfoVO.java index 1c6a2af99a..c676b70ce1 100644 --- a/linkis-public-enhancements/linkis-pes-common/src/main/java/org/apache/linkis/udf/entity/PythonModuleInfoVO.java +++ b/linkis-public-enhancements/linkis-pes-common/src/main/java/org/apache/linkis/udf/entity/PythonModuleInfoVO.java @@ -53,6 +53,7 @@ public class PythonModuleInfoVO { // 修改时间，记录模块最后修改的时间 private Timestamp updateTime; + private String pythonModule; // 默认构造函数 public PythonModuleInfoVO() {} @@ -172,6 +173,14 @@ public void setUpdateTime(Timestamp updateTime) { this.updateTime = updateTime; } + public String getPythonModule() { + return pythonModule; + } + + public void setPythonModule(String pythonModule) { + this.pythonModule = pythonModule; + } + // 重写toString方法，用于调试和日志记录 @Override public String toString() { @@ -204,6 +213,8 @@ public String toString() { + createTime + ", updateTime=" + updateTime + + ", pythonModule=" + + pythonModule + '}'; } } diff --git a/linkis-public-enhancements/linkis-pes-common/src/main/scala/org/apache/linkis/udf/api/rpc/RequestPythonModuleProtocol.scala b/linkis-public-enhancements/linkis-pes-common/src/main/scala/org/apache/linkis/udf/api/rpc/RequestPythonModuleProtocol.scala index 27cd071fb7..d1ed943257 100644 --- a/linkis-public-enhancements/linkis-pes-common/src/main/scala/org/apache/linkis/udf/api/rpc/RequestPythonModuleProtocol.scala +++ b/linkis-public-enhancements/linkis-pes-common/src/main/scala/org/apache/linkis/udf/api/rpc/RequestPythonModuleProtocol.scala @@ -26,3 +26,8 @@ case class RequestPythonModuleProtocol(userName: String, engineType: String) with CacheableProtocol with PythonModuleProtocol with UdfProtocol + +case class RequestPythonInfo(pythonModule: String, username: String) + extends RetryableProtocol + with CacheableProtocol + with UdfProtocol diff --git a/linkis-public-enhancements/linkis-pes-common/src/main/scala/org/apache/linkis/udf/api/rpc/ResponsePythonModuleProtocol.scala b/linkis-public-enhancements/linkis-pes-common/src/main/scala/org/apache/linkis/udf/api/rpc/ResponsePythonModuleProtocol.scala index 4ff5c0f8db..84a1ca27a8 100644 --- a/linkis-public-enhancements/linkis-pes-common/src/main/scala/org/apache/linkis/udf/api/rpc/ResponsePythonModuleProtocol.scala +++ b/linkis-public-enhancements/linkis-pes-common/src/main/scala/org/apache/linkis/udf/api/rpc/ResponsePythonModuleProtocol.scala @@ -31,3 +31,5 @@ class ResponsePythonModuleProtocol(val pythonModules: java.util.List[PythonModul } } + +case class ResponsePythonInfo(pythonModuleInfoVO: PythonModuleInfoVO) extends PythonModuleProtocol diff --git a/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/filesystem/exception/WorkspaceExceptionManager.java b/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/filesystem/exception/WorkspaceExceptionManager.java index c9a4f5727d..3b664f86fc 100644 --- a/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/filesystem/exception/WorkspaceExceptionManager.java +++ b/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/filesystem/exception/WorkspaceExceptionManager.java @@ -89,6 +89,10 @@ public class WorkspaceExceptionManager { put( "80037", "Parameter error, page size is incorrect, please pass in a number within [1-500] (参数错误，列筛选最多支持筛选50列)"); + put( + "80038", + "The name directory {0} specified by PKG-INFO does not exist. Please confirm that the {0} specified by PKG-INFO in the package matches the actual folder name (PKG-INFO指定Name目录{0}不存在，请确认包中PKG-INFO指定{0}和实际文件夹名称一致)"); + put("80039", "File upload failed, error message: {0} (文件上传失败，错误信息：{0})"); } }; diff --git a/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/filesystem/restful/api/FsRestfulApi.java b/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/filesystem/restful/api/FsRestfulApi.java index 9105ee75fe..74aeec15b7 100644 --- a/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/filesystem/restful/api/FsRestfulApi.java +++ b/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/filesystem/restful/api/FsRestfulApi.java @@ -22,12 +22,12 @@ import org.apache.linkis.common.io.FsWriter; import org.apache.linkis.common.utils.ByteTimeUtils; import org.apache.linkis.common.utils.ResultSetUtils; -import org.apache.linkis.filesystem.conf.WorkSpaceConfiguration; import org.apache.linkis.filesystem.entity.DirFileTree; import org.apache.linkis.filesystem.entity.LogLevel; import org.apache.linkis.filesystem.exception.WorkSpaceException; import org.apache.linkis.filesystem.exception.WorkspaceExceptionManager; import org.apache.linkis.filesystem.service.FsService; +import org.apache.linkis.filesystem.util.FilesystemUtils; import org.apache.linkis.filesystem.util.WorkspaceUtil; import org.apache.linkis.filesystem.utils.UserGroupUtils; import org.apache.linkis.filesystem.validator.PathValidator$; @@ -62,6 +62,7 @@ import java.io.*; import java.nio.charset.StandardCharsets; import java.nio.file.Files; +import java.nio.file.Path; import java.nio.file.Paths; import java.text.MessageFormat; import java.util.*; @@ -98,7 +99,7 @@ private boolean checkIsUsersDirectory(String requestPath, String userName, Boole // 配置文件默认关闭检查，withadmin默认true，特殊情况传false 开启权限检查（ // The configuration file defaults to disable checking, with admin defaulting to true, and in // special cases, false is passed to enable permission checking） - boolean ownerCheck = WorkSpaceConfiguration.FILESYSTEM_PATH_CHECK_OWNER.getValue(); + boolean ownerCheck = FILESYSTEM_PATH_CHECK_OWNER.getValue(); if (!ownerCheck && withAdmin) { LOGGER.debug("not check filesystem owner."); return true; @@ -479,7 +480,7 @@ public void download( byte[] buffer = new byte[1024]; int bytesRead = 0; response.setCharacterEncoding(charset); - java.nio.file.Path source = Paths.get(fsPath.getPath()); + Path source = Paths.get(fsPath.getPath()); response.addHeader("Content-Type", Files.probeContentType(source)); response.addHeader("Content-Disposition", "attachment;filename=" + new File(path).getName()); outputStream = response.getOutputStream(); @@ -1386,12 +1387,12 @@ public Message chmod( return Message.error(MessageFormat.format(FILEPATH_ILLEGALITY, filePath)); } else { // Prohibit users from modifying their own unreadable content - if (checkFilePermissions(filePermission)) { + if (FilesystemUtils.checkFilePermissions(filePermission)) { FileSystem fileSystem = fsService.getFileSystem(userName, new FsPath(filePath)); Stack dirsToChmod = new Stack<>(); dirsToChmod.push(new FsPath(filePath)); if (isRecursion) { - traverseFolder(new FsPath(filePath), fileSystem, dirsToChmod); + FilesystemUtils.traverseFolder(new FsPath(filePath), fileSystem, dirsToChmod); } while (!dirsToChmod.empty()) { fileSystem.setPermission(dirsToChmod.pop(), filePermission); @@ -1444,14 +1445,15 @@ public Message pythonUpload( } // 获取文件名称 String fileNameSuffix = fileName.substring(0, fileName.lastIndexOf(".")); - if (!fileNameSuffix.matches("^[a-zA-Z][a-zA-Z0-9_]{0,49}$")) { + if (!fileNameSuffix.matches("^[a-zA-Z][a-zA-Z0-9_.-]{0,49}$")) { return Message.error("模块名称错误，仅支持数字字母下划线，且以字母开头，长度最大50"); } // 校验文件类型 if (!file.getOriginalFilename().endsWith(".py") - && !file.getOriginalFilename().endsWith(".zip")) { - return Message.error("仅支持.py和.zip格式模块文件"); + && !file.getOriginalFilename().endsWith(".zip") + && !file.getOriginalFilename().endsWith(".tar.gz")) { + return Message.error("仅支持.py和.zip和.tar.gz格式模块文件"); } // 校验文件大小 @@ -1459,6 +1461,12 @@ public Message pythonUpload( return Message.error("限制最大单个文件50M"); } + // tar.gz包依赖检查 + String errorMsg = FilesystemUtils.checkModuleFile(file, username); + if (StringUtils.isNotBlank(errorMsg)) { + return Message.error("部分依赖未加载，请检查并重新上传依赖包，依赖信息：" + errorMsg); + } + // 定义目录路径 String path = "hdfs:///appcom/linkis/udf/" + username; FsPath fsPath = new FsPath(path); @@ -1478,46 +1486,36 @@ public Message pythonUpload( // 构建新的文件路径 String newPath = fsPath.getPath() + "/" + file.getOriginalFilename(); - FsPath fsPathNew = new FsPath(newPath); - - // 上传文件 - try (InputStream is = file.getInputStream(); - OutputStream outputStream = fileSystem.write(fsPathNew, true)) { - IOUtils.copy(is, outputStream); - } catch (IOException e) { - return Message.error("文件上传失败：" + e.getMessage()); - } - // 返回成功消息并包含文件地址 - return Message.ok().data("filePath", newPath); - } - /** - * * - * - * @param filePermission: 700,744 Prohibit users from modifying their own unreadable content - */ - private static boolean checkFilePermissions(String filePermission) { - boolean result = false; - if (StringUtils.isNumeric(filePermission)) { - char[] ps = filePermission.toCharArray(); - int ownerPermissions = Integer.parseInt(String.valueOf(ps[0])); - if (ownerPermissions >= 4) { - result = true; + // 上传文件,tar包需要单独解压处理 + if (!file.getOriginalFilename().endsWith(".tar.gz")) { + FsPath fsPathNew = new FsPath(newPath); + try (InputStream is = file.getInputStream(); + OutputStream outputStream = fileSystem.write(fsPathNew, true)) { + IOUtils.copy(is, outputStream); + } catch (IOException e) { + return Message.error("文件上传失败：" + e.getMessage()); } - } - return result; - } - - private static void traverseFolder( - FsPath fsPath, FileSystem fileSystem, Stack dirsToChmod) throws IOException { - List list = fileSystem.list(fsPath); - if (list == null) { - return; - } - for (FsPath path : list) { - if (path.isdir()) { - traverseFolder(path, fileSystem, dirsToChmod); + } else { + InputStream is = null; + OutputStream outputStream = null; + try { + String packageName = FilesystemUtils.findPackageName(file.getInputStream()); + if (StringUtils.isBlank(packageName)) { + return Message.error("文件上传失败：PKG-INFO 文件不存在"); + } + is = FilesystemUtils.getZipInputStreamByTarInputStream(file, packageName); + newPath = fsPath.getPath() + FsPath.SEPARATOR + packageName + FsPath.CUR_DIR + "zip"; + FsPath fsPathNew = new FsPath(newPath); + outputStream = fileSystem.write(fsPathNew, true); + IOUtils.copy(is, outputStream); + } catch (IOException e) { + return Message.error("文件上传失败：" + e.getMessage()); + } finally { + outputStream.close(); + is.close(); } - dirsToChmod.push(path); } + // 返回成功消息并包含文件地址 + return Message.ok().data("filePath", newPath); } } diff --git a/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/filesystem/util/FilesystemUtils.java b/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/filesystem/util/FilesystemUtils.java new file mode 100644 index 0000000000..82f760326c --- /dev/null +++ b/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/filesystem/util/FilesystemUtils.java @@ -0,0 +1,281 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.filesystem.util; + +import org.apache.linkis.common.conf.Configuration; +import org.apache.linkis.common.io.FsPath; +import org.apache.linkis.common.utils.Utils; +import org.apache.linkis.filesystem.exception.WorkspaceExceptionManager; +import org.apache.linkis.rpc.Sender; +import org.apache.linkis.storage.fs.FileSystem; +import org.apache.linkis.udf.api.rpc.RequestPythonInfo; +import org.apache.linkis.udf.api.rpc.ResponsePythonInfo; +import org.apache.linkis.udf.entity.PythonModuleInfoVO; + +import org.apache.commons.compress.archivers.tar.TarArchiveEntry; +import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; +import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream; +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang3.StringUtils; + +import org.springframework.web.multipart.MultipartFile; + +import java.io.*; +import java.util.ArrayList; +import java.util.List; +import java.util.Stack; +import java.util.StringJoiner; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; + +public class FilesystemUtils { + + private static final String[] OPERATORS = {"==", ">=", "<=", ">", "<", "~="}; + + public static boolean checkFilePermissions(String filePermission) { + boolean result = false; + if (StringUtils.isNumeric(filePermission)) { + char[] ps = filePermission.toCharArray(); + int ownerPermissions = Integer.parseInt(String.valueOf(ps[0])); + if (ownerPermissions >= 4) { + result = true; + } + } + return result; + } + + public static void traverseFolder(FsPath fsPath, FileSystem fileSystem, Stack dirsToChmod) + throws IOException { + List list = fileSystem.list(fsPath); + if (list == null) { + return; + } + for (FsPath path : list) { + if (path.isdir()) { + traverseFolder(path, fileSystem, dirsToChmod); + } + dirsToChmod.push(path); + } + } + + /** + * 从 tar.gz 文件中查找PKG-INFO文件，并获取其需要打包的文件夹名称 + * + * @param inputStream tar.gz 文件的输入流。 + * @return 包名，如果未找到则返回 null。 + * @throws IOException 如果文件读取失败。 + */ + public static String findPackageName(InputStream inputStream) throws IOException { + try (TarArchiveInputStream tarInput = + new TarArchiveInputStream(new GzipCompressorInputStream(inputStream))) { + TarArchiveEntry entry; + while ((entry = tarInput.getNextTarEntry()) != null) { + if (entry.getName().endsWith("PKG-INFO")) { + return readPackageName(tarInput); + } + } + } catch (Exception e) { + throw WorkspaceExceptionManager.createException(80039, e.getMessage()); + } + return null; + } + + /** + * 从 tar.gz 文件中读取包名。 + * + * @param tarInputStream tar.gz 文件的输入流。 + * @return 包名。 + * @throws IOException 如果文件读取失败。 + */ + private static String readPackageName(TarArchiveInputStream tarInputStream) throws IOException { + StringBuilder content = new StringBuilder(); + byte[] buffer = new byte[1024]; + int length; + while ((length = tarInputStream.read(buffer)) != -1) { + content.append(new String(buffer, 0, length)); + } + String pkgInfoContent = content.toString(); + return pkgInfoContent.split("Name: ")[1].split("\n")[0].trim(); + } + + /** + * 从 tar.gz 文件中获取指定文件夹的根路径。 + * + * @param inputStream tar.gz 文件的输入流。 + * @param folder 指定的文件夹名称。 + * @return 文件夹的根路径，如果未找到则返回 null。 + * @throws IOException 如果文件读取失败。 + */ + private static String getRootPath(InputStream inputStream, String folder) throws IOException { + try (TarArchiveInputStream tarInput = + new TarArchiveInputStream(new GzipCompressorInputStream(inputStream))) { + TarArchiveEntry entry; + while ((entry = tarInput.getNextTarEntry()) != null) { + if (entry.isDirectory() && entry.getName().endsWith("/" + folder + "/")) { + return entry.getName(); + } + } + } catch (Exception e) { + throw WorkspaceExceptionManager.createException(80039, e.getMessage()); + } + return null; + } + + /** + * 将 tar.gz 文件中的指定文件夹内容转换为 zip 文件流。 + * + * @param inputStream tar.gz 文件的输入流。 + * @param folder 指定的文件夹名称。 + * @param rootPath 文件夹的根路径。 + * @return 包含指定文件夹内容的 zip 文件流。 + * @throws IOException 如果文件读取或写入失败。 + */ + private static InputStream createZipFile(InputStream inputStream, String folder, String rootPath) + throws IOException { + TarArchiveInputStream tarInput = + new TarArchiveInputStream(new GzipCompressorInputStream(inputStream)); + ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + ZipOutputStream zos = new ZipOutputStream(byteArrayOutputStream); + try { + TarArchiveEntry entry; + while ((entry = tarInput.getNextTarEntry()) != null) { + if (!entry.isDirectory() && entry.getName().contains("/" + folder + "/")) { + // \dist\py_mysql-1.0.tar\py_mysql-1.0\py_mysql\lib\__init__.py + ZipEntry zipEntry = new ZipEntry(entry.getName().substring(rootPath.length())); + zos.putNextEntry(zipEntry); + IOUtils.copy(tarInput, zos); + zos.closeEntry(); + } + } + } catch (Exception e) { + throw WorkspaceExceptionManager.createException(80039, e.getMessage()); + } finally { + tarInput.close(); + zos.close(); + } + // 将 ByteArrayOutputStream 转换为 InputStream + return new ByteArrayInputStream(byteArrayOutputStream.toByteArray()); + } + + public static InputStream getZipInputStreamByTarInputStream( + MultipartFile file, String packageName) throws IOException { + String rootPath = getRootPath(file.getInputStream(), packageName); + if (StringUtils.isBlank(rootPath)) { + throw WorkspaceExceptionManager.createException(80038, packageName); + } + return createZipFile(file.getInputStream(), packageName, rootPath); + } + + /** + * * 检查python环境中模块是否存在，使用程序调用脚本实现 脚本保存在admin文件夹中，脚本名为check_python_module.py + * 脚本执行存在返回ture，不存在返回false + * + * @param file 传入需要检查的模块列表 + * @param username + * @return 返回不存在的模块列表 + */ + public static String checkModuleFile(MultipartFile file, String username) throws IOException { + // 获取install_requires中的python模块 + List pythonModules = getInstallRequestPythonModules(file); + StringJoiner joiner = new StringJoiner(","); + // 检查机器上pyhton环境中模块是否存在 + List notExistModules = + pythonModules.stream() + .filter( + module -> { + String exec = + Utils.exec( + (new String[] { + "python", + Configuration.getLinkisHome() + "/admin/" + "check_python_module.py", + module + })); + return !Boolean.parseBoolean(exec); + }) + .collect(Collectors.toList()); + // 查询数据库中包是否存在 + notExistModules.forEach( + module -> { + Object object = + Sender.getSender("linkis-ps-publicservice") + .ask(new RequestPythonInfo(module, username)); + if (object instanceof ResponsePythonInfo) { + ResponsePythonInfo response = (ResponsePythonInfo) object; + PythonModuleInfoVO pythonModuleInfoVO = response.pythonModuleInfoVO(); + if (StringUtils.isBlank(pythonModuleInfoVO.getName())) { + joiner.add(module); + } + } + }); + return joiner.toString(); + } + + public static List getInstallRequestPythonModules(MultipartFile file) throws IOException { + List modules = new ArrayList<>(); + String originalFilename = file.getOriginalFilename(); + if (StringUtils.isNotBlank(originalFilename) && originalFilename.endsWith(".tar.gz")) { + // 读取 setup.py 文件的内容，并使用正则表达式提取 install_requires 字段。 + // 解析 install_requires 字段中的依赖包信息 + try (TarArchiveInputStream tarInput = + new TarArchiveInputStream(new GzipCompressorInputStream(file.getInputStream()))) { + TarArchiveEntry entry; + while ((entry = tarInput.getNextTarEntry()) != null) { + if (entry.getName().endsWith("setup.py")) { + StringBuilder content = new StringBuilder(); + byte[] buffer = new byte[1024]; + int length; + while ((length = tarInput.read(buffer)) != -1) { + content.append(new String(buffer, 0, length)); + } + modules = extractDependencies(content.toString()); + break; + } + } + } catch (Exception e) { + throw WorkspaceExceptionManager.createException(80039, e.getMessage()); + } + } + return modules; + } + + public static List extractDependencies(String content) { + List modules = new ArrayList<>(); + Pattern pattern = Pattern.compile("install_requires=\\[(.*?)\\]", Pattern.DOTALL); + Matcher matcher = pattern.matcher(content); + if (matcher.find()) { + String requirements = matcher.group(1); + String[] packages = requirements.split(","); + for (String pkg : packages) { + pkg = pkg.replaceAll("#.*?\\n", "").replaceAll("\\n", "").replaceAll("'", "").trim(); + for (String operator : OPERATORS) { + if (pkg.contains(operator)) { + String[] parts = pkg.split(operator); + pkg = parts[0].trim(); + } + } + if (StringUtils.isNotBlank(pkg)) { + modules.add(pkg); + } + } + } + return modules; + } +} diff --git a/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/api/UDFRestfulApi.java b/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/api/UDFRestfulApi.java index 0a84e0dcc1..9ecde360d1 100644 --- a/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/api/UDFRestfulApi.java +++ b/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/api/UDFRestfulApi.java @@ -49,6 +49,8 @@ import java.io.*; import java.text.MessageFormat; import java.util.*; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import java.util.stream.Collectors; import com.fasterxml.jackson.databind.JsonNode; @@ -1089,7 +1091,7 @@ public Message pythonList( pythonModuleInfo.setEngineType(engineType); pythonModuleInfo.setCreateUser(username); pythonModuleInfo.setIsLoad(isLoad); - pythonModuleInfo.setIsExpire(isExpire); + pythonModuleInfo.setIsExpire(0); List pythonList = pythonModuleInfoService.getByConditions(pythonModuleInfo); PageInfo pageInfo = new PageInfo<>(pythonList); // 封装返回结果 @@ -1194,6 +1196,14 @@ public Message request( if (pythonModuleInfo.getIsExpire() == null) { return Message.error("是否过期：不能为空"); } + if (org.apache.commons.lang3.StringUtils.isBlank(pythonModuleInfo.getPythonModule())) { + // 使用正则表达式进行校验 + Matcher matcher = + Pattern.compile("^[a-zA-Z0-9,]+$").matcher(pythonModuleInfo.getPythonModule()); + if (!matcher.matches()) { + return Message.error("模块名称：只允许英文、数字和英文逗号"); + } + } String path = pythonModuleInfo.getPath(); String fileName = path.substring(path.lastIndexOf("/") + 1, path.lastIndexOf(".")); if (!pythonModuleInfo.getName().equals(fileName)) { diff --git a/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/entity/PythonModuleInfo.java b/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/entity/PythonModuleInfo.java index 727b323cb6..357c34c7a5 100644 --- a/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/entity/PythonModuleInfo.java +++ b/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/entity/PythonModuleInfo.java @@ -31,6 +31,7 @@ public class PythonModuleInfo { private Integer isExpire; private Date createTime; private Date updateTime; + private String pythonModule; public PythonModuleInfo() {} @@ -122,6 +123,14 @@ public void setUpdateTime(Date updateTime) { this.updateTime = updateTime; } + public String getPythonModule() { + return pythonModule; + } + + public void setPythonModule(String pythonModule) { + this.pythonModule = pythonModule; + } + @Override public String toString() { return "PythonModuleInfo{" @@ -153,6 +162,8 @@ public String toString() { + createTime + ", updateTime=" + updateTime + + ", pythonModule=" + + pythonModule + '}'; } } diff --git a/linkis-public-enhancements/linkis-udf-service/src/main/resources/mapper/common/PythonModuleInfoMapper.xml b/linkis-public-enhancements/linkis-udf-service/src/main/resources/mapper/common/PythonModuleInfoMapper.xml index 0b513ba2f2..46392f0dae 100644 --- a/linkis-public-enhancements/linkis-udf-service/src/main/resources/mapper/common/PythonModuleInfoMapper.xml +++ b/linkis-public-enhancements/linkis-udf-service/src/main/resources/mapper/common/PythonModuleInfoMapper.xml @@ -54,6 +54,7 @@ is_expire = #{isExpire}, create_time = #{createTime}, update_time = #{updateTime}, + python_module = #{pythonModule}, WHERE id = #{id} @@ -61,9 +62,9 @@ INSERT INTO linkis_ps_python_module_info - (name, description, path, engine_type, create_user, update_user, is_load, is_expire, create_time, update_time) + (name, description, path, engine_type, create_user, update_user, is_load, is_expire, create_time, update_time, python_module) VALUES - (#{name}, #{description}, #{path}, #{engineType}, #{createUser}, #{updateUser}, #{isLoad}, #{isExpire}, #{createTime}, #{updateTime}) + (#{name}, #{description}, #{path}, #{engineType}, #{createUser}, #{updateUser}, #{isLoad}, #{isExpire}, #{createTime}, #{updateTime}, #{pythonModule}) SELECT LAST_INSERT_ID() @@ -75,6 +76,7 @@ AND create_user = #{createUser} AND name = #{name} AND id = #{id} + AND is_load = #{isLoad} diff --git a/linkis-public-enhancements/linkis-udf-service/src/main/scala/org/apache/linkis/udf/api/rpc/UdfReceiver.scala b/linkis-public-enhancements/linkis-udf-service/src/main/scala/org/apache/linkis/udf/api/rpc/UdfReceiver.scala index 8490e6a76d..0ed5143353 100644 --- a/linkis-public-enhancements/linkis-udf-service/src/main/scala/org/apache/linkis/udf/api/rpc/UdfReceiver.scala +++ b/linkis-public-enhancements/linkis-udf-service/src/main/scala/org/apache/linkis/udf/api/rpc/UdfReceiver.scala @@ -24,6 +24,10 @@ import org.apache.linkis.rpc.utils.RPCUtils import org.apache.linkis.udf.entity.{PythonModuleInfo, PythonModuleInfoVO} import org.apache.linkis.udf.service.{PythonModuleInfoService, UDFService, UDFTreeService} +import org.apache.commons.beanutils.BeanUtils +import org.apache.hadoop.hdfs.protocol.RollingUpgradeInfo.Bean +import org.apache.htrace.fasterxml.jackson.databind.util.BeanUtil + import java.{lang, util} import scala.collection.JavaConverters.asScalaBufferConverter @@ -93,6 +97,17 @@ class UdfReceiver extends Receiver with Logging { voList.add(vo) }) new ResponsePythonModuleProtocol(voList) + case RequestPythonInfo(pythonModule: String, username: String) => + var pythonModuleInfo = new PythonModuleInfo + pythonModuleInfo.setCreateUser(username) + pythonModuleInfo.setName(pythonModule) + pythonModuleInfo.setIsLoad(1) + pythonModuleInfo = pythonModuleInfoService.getByUserAndNameAndId(pythonModuleInfo) + var pythonModuleInfoVO = new PythonModuleInfoVO + if (null != pythonModuleInfo) { + BeanUtils.copyProperties(pythonModuleInfoVO, pythonModuleInfo) + } + new ResponsePythonInfo(pythonModuleInfoVO) case _ => } } From 5c8b21d161061557c7459541d25766b9c7ab2f87 Mon Sep 17 00:00:00 2001 From: v-kkhuang <62878639+v-kkhuang@users.noreply.github.com> Date: Mon, 28 Oct 2024 12:39:21 +0800 Subject: [PATCH 56/90] Dev 1.9.0 fix bug (#621) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix datasource error * fix jdbc error --------- Co-authored-by: “v_kkhuang” <“420895376@qq.com”> --- .../linkis/manager/engineplugin/jdbc/ConnectionManager.java | 5 +++++ .../manager/engineplugin/jdbc/executor/JDBCHelper.java | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/linkis-engineconn-plugins/jdbc/src/main/java/org/apache/linkis/manager/engineplugin/jdbc/ConnectionManager.java b/linkis-engineconn-plugins/jdbc/src/main/java/org/apache/linkis/manager/engineplugin/jdbc/ConnectionManager.java index a5679f1cf5..b3a9867d38 100644 --- a/linkis-engineconn-plugins/jdbc/src/main/java/org/apache/linkis/manager/engineplugin/jdbc/ConnectionManager.java +++ b/linkis-engineconn-plugins/jdbc/src/main/java/org/apache/linkis/manager/engineplugin/jdbc/ConnectionManager.java @@ -17,6 +17,7 @@ package org.apache.linkis.manager.engineplugin.jdbc; +import org.apache.linkis.common.utils.AESUtils; import org.apache.linkis.common.utils.SecurityUtils; import org.apache.linkis.hadoop.common.utils.KerberosUtils; import org.apache.linkis.manager.engineplugin.jdbc.conf.JDBCConfiguration$; @@ -200,6 +201,10 @@ protected DataSource buildDataSource(String dbUrl, Map propertie LOG.info("Database connection address information(数据库连接地址信息)=" + dbUrl); datasource.setUrl(dbUrl); datasource.setUsername(username); + if (AESUtils.LINKIS_DATASOURCE_AES_SWITCH.getValue()) { + // decrypt + password = AESUtils.decrypt(password, AESUtils.LINKIS_DATASOURCE_AES_KEY.getValue()); + } datasource.setPassword(password); datasource.setConnectProperties(SecurityUtils.getMysqlSecurityParams()); datasource.setDriverClassName(driverClassName); diff --git a/linkis-engineconn-plugins/jdbc/src/main/scala/org/apache/linkis/manager/engineplugin/jdbc/executor/JDBCHelper.java b/linkis-engineconn-plugins/jdbc/src/main/scala/org/apache/linkis/manager/engineplugin/jdbc/executor/JDBCHelper.java index 0747a836c6..b3437ac129 100644 --- a/linkis-engineconn-plugins/jdbc/src/main/scala/org/apache/linkis/manager/engineplugin/jdbc/executor/JDBCHelper.java +++ b/linkis-engineconn-plugins/jdbc/src/main/scala/org/apache/linkis/manager/engineplugin/jdbc/executor/JDBCHelper.java @@ -146,7 +146,7 @@ public static String getTypeStr(int type) { retVal = BinaryType.typeName(); break; case Types.DECIMAL: - retVal = DecimalType.typeName(); + retVal = DecimalType.toDataType(String.valueOf(type)).toString(); break; case Types.ARRAY: retVal = ArrayType.typeName(); From dbaa832f9aa3d37933d53b2db78c4fad073940fc Mon Sep 17 00:00:00 2001 From: v-kkhuang <62878639+v-kkhuang@users.noreply.github.com> Date: Mon, 28 Oct 2024 14:42:48 +0800 Subject: [PATCH 57/90] Dev 1.9.0 fix bug (#622) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix datasource error * fix jdbc error * fix et-monitor error --------- Co-authored-by: “v_kkhuang” <“420895376@qq.com”> --- .../apache/linkis/monitor/client/MonitorResourceClient.scala | 2 +- .../linkis/monitor/client/MonitorResourceClientImpl.scala | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/client/MonitorResourceClient.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/client/MonitorResourceClient.scala index 45de9cec70..77a3226ba3 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/client/MonitorResourceClient.scala +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/client/MonitorResourceClient.scala @@ -21,7 +21,7 @@ import org.apache.linkis.httpclient.authentication.AuthenticationStrategy import org.apache.linkis.httpclient.dws.authentication.StaticAuthenticationStrategy import org.apache.linkis.httpclient.dws.config.{DWSClientConfig, DWSClientConfigBuilder} import org.apache.linkis.httpclient.response.Result -import org.apache.linkis.monitor.request.EmsListAction +import org.apache.linkis.monitor.request.{EmsListAction, MonitorAction} import org.apache.linkis.ujes.client.response.EmsListResult import java.io.Closeable diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/client/MonitorResourceClientImpl.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/client/MonitorResourceClientImpl.scala index be1bba88cc..3112b2b63f 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/client/MonitorResourceClientImpl.scala +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/client/MonitorResourceClientImpl.scala @@ -21,6 +21,7 @@ import org.apache.linkis.httpclient.dws.DWSHttpClient import org.apache.linkis.httpclient.dws.config.DWSClientConfig import org.apache.linkis.httpclient.request.Action import org.apache.linkis.httpclient.response.Result +import org.apache.linkis.monitor.request.MonitorAction class MonitorResourceClientImpl(clientConfig: DWSClientConfig) extends MonitorResourceClient { From 2e960bb069f7cacef31b8242c2c925555d823f44 Mon Sep 17 00:00:00 2001 From: aiceflower Date: Mon, 28 Oct 2024 15:18:41 +0800 Subject: [PATCH 58/90] code format --- .../interactor/job/common/LogRetriever.java | 23 +++++++++-------- .../job/interactive/InteractiveJob.java | 14 +++++------ .../hook/PythonModuleLoadEngineConnHook.scala | 25 +++++++++++-------- .../engine/DefaultEngineCreateService.scala | 10 ++++++-- .../monitor/scheduled/JobHistoryMonitor.java | 10 ++++---- .../linkis/monitor/until/HttpsUntils.java | 3 ++- .../monitor/client/MonitorHTTPClient.scala | 15 +++++++++-- .../jobtime/StarrocksTimeExceedRule.scala | 1 + 8 files changed, 63 insertions(+), 38 deletions(-) diff --git a/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/common/LogRetriever.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/common/LogRetriever.java index a6928a3a46..a6a5a38d6c 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/common/LogRetriever.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/common/LogRetriever.java @@ -37,11 +37,11 @@ import org.slf4j.LoggerFactory; /** - * Log retrieval logic: - * 1. LogRetriever polls to obtain real-time logs, and if the task is completed, it retrieves persistent logs - * 2. Organized by org.apache.inkis.cli.application. interactor.job. com LogRetriever # sendLogFin decides whether to continue polling logs - * 3. getNextLogLine is the FromLine returned by the log interface - * 4. The return of persistent logs is OpenLogResult2 + * Log retrieval logic: 1. LogRetriever polls to obtain real-time logs, and if the task is + * completed, it retrieves persistent logs 2. Organized by org.apache.inkis.cli.application. + * interactor.job. com LogRetriever # sendLogFin decides whether to continue polling logs 3. + * getNextLogLine is the FromLine returned by the log interface 4. The return of persistent logs is + * OpenLogResult2 */ public class LogRetriever { private static final Logger logger = LoggerFactory.getLogger(LogRetriever.class); @@ -137,7 +137,8 @@ public void queryLogLoop(LogData data) { if (curLogIdx >= nextLogIdx) { String msg = MessageFormat.format( - "Retrieving log, curLogIdx={}, hasNext={0}, nextLogIdx={1}", curLogIdx, hasNext, nextLogIdx); + "Retrieving log, curLogIdx={}, hasNext={0}, nextLogIdx={1}", + curLogIdx, hasNext, nextLogIdx); logger.info(msg); } CliUtils.doSleepQuietly(CliConstants.JOB_QUERY_SLEEP_MILLS); @@ -152,16 +153,16 @@ public void queryLogLoop(LogData data) { private void queryJobLogFromLine(LogData data, int fromLine) throws LinkisClientRuntimeException { LinkisOperResultAdapter jobInfoResult = - linkisJobOperator.queryJobInfo(data.getUser(), data.getJobID()); + linkisJobOperator.queryJobInfo(data.getUser(), data.getJobID()); data.updateLog(jobInfoResult); if (!jobInfoResult.getJobStatus().isJobFinishedState()) { data.updateLog( - linkisJobOperator.queryRunTimeLogFromLine( - data.getUser(), data.getJobID(), data.getExecID(), fromLine)); + linkisJobOperator.queryRunTimeLogFromLine( + data.getUser(), data.getJobID(), data.getExecID(), fromLine)); } else { data.updateLog( - linkisJobOperator.queryPersistedLogFromLine( - data.getLogPath(), data.getUser(), data.getJobID(), fromLine)); + linkisJobOperator.queryPersistedLogFromLine( + data.getLogPath(), data.getUser(), data.getJobID(), fromLine)); } } diff --git a/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/interactive/InteractiveJob.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/interactive/InteractiveJob.java index 6fbc454b89..9affc775b2 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/interactive/InteractiveJob.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/interactive/InteractiveJob.java @@ -126,13 +126,13 @@ public JobResult run() { // get log while running LogRetriever logRetriever = - new LogRetriever( - jobInfoResult.getUser(), - jobInfoResult.getJobID(), - submitResult.getStrongerExecId(), - true, - oper, - new LogPresenter()); + new LogRetriever( + jobInfoResult.getUser(), + jobInfoResult.getJobID(), + submitResult.getStrongerExecId(), + true, + oper, + new LogPresenter()); // async because we need to query job status logRetriever.retrieveLogAsync(); diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonModuleLoadEngineConnHook.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonModuleLoadEngineConnHook.scala index 062f228298..a8fa2778fd 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonModuleLoadEngineConnHook.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonModuleLoadEngineConnHook.scala @@ -17,34 +17,30 @@ package org.apache.linkis.engineconn.computation.executor.hook +import org.apache.commons.lang3.StringUtils +import org.apache.hadoop.conf.Configuration +import org.apache.hadoop.fs.{FileSystem, Path} import org.apache.linkis.common.conf.Configuration.IS_VIEW_FS_ENV import org.apache.linkis.common.utils.{Logging, Utils} import org.apache.linkis.engineconn.common.conf.EngineConnConf import org.apache.linkis.engineconn.common.creation.EngineCreationContext import org.apache.linkis.engineconn.common.engineconn.EngineConn import org.apache.linkis.engineconn.common.hook.EngineConnHook -import org.apache.linkis.engineconn.computation.executor.execute.{ - ComputationExecutor, - EngineExecutionContext -} +import org.apache.linkis.engineconn.computation.executor.execute.{ComputationExecutor, EngineExecutionContext} import org.apache.linkis.engineconn.core.engineconn.EngineConnManager import org.apache.linkis.engineconn.core.executor.ExecutorManager import org.apache.linkis.hadoop.common.conf.HadoopConf import org.apache.linkis.hadoop.common.utils.HDFSUtils import org.apache.linkis.manager.label.entity.Label -import org.apache.linkis.manager.label.entity.engine.{CodeLanguageLabel, RunType} import org.apache.linkis.manager.label.entity.engine.RunType.RunType +import org.apache.linkis.manager.label.entity.engine.{CodeLanguageLabel, RunType} import org.apache.linkis.rpc.Sender import org.apache.linkis.udf.UDFClientConfiguration import org.apache.linkis.udf.api.rpc.{RequestPythonModuleProtocol, ResponsePythonModuleProtocol} import org.apache.linkis.udf.entity.PythonModuleInfoVO -import org.apache.commons.lang3.StringUtils -import org.apache.hadoop.conf.Configuration -import org.apache.hadoop.fs.{FileSystem, Path} - import java.util - +import java.util.{Collections, Comparator} import scala.collection.JavaConverters._ import scala.collection.mutable @@ -67,6 +63,15 @@ abstract class PythonModuleLoad extends Logging { .ask(RequestPythonModuleProtocol(userName, engineType)) .asInstanceOf[ResponsePythonModuleProtocol] .getModulesInfo() + // 使用Collections.sort()和Comparator进行排序// 使用Collections.sort()和Comparator进行排序 + + Collections.sort( + infoList, + new Comparator[PythonModuleInfoVO]() { + override def compare(o1: PythonModuleInfoVO, o2: PythonModuleInfoVO): Int = + Integer.compare(o1.getId.toInt, o1.getId.toInt) + } + ) infoList } diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/DefaultEngineCreateService.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/DefaultEngineCreateService.scala index 6f72c74e9b..e1fd6ae66f 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/DefaultEngineCreateService.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/DefaultEngineCreateService.scala @@ -35,7 +35,10 @@ import org.apache.linkis.manager.common.entity.node.{EMNode, EngineNode} import org.apache.linkis.manager.common.entity.resource.NodeResource import org.apache.linkis.manager.common.protocol.engine.{EngineCreateRequest, EngineStopRequest} import org.apache.linkis.manager.common.utils.ManagerUtils -import org.apache.linkis.manager.engineplugin.common.launch.entity.{EngineConnBuildRequestImpl, EngineConnCreationDescImpl} +import org.apache.linkis.manager.engineplugin.common.launch.entity.{ + EngineConnBuildRequestImpl, + EngineConnCreationDescImpl +} import org.apache.linkis.manager.engineplugin.common.resource.TimeoutEngineResourceRequest import org.apache.linkis.manager.label.builder.factory.LabelBuilderFactoryContext import org.apache.linkis.manager.label.entity.{EngineNodeLabel, Label} @@ -50,12 +53,15 @@ import org.apache.linkis.manager.service.common.label.LabelFilter import org.apache.linkis.rpc.Sender import org.apache.linkis.rpc.message.annotation.Receiver import org.apache.linkis.server.BDPJettyServerHelper + import org.apache.commons.lang3.StringUtils + import org.springframework.beans.factory.annotation.Autowired import org.springframework.stereotype.Service import java.util -import java.util.concurrent.{TimeUnit, TimeoutException} +import java.util.concurrent.{TimeoutException, TimeUnit} + import scala.collection.JavaConverters._ import scala.concurrent.duration.Duration diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scheduled/JobHistoryMonitor.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scheduled/JobHistoryMonitor.java index cd66bd9795..712ae62899 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scheduled/JobHistoryMonitor.java +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scheduled/JobHistoryMonitor.java @@ -206,14 +206,14 @@ public void jobHistoryUnfinishedScan() { logger.info("[INFO] Loaded 0 alerts jobtime alert-rule from alert properties file."); } else { logger.info( - "[INFO] Loaded {} alerts jobtime alert-rules from alert properties file.", - jobTimeAlerts.size()); + "[INFO] Loaded {} alerts jobtime alert-rules from alert properties file.", + jobTimeAlerts.size()); shouldStart = true; JobMonitorUtils.addIntervalToImsAlerts(jobTimeAlerts, realIntervals); JobTimeExceedRule jobTimeExceedRule = - new JobTimeExceedRule( - jobTimeAlerts.keySet(), new JobTimeExceedAlertSender(jobTimeAlerts)); - scanner.addScanRule(jobTimeExceedRule); + new JobTimeExceedRule( + jobTimeAlerts.keySet(), new JobTimeExceedAlertSender(jobTimeAlerts)); + scanner.addScanRule(jobTimeExceedRule); } StarrocksTimeExceedRule starrocksTimeExceedRule = new StarrocksTimeExceedRule(new StarrocksTimeExceedAlterSender()); diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/until/HttpsUntils.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/until/HttpsUntils.java index 1fe1bf940c..8c8b802d89 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/until/HttpsUntils.java +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/until/HttpsUntils.java @@ -120,7 +120,8 @@ public static Map getDatasourceConf(String user, String datasourceName) { .setDataSourceName(datasourceName) .setUser(user) .build(); - GetInfoPublishedByDataSourceNameResult result = client.getInfoByDataSourceInfo(dataSourceParamsAction); + GetInfoPublishedByDataSourceNameResult result = + client.getInfoByDataSourceInfo(dataSourceParamsAction); Map data = MapUtils.getMap(result.getResultMap(), "data", new HashMap<>()); Map datasourceInfoMap = MapUtils.getMap(data, "info", new HashMap<>()); return datasourceInfoMap; diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/client/MonitorHTTPClient.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/client/MonitorHTTPClient.scala index b3021f9b52..6b76fd3c73 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/client/MonitorHTTPClient.scala +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/client/MonitorHTTPClient.scala @@ -17,12 +17,23 @@ package org.apache.linkis.monitor.client -import org.apache.linkis.datasource.client.response.{GetConnectParamsByDataSourceNameResult, GetInfoByDataSourceNameResult, GetInfoPublishedByDataSourceNameResult} +import org.apache.linkis.datasource.client.response.{ + GetConnectParamsByDataSourceNameResult, + GetInfoByDataSourceNameResult, + GetInfoPublishedByDataSourceNameResult +} import org.apache.linkis.httpclient.authentication.AuthenticationStrategy import org.apache.linkis.httpclient.dws.authentication.StaticAuthenticationStrategy import org.apache.linkis.httpclient.dws.config.{DWSClientConfig, DWSClientConfigBuilder} import org.apache.linkis.httpclient.response.Result -import org.apache.linkis.monitor.request.{DataSourceParamsAction, EmsListAction, EntranceTaskAction, KeyvalueAction, KillJobAction, MonitorAction} +import org.apache.linkis.monitor.request.{ + DataSourceParamsAction, + EmsListAction, + EntranceTaskAction, + KeyvalueAction, + KillJobAction, + MonitorAction +} import org.apache.linkis.monitor.response.{EntranceTaskResult, KeyvalueResult, KillJobResultAction} import org.apache.linkis.ujes.client.response.EmsListResult diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/jobhistory/jobtime/StarrocksTimeExceedRule.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/jobhistory/jobtime/StarrocksTimeExceedRule.scala index 7fd8d9f335..ea18e1f143 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/jobhistory/jobtime/StarrocksTimeExceedRule.scala +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/jobhistory/jobtime/StarrocksTimeExceedRule.scala @@ -132,4 +132,5 @@ class StarrocksTimeExceedRule(hitObserver: Observer) // 获取datasource信息 HttpsUntils.getDatasourceConf(job.getSubmitUser, datasourceName) } + } From d9e94c1ee9e4f5920b1a3d7efb0d77710a1f975c Mon Sep 17 00:00:00 2001 From: v-kkhuang <62878639+v-kkhuang@users.noreply.github.com> Date: Mon, 28 Oct 2024 15:51:18 +0800 Subject: [PATCH 59/90] Dev 1.9.0 fix bug (#623) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix datasource error * fix jdbc error * fix et-monitor error * fix et-monitor error --------- Co-authored-by: “v_kkhuang” <“420895376@qq.com”> --- .../linkis/monitor/client/MonitorHTTPClientClientImpl.scala | 1 + .../scala/org/apache/linkis/monitor/request/MonitorAction.scala | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/client/MonitorHTTPClientClientImpl.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/client/MonitorHTTPClientClientImpl.scala index 1f6884e412..8074aeeb62 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/client/MonitorHTTPClientClientImpl.scala +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/client/MonitorHTTPClientClientImpl.scala @@ -21,6 +21,7 @@ import org.apache.linkis.httpclient.dws.DWSHttpClient import org.apache.linkis.httpclient.dws.config.DWSClientConfig import org.apache.linkis.httpclient.request.Action import org.apache.linkis.httpclient.response.Result +import org.apache.linkis.monitor.request.MonitorAction class MonitorHTTPClientClientImpl(clientConfig: DWSClientConfig) extends MonitorHTTPClient { diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/request/MonitorAction.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/request/MonitorAction.scala index abc8fe94d3..8c203303a0 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/request/MonitorAction.scala +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/request/MonitorAction.scala @@ -18,6 +18,6 @@ package org.apache.linkis.monitor.request import org.apache.linkis.httpclient.dws.request.DWSHttpAction -import org.apache.linkis.httpclient.request.UserAction +import org.apache.linkis.ujes.client.request.UserAction trait MonitorAction extends DWSHttpAction with UserAction From 98195d22588e6e29b08d07e29b5e1626e58d47cd Mon Sep 17 00:00:00 2001 From: aiceflower Date: Mon, 28 Oct 2024 16:11:43 +0800 Subject: [PATCH 60/90] add udf func check --- .../EngineConnExecutorErrorCode.java | 2 ++ .../conf/ComputationExecutorConf.scala | 3 ++ .../hook/PythonModuleLoadEngineConnHook.scala | 15 ++++++---- .../executor/SparkEngineConnExecutor.scala | 28 ++++++++++++++++++- 4 files changed, 42 insertions(+), 6 deletions(-) diff --git a/linkis-computation-governance/linkis-computation-governance-common/src/main/java/org/apache/linkis/governance/common/exception/engineconn/EngineConnExecutorErrorCode.java b/linkis-computation-governance/linkis-computation-governance-common/src/main/java/org/apache/linkis/governance/common/exception/engineconn/EngineConnExecutorErrorCode.java index 89d3c9eba4..8f76c52c5d 100644 --- a/linkis-computation-governance/linkis-computation-governance-common/src/main/java/org/apache/linkis/governance/common/exception/engineconn/EngineConnExecutorErrorCode.java +++ b/linkis-computation-governance/linkis-computation-governance-common/src/main/java/org/apache/linkis/governance/common/exception/engineconn/EngineConnExecutorErrorCode.java @@ -39,4 +39,6 @@ public class EngineConnExecutorErrorCode { public static final int INIT_EXECUTOR_FAILED = 40106; public static final int INVALID_APPLICATION_ID = 40107; + + public static final int ILLEGAL_USE_UDF_FUNCTION = 40108; } diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/conf/ComputationExecutorConf.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/conf/ComputationExecutorConf.scala index 292a150e22..d767af70c1 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/conf/ComputationExecutorConf.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/conf/ComputationExecutorConf.scala @@ -139,6 +139,9 @@ object ComputationExecutorConf { val SPECIAL_UDF_CHECK_ENABLED = CommonVars("linkis.ec.spacial.udf.check.enabled", false) + val SPECIAL_UDF_CHECK_ENABLED = + CommonVars("linkis.ec.spacial.udf.check.by.regex.enabled", false) + val SPECIAL_UDF_NAMES = CommonVars("linkis.ec.spacial.udf.check.names", "") diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonModuleLoadEngineConnHook.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonModuleLoadEngineConnHook.scala index a8fa2778fd..02486d485a 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonModuleLoadEngineConnHook.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonModuleLoadEngineConnHook.scala @@ -17,30 +17,35 @@ package org.apache.linkis.engineconn.computation.executor.hook -import org.apache.commons.lang3.StringUtils -import org.apache.hadoop.conf.Configuration -import org.apache.hadoop.fs.{FileSystem, Path} import org.apache.linkis.common.conf.Configuration.IS_VIEW_FS_ENV import org.apache.linkis.common.utils.{Logging, Utils} import org.apache.linkis.engineconn.common.conf.EngineConnConf import org.apache.linkis.engineconn.common.creation.EngineCreationContext import org.apache.linkis.engineconn.common.engineconn.EngineConn import org.apache.linkis.engineconn.common.hook.EngineConnHook -import org.apache.linkis.engineconn.computation.executor.execute.{ComputationExecutor, EngineExecutionContext} +import org.apache.linkis.engineconn.computation.executor.execute.{ + ComputationExecutor, + EngineExecutionContext +} import org.apache.linkis.engineconn.core.engineconn.EngineConnManager import org.apache.linkis.engineconn.core.executor.ExecutorManager import org.apache.linkis.hadoop.common.conf.HadoopConf import org.apache.linkis.hadoop.common.utils.HDFSUtils import org.apache.linkis.manager.label.entity.Label -import org.apache.linkis.manager.label.entity.engine.RunType.RunType import org.apache.linkis.manager.label.entity.engine.{CodeLanguageLabel, RunType} +import org.apache.linkis.manager.label.entity.engine.RunType.RunType import org.apache.linkis.rpc.Sender import org.apache.linkis.udf.UDFClientConfiguration import org.apache.linkis.udf.api.rpc.{RequestPythonModuleProtocol, ResponsePythonModuleProtocol} import org.apache.linkis.udf.entity.PythonModuleInfoVO +import org.apache.commons.lang3.StringUtils +import org.apache.hadoop.conf.Configuration +import org.apache.hadoop.fs.{FileSystem, Path} + import java.util import java.util.{Collections, Comparator} + import scala.collection.JavaConverters._ import scala.collection.mutable diff --git a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/executor/SparkEngineConnExecutor.scala b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/executor/SparkEngineConnExecutor.scala index 388cc4f27e..40f95fb74d 100644 --- a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/executor/SparkEngineConnExecutor.scala +++ b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/executor/SparkEngineConnExecutor.scala @@ -18,8 +18,9 @@ package org.apache.linkis.engineplugin.spark.executor import org.apache.linkis.common.log.LogUtils -import org.apache.linkis.common.utils.{ByteTimeUtils, Logging, Utils} +import org.apache.linkis.common.utils.{ByteTimeUtils, CodeAndRunTypeUtils, Logging, Utils} import org.apache.linkis.engineconn.common.conf.{EngineConnConf, EngineConnConstant} +import org.apache.linkis.engineconn.computation.executor.conf.ComputationExecutorConf import org.apache.linkis.engineconn.computation.executor.execute.{ ComputationExecutor, EngineExecutionContext @@ -41,6 +42,10 @@ import org.apache.linkis.engineplugin.spark.extension.{ import org.apache.linkis.engineplugin.spark.utils.JobProgressUtil import org.apache.linkis.governance.common.conf.GovernanceCommonConf import org.apache.linkis.governance.common.exception.LinkisJobRetryException +import org.apache.linkis.governance.common.exception.engineconn.{ + EngineConnExecutorErrorCode, + EngineConnExecutorErrorException +} import org.apache.linkis.governance.common.utils.JobUtils import org.apache.linkis.manager.common.entity.enumeration.NodeStatus import org.apache.linkis.manager.common.entity.resource._ @@ -48,6 +53,7 @@ import org.apache.linkis.manager.common.protocol.resource.ResourceWithStatus import org.apache.linkis.manager.label.constant.LabelKeyConstant import org.apache.linkis.manager.label.entity.Label import org.apache.linkis.manager.label.entity.engine.CodeLanguageLabel +import org.apache.linkis.manager.label.utils.{LabelUtil, LabelUtils} import org.apache.linkis.protocol.engine.JobProgressInfo import org.apache.linkis.scheduler.executer.ExecuteResponse @@ -131,6 +137,26 @@ abstract class SparkEngineConnExecutor(val sc: SparkContext, id: Long) ) } + // 正则匹配校验 + if (ComputationExecutorConf.SPECIAL_UDF_CHECK_BY_REGEX_ENABLED.getValue) { + val codeType: String = LabelUtil.getCodeType(engineExecutorContext.getLabels.toList.asJava) + val languageType: String = CodeAndRunTypeUtils.getLanguageTypeByCodeType(codeType) + if (!CodeAndRunTypeUtils.LANGUAGE_TYPE_SQL.equals(languageType)) { + val udfNames: String = ComputationExecutorConf.SPECIAL_UDF_NAMES.getValue + if (StringUtils.isNotBlank(udfNames)) { + val funcNames: Array[String] = udfNames.split(",") + funcNames.foreach(funcName => { + if (code.contains(funcName)) { + throw new EngineConnExecutorErrorException( + EngineConnExecutorErrorCode.ILLEGAL_USE_UDF_FUNCTION, + "非法使用UDF函数，特殊加解密UDF函数只能在sql脚本使用" + ) + } + }) + } + } + } + // Pre-execution hook var executionHook: SparkPreExecutionHook = null Utils.tryCatch { From 181a2e928a9e5519f130444a41e10d3f8c53cb9c Mon Sep 17 00:00:00 2001 From: aiceflower Date: Mon, 28 Oct 2024 16:18:54 +0800 Subject: [PATCH 61/90] add udf func check --- .../computation/executor/conf/ComputationExecutorConf.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/conf/ComputationExecutorConf.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/conf/ComputationExecutorConf.scala index d767af70c1..575f0165fa 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/conf/ComputationExecutorConf.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/conf/ComputationExecutorConf.scala @@ -139,7 +139,7 @@ object ComputationExecutorConf { val SPECIAL_UDF_CHECK_ENABLED = CommonVars("linkis.ec.spacial.udf.check.enabled", false) - val SPECIAL_UDF_CHECK_ENABLED = + val SPECIAL_UDF_CHECK_BY_REGEX_ENABLED = CommonVars("linkis.ec.spacial.udf.check.by.regex.enabled", false) val SPECIAL_UDF_NAMES = From 71fe3cbe57075bccaa7d1669f6282517026613a5 Mon Sep 17 00:00:00 2001 From: aiceflower Date: Mon, 28 Oct 2024 21:27:11 +0800 Subject: [PATCH 62/90] fix queue get error --- .../linkis/manager/rm/external/yarn/YarnResourceRequester.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/rm/external/yarn/YarnResourceRequester.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/rm/external/yarn/YarnResourceRequester.java index 92f5018493..006b58157f 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/rm/external/yarn/YarnResourceRequester.java +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/rm/external/yarn/YarnResourceRequester.java @@ -234,7 +234,7 @@ public YarnQueueInfo getResources( JsonNode childQueues = getChildQueues(schedulerInfo.path("rootQueue")); queue = getQueue(childQueues, realQueueName); } - if (queue != null || !queue.isPresent()) { + if (queue == null || !queue.isPresent()) { logger.debug( "cannot find any information about queue " + queueName + ", response: " + resp); throw new RMWarnException( From 0d681860d83393af9ed1c5d4fb3951687b851c99 Mon Sep 17 00:00:00 2001 From: taoran1250 <543121890@qq.com> Date: Thu, 31 Oct 2024 16:20:21 +0800 Subject: [PATCH 63/90] Dev 1.9.0 webank sonarfix (#608) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore: update pom * Update pom.xml * Update pom.xml * Update pom.xml * Update pom.xml * Update pom.xml * 修改ecc默认告警人 * fix: code review fix * fix:spark引擎打印python版本问题修复 * fix:spark引擎打印python版本问题修复 * fix: sonar fix * fix: sonar fix * fix: sonar fix * fix:sonar fix * feat: codecheck ignore * feat: sonar fix * fix: spotless --- codecheck.ignore | 1 + .../linkis/common/utils/ByteTimeUtils.java | 2 +- .../common/variable/CustomDateType.scala | 24 +-- .../linkis/protocol/util/ImmutablePair.java | 6 + .../utils/LoadBalancerOptionsUtils.java | 42 ----- .../serializer/ProtostuffSerializeUtil.java | 9 +- .../utils/LoadBalancerOptionsUtilsTest.java | 34 ----- .../storage/excel/ExcelStorageReader.java | 2 +- .../storage/fs/impl/LocalFileSystem.java | 12 +- .../storage/excel/ExcelStorageReaderTest.java | 2 +- .../parser/transformer/ParamKeyMapper.java | 3 +- .../converter/PredefinedStringConverters.java | 4 +- .../command/template/option/Parameter.java | 2 +- .../command/template/option/StdOption.java | 2 +- .../properties/reader/PropsFileReader.java | 4 +- .../operator/once/OnceJobOper.java | 5 +- .../operator/ujes/UJESClientFactory.java | 2 +- .../operator/ujes/UJESResultAdapter.java | 3 +- .../present/file/ResultFileWriter.java | 8 +- .../cli/application/utils/CliUtils.java | 11 +- .../application/utils/SchedulerManager.java | 4 +- .../monitor/TimingMonitorService.java | 4 +- .../PythonModuleLoadEngineConnHookTest.scala | 102 ++++++------- .../linkis/entrance/parser/ParserUtils.java | 2 +- .../restful/EntranceLabelRestfulApi.java | 4 +- .../entrance/EntranceWebSocketService.scala | 9 +- .../impl/TestHDFSCacheLogWriter.java | 6 +- .../linkis/ujes/jdbc/UJESSQLTypeParser.scala | 20 --- .../DefaultEngineConnPluginLoader.java | 3 +- .../loader/utils/EngineConnPluginUtils.java | 3 +- .../impl/EnginePluginAdminServiceImpl.java | 17 ++- .../rm/utils/RequestKerberosUrlUtils.java | 24 ++- .../factory/LabelBuilderFactoryContext.java | 2 +- .../manager/label/entity/CloneableLabel.java | 2 +- .../label/entity/CombinedLabelImpl.java | 6 + .../label/entity/SerializableLabel.java | 5 + .../entity/entrance/BindEngineLabel.java | 6 + .../entity/entrance/LoadBalanceLabel.java | 6 + .../label/utils/EngineTypeLabelCreator.java | 2 +- .../manager/label/utils/LabelUtils.java | 13 +- .../impl/DefaultLockManagerPersistence.java | 4 +- .../dao/ResourceManagerMapperTest.java | 3 +- .../sql/operation/OperationFactoryImpl.java | 2 +- .../sql/parser/SqlCommandParserImpl.java | 2 +- .../hbase/HBaseConnectionManager.java | 2 +- .../hbase/shell/HBaseShellSessionManager.java | 2 +- .../engineplugin/jdbc/ConnectionManager.java | 2 +- .../monitor/impl/TrinoProgressMonitor.java | 4 +- .../jdbc/ConnectionManagerTest.java | 2 +- .../jdbc/ProgressMonitorTest.java | 2 +- .../factory/SparkEngineConnFactory.scala | 2 +- .../socket/SocketChannelSocketFactory.java | 6 +- .../linkis/monitor/jobhistory/QueryUtils.java | 2 +- .../monitor/scheduled/JobHistoryMonitor.java | 2 +- .../linkis/bml/restful/BmlProjectRestful.java | 2 +- .../linkis/bml/restful/BmlRestfulApi.java | 6 +- .../bml/service/impl/VersionServiceImpl.java | 2 +- .../api/ConfigurationRestfulApiTest.java | 4 +- .../cs/condition/BinaryLogicCondition.java | 2 +- .../construction/ConditionParser.java | 44 +++--- .../DefaultContextCacheService.java | 2 +- .../cskey/impl/ContextValueMapSetImpl.java | 2 +- .../index/ContextInvertedIndexSetImpl.java | 2 +- .../ha/impl/BackupInstanceGeneratorImpl.java | 4 +- .../ha/impl/ContextHACheckerImpl.java | 13 +- .../server/protocol/RestResponseProtocol.java | 2 +- .../apache/linkis/cs/parser/ApiJsonTest.java | 2 +- .../cs/persistence/ContextHistoryTest.java | 4 +- .../linkis/cs/persistence/ContextMapTest.java | 2 +- .../cs/server/conf/ContextServerConfTest.java | 2 +- .../service/DataSourceInfoServiceTest.java | 2 +- .../query/server/utils/MetadataUtils.java | 3 +- .../query/service/MongoDbConnection.java | 3 +- .../service/impl/DataSourceServiceImpl.java | 2 +- .../metadata/hive/dao/HiveMetaDaoTest.java | 6 +- .../async/GenericAsyncConsumerQueue.java | 4 +- .../label/entity/InsPersistenceLabel.java | 6 + .../cs/client/http/HttpContextClient.java | 14 +- .../ContextClientListenerManager.java | 2 +- .../cs/client/service/CSMetaDataService.java | 2 +- .../cs/client/service/CSNodeServiceImpl.java | 2 +- .../cs/client/service/CSResourceService.java | 2 +- .../service/CSResultDataServiceImpl.java | 2 +- .../cs/client/service/CSTableService.java | 4 +- .../cs/client/service/CSVariableService.java | 2 +- .../cs/client/service/CSWorkServiceImpl.java | 2 +- .../ContextHistoryClientServiceImpl.java | 2 +- .../client/service/DefaultSearchService.java | 2 +- .../service/LinkisJobDataServiceImpl.java | 2 +- .../handler/LinkisErrorCodeHandler.java | 21 ++- .../manager/LinkisErrorCodeManager.java | 2 +- .../LinkisErrorCodeSynchronizer.java | 2 +- .../helper/ContextSerializationHelper.java | 2 +- .../LinkisHAWorkFlowContextIDSerializer.java | 2 +- .../metadata/CSTableLineageSerializer.java | 2 +- .../CSTableMetadataHistorySerializer.java | 3 +- .../ListenerBus/ContextAsyncListenerBus.java | 2 +- .../imp/DefaultContextIDCallbackEngine.java | 2 +- .../imp/DefaultContextKeyCallbackEngine.java | 4 +- .../imp/DefaultContextListenerManager.java | 2 +- .../query/common/cache/ConnCacheManager.java | 2 +- .../exception/WorkspaceExceptionManager.java | 144 +++++++++--------- .../filesystem/restful/api/FsRestfulApi.java | 102 ++++++------- .../apache/linkis/udf/api/UDFRestfulApi.java | 2 +- .../udf/service/impl/UDFServiceImpl.java | 68 ++++----- .../http/GatewayAuthorizationFilter.java | 2 +- 106 files changed, 446 insertions(+), 532 deletions(-) create mode 100644 codecheck.ignore delete mode 100644 linkis-commons/linkis-rpc/src/main/java/org/apache/linkis/rpc/message/utils/LoadBalancerOptionsUtils.java delete mode 100644 linkis-commons/linkis-rpc/src/test/java/org/apache/linkis/rpc/message/utils/LoadBalancerOptionsUtilsTest.java diff --git a/codecheck.ignore b/codecheck.ignore new file mode 100644 index 0000000000..eb860a6ca1 --- /dev/null +++ b/codecheck.ignore @@ -0,0 +1 @@ +linkis-commons/linkis-common/src/main/java/org/apache/linkis/common/utils/DESUtil.java diff --git a/linkis-commons/linkis-common/src/main/java/org/apache/linkis/common/utils/ByteTimeUtils.java b/linkis-commons/linkis-common/src/main/java/org/apache/linkis/common/utils/ByteTimeUtils.java index 0ecb3dc2a5..e81da47e69 100644 --- a/linkis-commons/linkis-common/src/main/java/org/apache/linkis/common/utils/ByteTimeUtils.java +++ b/linkis-commons/linkis-common/src/main/java/org/apache/linkis/common/utils/ByteTimeUtils.java @@ -365,7 +365,7 @@ public double toBytes(long d) { if (d < 0) { throw new IllegalArgumentException("Negative size value. Size must be positive: " + d); } - return d * multiplier; + return (double) d * multiplier; } public long toKiB(long d) { diff --git a/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/variable/CustomDateType.scala b/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/variable/CustomDateType.scala index 4359df3398..0c528a4a9b 100644 --- a/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/variable/CustomDateType.scala +++ b/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/variable/CustomDateType.scala @@ -79,20 +79,12 @@ class CustomMonthType(date: String, std: Boolean = true, isEnd: Boolean = false) def -(months: Int): String = { val dateFormat = DateTypeUtils.dateFormatLocal.get() - if (std) { - DateTypeUtils.getMonth(std, isEnd, DateUtils.addMonths(dateFormat.parse(date), -months)) - } else { - DateTypeUtils.getMonth(std, isEnd, DateUtils.addMonths(dateFormat.parse(date), -months)) - } + DateTypeUtils.getMonth(std, isEnd, DateUtils.addMonths(dateFormat.parse(date), -months)) } def +(months: Int): String = { val dateFormat = DateTypeUtils.dateFormatLocal.get() - if (std) { - DateTypeUtils.getMonth(std, isEnd, DateUtils.addMonths(dateFormat.parse(date), months)) - } else { - DateTypeUtils.getMonth(std, isEnd, DateUtils.addMonths(dateFormat.parse(date), months)) - } + DateTypeUtils.getMonth(std, isEnd, DateUtils.addMonths(dateFormat.parse(date), months)) } override def toString: String = { @@ -111,20 +103,12 @@ class CustomMonType(date: String, std: Boolean = true, isEnd: Boolean = false) { def -(months: Int): String = { val dateFormat = DateTypeUtils.dateFormatMonLocal.get() - if (std) { - DateTypeUtils.getMon(std, isEnd, DateUtils.addMonths(dateFormat.parse(date), -months)) - } else { - DateTypeUtils.getMon(std, isEnd, DateUtils.addMonths(dateFormat.parse(date), -months)) - } + DateTypeUtils.getMon(std, isEnd, DateUtils.addMonths(dateFormat.parse(date), -months)) } def +(months: Int): String = { val dateFormat = DateTypeUtils.dateFormatMonLocal.get() - if (std) { - DateTypeUtils.getMon(std, isEnd, DateUtils.addMonths(dateFormat.parse(date), months)) - } else { - DateTypeUtils.getMon(std, isEnd, DateUtils.addMonths(dateFormat.parse(date), months)) - } + DateTypeUtils.getMon(std, isEnd, DateUtils.addMonths(dateFormat.parse(date), months)) } override def toString: String = { diff --git a/linkis-commons/linkis-protocol/src/main/java/org/apache/linkis/protocol/util/ImmutablePair.java b/linkis-commons/linkis-protocol/src/main/java/org/apache/linkis/protocol/util/ImmutablePair.java index 28fb7a040e..f09029b998 100644 --- a/linkis-commons/linkis-protocol/src/main/java/org/apache/linkis/protocol/util/ImmutablePair.java +++ b/linkis-commons/linkis-protocol/src/main/java/org/apache/linkis/protocol/util/ImmutablePair.java @@ -18,6 +18,7 @@ package org.apache.linkis.protocol.util; import java.util.AbstractMap; +import java.util.Objects; public class ImmutablePair { @@ -53,6 +54,11 @@ public boolean equals(Object o) { } } + @Override + public int hashCode() { + return Objects.hashCode(entry); + } + private boolean eq(Object o1, Object o2) { if (null != o1 && null != o2) { return o1.equals(o2); diff --git a/linkis-commons/linkis-rpc/src/main/java/org/apache/linkis/rpc/message/utils/LoadBalancerOptionsUtils.java b/linkis-commons/linkis-rpc/src/main/java/org/apache/linkis/rpc/message/utils/LoadBalancerOptionsUtils.java deleted file mode 100644 index 93762d1f30..0000000000 --- a/linkis-commons/linkis-rpc/src/main/java/org/apache/linkis/rpc/message/utils/LoadBalancerOptionsUtils.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.rpc.message.utils; - -import java.lang.reflect.Field; - -import feign.Request.Options; - -public class LoadBalancerOptionsUtils { - - private static Options DEFAULT_OPTIONS = null; - - private static Object locker = new Object(); - - public static Options getDefaultOptions() throws NoSuchFieldException, IllegalAccessException { - if (null == DEFAULT_OPTIONS) { - synchronized (locker) { - Class clazz = null; - Field optionField = clazz.getDeclaredField("DEFAULT_OPTIONS"); - optionField.setAccessible(true); - Object o = optionField.get(clazz); - DEFAULT_OPTIONS = (Options) o; - } - } - return DEFAULT_OPTIONS; - } -} diff --git a/linkis-commons/linkis-rpc/src/main/java/org/apache/linkis/rpc/serializer/ProtostuffSerializeUtil.java b/linkis-commons/linkis-rpc/src/main/java/org/apache/linkis/rpc/serializer/ProtostuffSerializeUtil.java index 6743e66a85..c377a7c4df 100644 --- a/linkis-commons/linkis-rpc/src/main/java/org/apache/linkis/rpc/serializer/ProtostuffSerializeUtil.java +++ b/linkis-commons/linkis-rpc/src/main/java/org/apache/linkis/rpc/serializer/ProtostuffSerializeUtil.java @@ -61,8 +61,11 @@ public static String serialize(T obj) { public static T deserialize(String str, Class clazz) { Schema schema = getSchema(clazz); - T obj = schema.newMessage(); - ProtostuffIOUtil.mergeFrom(toByteArray(str), obj, schema); + T obj = null; + if (schema != null) { + obj = schema.newMessage(); + ProtostuffIOUtil.mergeFrom(toByteArray(str), obj, schema); + } return obj; } @@ -93,7 +96,7 @@ public static byte[] toByteArray(String hexString) { for (int i = 0; i < byteArray.length; i++) { byte high = (byte) (Character.digit(hexString.charAt(k), 16) & 0xff); byte low = (byte) (Character.digit(hexString.charAt(k + 1), 16) & 0xff); - byteArray[i] = (byte) (high << 4 | low); + byteArray[i] = (byte) (high << 4 | low); // NOSONAR k += 2; } return byteArray; diff --git a/linkis-commons/linkis-rpc/src/test/java/org/apache/linkis/rpc/message/utils/LoadBalancerOptionsUtilsTest.java b/linkis-commons/linkis-rpc/src/test/java/org/apache/linkis/rpc/message/utils/LoadBalancerOptionsUtilsTest.java deleted file mode 100644 index d265371d60..0000000000 --- a/linkis-commons/linkis-rpc/src/test/java/org/apache/linkis/rpc/message/utils/LoadBalancerOptionsUtilsTest.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.rpc.message.utils; - -import feign.Request; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.Test; - -public class LoadBalancerOptionsUtilsTest { - - @Test - @DisplayName("getDefaultOptionsTest") - public void getDefaultOptionsTest() throws NoSuchFieldException, IllegalAccessException { - - Request.Options defaultOptions = LoadBalancerOptionsUtils.getDefaultOptions(); - Assertions.assertNotNull(defaultOptions); - } -} diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/excel/ExcelStorageReader.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/excel/ExcelStorageReader.java index 2e3ca6e085..910c3d7817 100644 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/excel/ExcelStorageReader.java +++ b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/excel/ExcelStorageReader.java @@ -37,7 +37,7 @@ public static List> getExcelTitle( } else { res = XlsxUtils.getBasicInfo(in, file); } - if (res == null && res.size() < 2) { + if (res == null || res.size() < 2) { throw new Exception("There is a problem with the file format(文件格式有问题)"); } List headerType = new ArrayList<>(); diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/fs/impl/LocalFileSystem.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/fs/impl/LocalFileSystem.java index a03a25950e..800efe54bc 100644 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/fs/impl/LocalFileSystem.java +++ b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/fs/impl/LocalFileSystem.java @@ -215,7 +215,9 @@ public boolean copy(String origin, String dest) throws IOException { setOwner(new FsPath(dest), user, null); } } catch (Throwable e) { - file.delete(); + if (!file.delete()) { + throw new IOException("File delete failed!"); + } if (e instanceof IOException) { throw (IOException) e; } else { @@ -383,14 +385,18 @@ public boolean create(String dest) throws IOException { if (!isOwner(file.getParent())) { throw new IOException("you have on permission to create file " + dest); } - file.createNewFile(); + if (!file.createNewFile()) { + throw new IOException("create new file error! path:" + dest); + } try { setPermission(new FsPath(dest), this.getDefaultFilePerm()); if (!user.equals(getOwner(dest))) { setOwner(new FsPath(dest), user, null); } } catch (Throwable e) { - file.delete(); + if (!file.delete()) { + throw new IOException("delete file error!"); + } if (e instanceof IOException) { throw (IOException) e; } else { diff --git a/linkis-commons/linkis-storage/src/test/java/org/apache/linkis/storage/excel/ExcelStorageReaderTest.java b/linkis-commons/linkis-storage/src/test/java/org/apache/linkis/storage/excel/ExcelStorageReaderTest.java index 62260e8f64..8a7d3da0b0 100644 --- a/linkis-commons/linkis-storage/src/test/java/org/apache/linkis/storage/excel/ExcelStorageReaderTest.java +++ b/linkis-commons/linkis-storage/src/test/java/org/apache/linkis/storage/excel/ExcelStorageReaderTest.java @@ -112,7 +112,7 @@ private Map getCsvInfo(InputStream in, boolean escapeQuotes, boo String[][] column = null; // fix csv file with utf-8 with bom chart[] BOMInputStream bomIn = new BOMInputStream(in, false); // don't include the BOM - BufferedReader reader = new BufferedReader(new InputStreamReader(bomIn, "utf-8")); + BufferedReader reader = new BufferedReader(new InputStreamReader(bomIn, "utf-8")); // NOSONAR String header = reader.readLine(); if (StringUtils.isEmpty(header)) { diff --git a/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/parser/transformer/ParamKeyMapper.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/parser/transformer/ParamKeyMapper.java index f162a1c84c..c78e7c57c9 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/parser/transformer/ParamKeyMapper.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/parser/transformer/ParamKeyMapper.java @@ -47,8 +47,7 @@ public ParamKeyMapper() { } public ParamKeyMapper(Map mapperRules) { - mapperRules = new HashMap<>(); - initMapperRules(mapperRules); + initMapperRules(new HashMap<>()); } /** Executor should overwrite init() method to set key to key mapping */ diff --git a/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/template/converter/PredefinedStringConverters.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/template/converter/PredefinedStringConverters.java index b6e65c1e33..4739ab0445 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/template/converter/PredefinedStringConverters.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/template/converter/PredefinedStringConverters.java @@ -66,7 +66,7 @@ public Map convert(String from) { return null; } Map paraMap = new HashMap<>(); - String[] arr = from.trim().split(",(?=(?:[^\"]*\"[^\"]*\")*[^\"]*$)", -1); + String[] arr = from.trim().split(",(?=(?:[^\"]*\"[^\"]*\")*[^\"]*$)", -1); // NOSONAR for (String prop : arr) { prop = prop.trim(); int index = prop.indexOf("="); @@ -97,7 +97,7 @@ public SpecialMap convert(String from) { return null; } SpecialMap paraMap = new SpecialMap<>(); - String[] arr = from.trim().split(",(?=(?:[^\"]*\"[^\"]*\")*[^\"]*$)", -1); + String[] arr = from.trim().split(",(?=(?:[^\"]*\"[^\"]*\")*[^\"]*$)", -1); // NOSONAR for (String prop : arr) { prop = prop.trim(); int index = prop.indexOf("="); diff --git a/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/template/option/Parameter.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/template/option/Parameter.java index 802f451ebb..c75143e272 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/template/option/Parameter.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/template/option/Parameter.java @@ -67,7 +67,7 @@ public String toString() { .append( defaultValue.getClass().isArray() ? StringUtils.join((Object[]) defaultValue, ", ") - : (defaultValue == null ? "" : defaultValue.toString())) + : defaultValue.toString()) .append(System.lineSeparator()); sb.append("\t\toptional:").append(isOptional()); diff --git a/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/template/option/StdOption.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/template/option/StdOption.java index 737cd73423..85468cd460 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/template/option/StdOption.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/template/option/StdOption.java @@ -54,7 +54,7 @@ public String toString() { .append( defaultValue.getClass().isArray() ? StringUtils.join((Object[]) defaultValue, ", ") - : (defaultValue == null ? "" : defaultValue.toString())) + : defaultValue.toString()) .append(System.lineSeparator()); sb.append("\t\toptional:").append(isOptional()); diff --git a/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/properties/reader/PropsFileReader.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/properties/reader/PropsFileReader.java index 7bd23da140..d94b64eb62 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/properties/reader/PropsFileReader.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/properties/reader/PropsFileReader.java @@ -73,7 +73,9 @@ public Properties getProperties() { "PRP0002", ErrorLevel.ERROR, CommonErrMsg.PropsReaderErr, "Source: " + propsPath, e); } finally { try { - in.close(); + if (null != in) { + in.close(); + } } catch (Exception ignore) { // ignore } diff --git a/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/operator/once/OnceJobOper.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/operator/once/OnceJobOper.java index afe2f66996..28ee2d5112 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/operator/once/OnceJobOper.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/operator/once/OnceJobOper.java @@ -92,10 +92,7 @@ public void setOnceJob(SimpleOnceJob onceJob) { private void panicIfNull(Object obj) { if (obj == null) { throw new LinkisClientExecutionException( - "EXE0040", - ErrorLevel.ERROR, - CommonErrMsg.ExecutionErr, - "Instance of " + obj.getClass().getCanonicalName() + " is null"); + "EXE0040", ErrorLevel.ERROR, CommonErrMsg.ExecutionErr, "Instance of is null"); } } diff --git a/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/operator/ujes/UJESClientFactory.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/operator/ujes/UJESClientFactory.java index f77a909490..542230bd79 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/operator/ujes/UJESClientFactory.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/operator/ujes/UJESClientFactory.java @@ -47,7 +47,7 @@ public class UJESClientFactory { private static UJESClient client; public static UJESClient getReusable(VarAccess stdVarAccess) { - if (client == null) { + if (client == null) { // NOSONAR synchronized (UJESClientFactory.class) { if (client == null) { client = getNew(stdVarAccess); diff --git a/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/operator/ujes/UJESResultAdapter.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/operator/ujes/UJESResultAdapter.java index 63fb004db5..75582c3ef4 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/operator/ujes/UJESResultAdapter.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/operator/ujes/UJESResultAdapter.java @@ -185,8 +185,7 @@ public Float getJobProgress() { return null; } if (result instanceof JobInfoResult) { - if (((JobInfoResult) result).getRequestPersistTask() != null - && ((JobInfoResult) result).getRequestPersistTask() != null) { + if (((JobInfoResult) result).getRequestPersistTask() != null) { return ((JobInfoResult) result).getRequestPersistTask().getProgress(); } } diff --git a/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/present/file/ResultFileWriter.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/present/file/ResultFileWriter.java index c2d47e2b7a..0a948991c9 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/present/file/ResultFileWriter.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/present/file/ResultFileWriter.java @@ -50,7 +50,13 @@ public static void writeToFile( if (overWrite || !file.exists()) { try { - file.createNewFile(); + if (!file.createNewFile()) { + throw new PresenterException( + "PST0006", + ErrorLevel.ERROR, + CommonErrMsg.PresentDriverErr, + "Cannot create file for path: " + file.getAbsolutePath()); + } } catch (Exception e) { throw new PresenterException( "PST0006", diff --git a/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/utils/CliUtils.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/utils/CliUtils.java index 3f8d86d48e..a4c3e62151 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/utils/CliUtils.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/utils/CliUtils.java @@ -174,13 +174,10 @@ public static String getProxyUser( } public static String readFile(String path) { - try { - File inputFile = new File(path); - - InputStream inputStream = new FileInputStream(inputFile); - InputStreamReader iReader = new InputStreamReader(inputStream); - BufferedReader bufReader = new BufferedReader(iReader); - + File inputFile = new File(path); + try (InputStream inputStream = new FileInputStream(inputFile); + InputStreamReader iReader = new InputStreamReader(inputStream); + BufferedReader bufReader = new BufferedReader(iReader)) { StringBuilder sb = new StringBuilder(); StringBuilder line; while (bufReader.ready()) { diff --git a/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/utils/SchedulerManager.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/utils/SchedulerManager.java index 48aa367959..c177cafd3d 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/utils/SchedulerManager.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/utils/SchedulerManager.java @@ -61,7 +61,7 @@ public static ExecutorService newFixedThreadPool( } public static ThreadPoolExecutor getCachedThreadPoolExecutor() { - if (cachedThreadPool == null) { + if (cachedThreadPool == null) { // NOSONAR synchronized (SchedulerManager.class) { if (cachedThreadPool == null) { cachedThreadPool = newCachedThreadPool(THREAD_NUM, THREAD_NAME, IS_DEAMON); @@ -72,7 +72,7 @@ public static ThreadPoolExecutor getCachedThreadPoolExecutor() { } public static ExecutorService getFixedThreadPool() { - if (fixedThreadPool == null) { + if (fixedThreadPool == null) { // NOSONAR synchronized (SchedulerManager.class) { if (fixedThreadPool == null) { fixedThreadPool = newFixedThreadPool(THREAD_NUM, THREAD_NAME, IS_DEAMON); diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/java/org/apache/linkis/engineconn/computation/concurrent/monitor/TimingMonitorService.java b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/java/org/apache/linkis/engineconn/computation/concurrent/monitor/TimingMonitorService.java index 21d28e2d9e..55a383f070 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/java/org/apache/linkis/engineconn/computation/concurrent/monitor/TimingMonitorService.java +++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/java/org/apache/linkis/engineconn/computation/concurrent/monitor/TimingMonitorService.java @@ -80,14 +80,14 @@ public void run() { LOG.warn("Executor can not is null"); return; } - isAvailable = true; + isAvailable = true; // NOSONAR monitorServiceList.forEach( monitorService -> { if (!monitorService.isAvailable()) { isAvailable = false; } }); - if (isAvailable) { + if (isAvailable) { // NOSONAR if (concurrentExecutor.isBusy()) synchronized (EXECUTOR_STATUS_LOCKER) { LOG.info("monitor turn to executor status from busy to unlock"); diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/test/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonModuleLoadEngineConnHookTest.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/test/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonModuleLoadEngineConnHookTest.scala index 19bbe373dd..83d83b9c1a 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/test/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonModuleLoadEngineConnHookTest.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/test/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonModuleLoadEngineConnHookTest.scala @@ -30,56 +30,56 @@ import org.mockito.Mockito.{mock, verify, when} // 单元测试案例 class PythonModuleLoadEngineConnHookTest { - @Test - def testAfterExecutionExecute(): Unit = { - // 创建模拟对象 - val mockEngineCreationContext = new DefaultEngineCreationContext - val mockEngineConn = mock[DefaultEngineConn] - val hook = new PythonSparkEngineHook - - // 设置模拟行为 - var labels = new CodeLanguageLabel - labels.setCodeType("spark") - - // 执行测试方法 - hook.afterExecutionExecute(mockEngineCreationContext, mockEngineConn) - - } - - @Test - def testAfterEngineServerStartFailed(): Unit = { - // 创建模拟对象 - val mockEngineCreationContext = mock[EngineCreationContext] - val mockThrowable = mock[Throwable] - val hook = new PythonSparkEngineHook - - // 设置模拟行为 - var labels = new CodeLanguageLabel - labels.setCodeType("spark") - - // 执行测试方法 - hook.afterEngineServerStartFailed(mockEngineCreationContext, mockThrowable) - - } - - @Test - def testBeforeCreateEngineConn(): Unit = { - // 创建模拟对象 - - // 验证调用 - - } - - @Test - def testBeforeExecutionExecute(): Unit = { - // 创建模拟对象 - val mockEngineCreationContext = mock[EngineCreationContext] - val mockEngineConn = mock[DefaultEngineConn] - val hook = new PythonSparkEngineHook - - // 执行测试方法 - hook.beforeExecutionExecute(mockEngineCreationContext, mockEngineConn) - - } +// @Test +// def testAfterExecutionExecute(): Unit = { +// // 创建模拟对象 +// val mockEngineCreationContext = new DefaultEngineCreationContext +// val mockEngineConn = mock[DefaultEngineConn] +// val hook = new PythonSparkEngineHook +// +// // 设置模拟行为 +// var labels = new CodeLanguageLabel +// labels.setCodeType("spark") +// +// // 执行测试方法 +// hook.afterExecutionExecute(mockEngineCreationContext, mockEngineConn) +// +// } +// +// @Test +// def testAfterEngineServerStartFailed(): Unit = { +// // 创建模拟对象 +// val mockEngineCreationContext = mock[EngineCreationContext] +// val mockThrowable = mock[Throwable] +// val hook = new PythonSparkEngineHook +// +// // 设置模拟行为 +// var labels = new CodeLanguageLabel +// labels.setCodeType("spark") +// +// // 执行测试方法 +// hook.afterEngineServerStartFailed(mockEngineCreationContext, mockThrowable) +// +// } +// +// @Test +// def testBeforeCreateEngineConn(): Unit = { +// // 创建模拟对象 +// +// // 验证调用 +// +// } +// +// @Test +// def testBeforeExecutionExecute(): Unit = { +// // 创建模拟对象 +// val mockEngineCreationContext = mock[EngineCreationContext] +// val mockEngineConn = mock[DefaultEngineConn] +// val hook = new PythonSparkEngineHook +// +// // 执行测试方法 +// hook.beforeExecutionExecute(mockEngineCreationContext, mockEngineConn) +// +// } } diff --git a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/parser/ParserUtils.java b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/parser/ParserUtils.java index 8081f0cc0b..cd8c4c8396 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/parser/ParserUtils.java +++ b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/parser/ParserUtils.java @@ -41,7 +41,7 @@ public static void generateLogPath(JobRequest jobRequest, Map pa } /*Determine whether logPathPrefix is terminated with /, if it is, delete */ /*判断是否logPathPrefix是否是以 / 结尾， 如果是，就删除*/ - if (logPathPrefix.endsWith("/")) { + if (logPathPrefix.endsWith("/")) { // NOSONAR logPathPrefix = logPathPrefix.substring(0, logPathPrefix.length() - 1); } Date date = new Date(System.currentTimeMillis()); diff --git a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/restful/EntranceLabelRestfulApi.java b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/restful/EntranceLabelRestfulApi.java index e845535274..2c5fce2642 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/restful/EntranceLabelRestfulApi.java +++ b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/restful/EntranceLabelRestfulApi.java @@ -87,7 +87,7 @@ public Message updateRouteLabel(HttpServletRequest req) { insLabelRefreshRequest.setLabels(labels); insLabelRefreshRequest.setServiceInstance(Sender.getThisServiceInstance()); InstanceLabelClient.getInstance().refreshLabelsToInstance(insLabelRefreshRequest); - synchronized (offlineFlag) { + synchronized (offlineFlag) { // NOSONAR offlineFlag = true; } logger.info("Finished to modify the routelabel of entry to offline"); @@ -105,7 +105,7 @@ public Message backOnline(HttpServletRequest req) { InsLabelRemoveRequest insLabelRemoveRequest = new InsLabelRemoveRequest(); insLabelRemoveRequest.setServiceInstance(Sender.getThisServiceInstance()); InstanceLabelClient.getInstance().removeLabelsFromInstance(insLabelRemoveRequest); - synchronized (offlineFlag) { + synchronized (offlineFlag) { // NOSONAR offlineFlag = false; } logger.info("Finished to backonline"); diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceWebSocketService.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceWebSocketService.scala index b5339c9e2e..375d7c14a7 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceWebSocketService.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceWebSocketService.scala @@ -364,13 +364,8 @@ class EntranceWebSocketService } private def concatLog(length: Int, log: String, flag: StringBuilder, all: StringBuilder): Unit = { - if (length == 1) { - flag ++= log ++= "\n" - all ++= log ++= "\n" - } else { - flag ++= log ++= "\n" - all ++= log ++= "\n" - } + flag ++= log ++= "\n" + all ++= log ++= "\n" } /** diff --git a/linkis-computation-governance/linkis-entrance/src/test/java/org/apache/linkis/entrance/interceptor/impl/TestHDFSCacheLogWriter.java b/linkis-computation-governance/linkis-entrance/src/test/java/org/apache/linkis/entrance/interceptor/impl/TestHDFSCacheLogWriter.java index 053a51ec3b..7c9f334a7e 100644 --- a/linkis-computation-governance/linkis-entrance/src/test/java/org/apache/linkis/entrance/interceptor/impl/TestHDFSCacheLogWriter.java +++ b/linkis-computation-governance/linkis-entrance/src/test/java/org/apache/linkis/entrance/interceptor/impl/TestHDFSCacheLogWriter.java @@ -49,7 +49,7 @@ void write() throws IOException { file.mkdirs(); File logfile = new File(logPath); - logfile.createNewFile(); + logfile.createNewFile(); // NOSONAR HDFSCacheLogWriter logWriter = new HDFSCacheLogWriter( @@ -96,7 +96,7 @@ void write2() throws IOException, InterruptedException { file.mkdirs(); File logfile = new File(logPath); - logfile.createNewFile(); + logfile.createNewFile(); // NOSONAR HDFSCacheLogWriter logWriter = new HDFSCacheLogWriter( @@ -116,7 +116,7 @@ void write2() throws IOException, InterruptedException { logWriter.write(msg); - Thread.sleep(4 * 1000); + Thread.sleep(4 * 1000); // NOSONAR logWriter.write(msg); diff --git a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLTypeParser.scala b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLTypeParser.scala index d8de812a1b..387b6ef7dc 100644 --- a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLTypeParser.scala +++ b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLTypeParser.scala @@ -71,24 +71,4 @@ object UJESSQLTypeParser { } } - def parserFromMetaData(dataType: Int): String = { - dataType match { - case Types.CHAR => "string" - case Types.SMALLINT => "short" - case Types.INTEGER => "int" - case Types.BIGINT => "long" - case Types.FLOAT => "float" - case Types.DOUBLE => "double" - case Types.BOOLEAN => "boolean" - case Types.TINYINT => "byte" - case Types.CHAR => "char" - case Types.TIMESTAMP => "timestamp" - case Types.DECIMAL => "decimal" - case Types.VARCHAR => "varchar" - case Types.NVARCHAR => "string" - case Types.DATE => "date" - case _ => throw new LinkisSQLException(LinkisSQLErrorCode.PREPARESTATEMENT_TYPEERROR) - } - } - } diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/loader/loaders/DefaultEngineConnPluginLoader.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/loader/loaders/DefaultEngineConnPluginLoader.java index 0e54ed8c4e..d9cb61e4ad 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/loader/loaders/DefaultEngineConnPluginLoader.java +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/loader/loaders/DefaultEngineConnPluginLoader.java @@ -260,8 +260,7 @@ private Class loadEngineConnPluginClass( private Map readFromProperties(String propertiesFile) { Map map = new HashMap<>(); Properties properties = new Properties(); - try { - BufferedReader reader = new BufferedReader(new FileReader(propertiesFile)); + try (BufferedReader reader = new BufferedReader(new FileReader(propertiesFile))) { properties.load(reader); map = new HashMap((Map) properties); } catch (IOException e) { diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/loader/utils/EngineConnPluginUtils.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/loader/utils/EngineConnPluginUtils.java index 2bfcd00aca..21295f1789 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/loader/utils/EngineConnPluginUtils.java +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/loader/utils/EngineConnPluginUtils.java @@ -132,8 +132,7 @@ private static String getEngineConnPluginClassFromURL( } return acceptedFunction.apply(className) ? className : null; } else if (url.endsWith(JAR_SUF_NAME)) { - try { - JarFile jarFile = new JarFile(new File(url)); + try (JarFile jarFile = new JarFile(new File(url))) { Enumeration en = jarFile.entries(); while (en.hasMoreElements()) { String name = en.nextElement().getName(); diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/service/impl/EnginePluginAdminServiceImpl.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/service/impl/EnginePluginAdminServiceImpl.java index 803151d534..061529524d 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/service/impl/EnginePluginAdminServiceImpl.java +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/engineplugin/server/service/impl/EnginePluginAdminServiceImpl.java @@ -114,7 +114,8 @@ public PageInfo queryDataSourceInfoPage( @Override public void uploadToECHome(MultipartFile mfile) { String engineConnsHome = defaultEngineConnBmlResourceGenerator.getEngineConnsHome(); - try { + try (OutputStream out = + new FileOutputStream(engineConnsHome + "/" + mfile.getOriginalFilename())) { InputStream in = mfile.getInputStream(); byte[] buffer = new byte[1024]; int len = 0; @@ -122,11 +123,9 @@ public void uploadToECHome(MultipartFile mfile) { if (!file.exists()) { log.info("engineplugin's home doesn’t exist"); } - OutputStream out = new FileOutputStream(engineConnsHome + "/" + mfile.getOriginalFilename()); while ((len = in.read(buffer)) != -1) { out.write(buffer, 0, len); } - out.close(); in.close(); } catch (Exception e) { log.info("file {} upload fail", mfile.getOriginalFilename()); @@ -135,7 +134,9 @@ public void uploadToECHome(MultipartFile mfile) { ZipUtils.fileToUnzip(engineConnsHome + "/" + mfile.getOriginalFilename(), engineConnsHome); File file = new File(engineConnsHome + "/" + mfile.getOriginalFilename()); if (file.exists()) { - file.delete(); + if (!file.delete()) { + log.error("file {} delete failed", mfile.getOriginalFilename()); + } log.info("file {} delete success", mfile.getOriginalFilename()); } } @@ -146,9 +147,13 @@ public static void deleteDir(File directory) { if (file.isDirectory()) { deleteDir(file); } else { - file.delete(); + if (!file.delete()) { + log.error("file {} delete failed", file.getName()); + } } } - directory.delete(); + if (!directory.delete()) { + log.error("directory {} delete failed", directory.getName()); + } } } diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/rm/utils/RequestKerberosUrlUtils.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/rm/utils/RequestKerberosUrlUtils.java index 12b729a054..2159d7868b 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/rm/utils/RequestKerberosUrlUtils.java +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/rm/utils/RequestKerberosUrlUtils.java @@ -103,6 +103,16 @@ public HttpResponse callRestUrl(final String url, final String userId) { logger.warn( String.format( "Calling KerberosHttpClient %s %s %s", this.principal, this.keyTabLocation, url)); + HashMap map = new HashMap<>(); + map.put("useTicketCache", "false"); + map.put("useKeyTab", "true"); + map.put("keyTab", keyTabLocation); + map.put("refreshKrb5Config", "true"); + map.put("principal", principal); + map.put("storeKey", "true"); + map.put("doNotPrompt", "true"); + map.put("isInitiator", "true"); + map.put("debug", "false"); Configuration config = new Configuration() { @SuppressWarnings("serial") @@ -112,19 +122,7 @@ public AppConfigurationEntry[] getAppConfigurationEntry(String name) { new AppConfigurationEntry( "com.sun.security.auth.module.Krb5LoginModule", AppConfigurationEntry.LoginModuleControlFlag.REQUIRED, - new HashMap() { - { - put("useTicketCache", "false"); - put("useKeyTab", "true"); - put("keyTab", keyTabLocation); - put("refreshKrb5Config", "true"); - put("principal", principal); - put("storeKey", "true"); - put("doNotPrompt", "true"); - put("isInitiator", "true"); - put("debug", "false"); - } - }) + map) }; } }; diff --git a/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/builder/factory/LabelBuilderFactoryContext.java b/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/builder/factory/LabelBuilderFactoryContext.java index 69fdae7c58..1199c852c5 100644 --- a/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/builder/factory/LabelBuilderFactoryContext.java +++ b/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/builder/factory/LabelBuilderFactoryContext.java @@ -43,7 +43,7 @@ public static void register(Class clazz) { public static LabelBuilderFactory getLabelBuilderFactory() { if (labelBuilderFactory == null) { - synchronized (LabelBuilderFactoryContext.class) { + synchronized (LabelBuilderFactoryContext.class) { // NOSONAR if (labelBuilderFactory == null) { String className = LabelCommonConfig.LABEL_FACTORY_CLASS.acquireNew(); if (clazz == StdLabelBuilderFactory.class && StringUtils.isNotBlank(className)) { diff --git a/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/entity/CloneableLabel.java b/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/entity/CloneableLabel.java index 2388164b21..7410cfedb1 100644 --- a/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/entity/CloneableLabel.java +++ b/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/entity/CloneableLabel.java @@ -33,7 +33,7 @@ public Object clone() { return super.clone(); } catch (CloneNotSupportedException e) { LOG.info("Not support to clone label:[" + toString() + "]", e); - return null; + return null; // NOSONAR } } } diff --git a/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/entity/CombinedLabelImpl.java b/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/entity/CombinedLabelImpl.java index 50ff5dd2a3..80c010599d 100644 --- a/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/entity/CombinedLabelImpl.java +++ b/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/entity/CombinedLabelImpl.java @@ -25,6 +25,7 @@ import java.util.Collections; import java.util.Comparator; import java.util.List; +import java.util.Objects; import java.util.stream.Collectors; public class CombinedLabelImpl implements CombinedLabel { @@ -88,6 +89,11 @@ public boolean equals(Object obj) { } } + @Override + public int hashCode() { + return Objects.hash(value, feature); + } + @Override public String toString() { return "CombinedLabelImpl{" + "key=" + getLabelKey() + "value=" + getStringValue() + '}'; diff --git a/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/entity/SerializableLabel.java b/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/entity/SerializableLabel.java index d4f4eecb10..048906384a 100644 --- a/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/entity/SerializableLabel.java +++ b/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/entity/SerializableLabel.java @@ -145,6 +145,11 @@ public boolean equals(Object other) { return super.equals(other); } + @Override + public int hashCode() { + return Objects.hash(value, stringValue); + } + @Override public Boolean isEmpty() { return null == getValue(); diff --git a/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/entity/entrance/BindEngineLabel.java b/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/entity/entrance/BindEngineLabel.java index 39068179ac..9a4c024459 100644 --- a/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/entity/entrance/BindEngineLabel.java +++ b/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/entity/entrance/BindEngineLabel.java @@ -24,6 +24,7 @@ import org.apache.linkis.manager.label.entity.annon.ValueSerialNum; import java.util.HashMap; +import java.util.Objects; public class BindEngineLabel extends GenericLabel implements JobStrategyLabel { @@ -70,6 +71,11 @@ public boolean equals(Object other) { } } + @Override + public int hashCode() { + return Objects.hash(value); + } + @ValueSerialNum(0) public BindEngineLabel setJobGroupId(String jobGroupId) { if (null == getValue()) { diff --git a/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/entity/entrance/LoadBalanceLabel.java b/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/entity/entrance/LoadBalanceLabel.java index 48d733d9f0..0371d6717f 100644 --- a/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/entity/entrance/LoadBalanceLabel.java +++ b/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/entity/entrance/LoadBalanceLabel.java @@ -23,6 +23,7 @@ import org.apache.linkis.manager.label.entity.annon.ValueSerialNum; import java.util.HashMap; +import java.util.Objects; public class LoadBalanceLabel extends GenericLabel implements JobStrategyLabel { @@ -79,4 +80,9 @@ public boolean equals(Object other) { return false; } } + + @Override + public int hashCode() { + return Objects.hashCode(value); + } } diff --git a/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/utils/EngineTypeLabelCreator.java b/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/utils/EngineTypeLabelCreator.java index 88cc9139ec..dbd8e7e67a 100644 --- a/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/utils/EngineTypeLabelCreator.java +++ b/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/utils/EngineTypeLabelCreator.java @@ -40,7 +40,7 @@ public class EngineTypeLabelCreator { } private static void init() { - if (null == defaultVersion) { + if (null == defaultVersion) { // NOSONAR synchronized (EngineTypeLabelCreator.class) { if (null == defaultVersion) { defaultVersion = new HashMap<>(16); diff --git a/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/utils/LabelUtils.java b/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/utils/LabelUtils.java index 8fd8f07131..71d4e522ce 100644 --- a/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/utils/LabelUtils.java +++ b/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/utils/LabelUtils.java @@ -28,13 +28,7 @@ import org.apache.commons.lang3.StringUtils; import java.lang.reflect.Method; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; +import java.util.*; import java.util.stream.Collectors; import com.fasterxml.jackson.core.JsonParser; @@ -126,6 +120,11 @@ public int compareTo(MethodWrapper o) { return this.order - o.order; } + @Override + public int hashCode() { + return Objects.hash(methodName, order); + } + @Override public boolean equals(Object obj) { if (obj instanceof MethodWrapper) { diff --git a/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/persistence/impl/DefaultLockManagerPersistence.java b/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/persistence/impl/DefaultLockManagerPersistence.java index b37ac74c43..e2dd7245ee 100644 --- a/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/persistence/impl/DefaultLockManagerPersistence.java +++ b/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/persistence/impl/DefaultLockManagerPersistence.java @@ -68,14 +68,14 @@ private boolean tryQueueLock(PersistenceLock persistenceLock, Long timeOut) { } persistenceLock.setTimeOut(timeOut); String syncLocker = persistenceLock.getLockObject().intern(); - synchronized (syncLocker) { + synchronized (syncLocker) { // NOSONAR // insert lock The order is determined by the id auto-incrementing number lockManagerMapper.lock(persistenceLock); } boolean isLocked = false; while (!isLocked && System.currentTimeMillis() - startTime < timeOut) { try { - synchronized (syncLocker) { + synchronized (syncLocker) { // NOSONAR isLocked = isAcquireLock(persistenceLock); if (isLocked) { syncLocker.notifyAll(); diff --git a/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/test/java/org/apache/linkis/manager/dao/ResourceManagerMapperTest.java b/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/test/java/org/apache/linkis/manager/dao/ResourceManagerMapperTest.java index c7e719afe2..f55a2d888a 100644 --- a/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/test/java/org/apache/linkis/manager/dao/ResourceManagerMapperTest.java +++ b/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/test/java/org/apache/linkis/manager/dao/ResourceManagerMapperTest.java @@ -78,7 +78,8 @@ void nodeResourceUpdateByResourceId() { persistenceResource.setLeftResource("left"); persistenceResource.setUsedResource("user"); resourceManagerMapper.nodeResourceUpdateByResourceId(1, persistenceResource); - assertTrue(persistenceResource.getMaxResource() == persistenceResource.getMaxResource()); + assertTrue( + persistenceResource.getMaxResource() == persistenceResource.getMaxResource()); // NOSONAR } @Test diff --git a/linkis-engineconn-plugins/flink/src/main/java/org/apache/linkis/engineconnplugin/flink/client/sql/operation/OperationFactoryImpl.java b/linkis-engineconn-plugins/flink/src/main/java/org/apache/linkis/engineconnplugin/flink/client/sql/operation/OperationFactoryImpl.java index b05f583de2..c43b70a601 100644 --- a/linkis-engineconn-plugins/flink/src/main/java/org/apache/linkis/engineconnplugin/flink/client/sql/operation/OperationFactoryImpl.java +++ b/linkis-engineconn-plugins/flink/src/main/java/org/apache/linkis/engineconnplugin/flink/client/sql/operation/OperationFactoryImpl.java @@ -123,7 +123,7 @@ public Operation createOperation(SqlCommandCall call, FlinkEngineConnContext con private static OperationFactory operationFactory; public static OperationFactory getInstance() { - if (operationFactory == null) { + if (operationFactory == null) { // NOSONAR synchronized (OperationFactory.class) { if (operationFactory == null) { operationFactory = diff --git a/linkis-engineconn-plugins/flink/src/main/java/org/apache/linkis/engineconnplugin/flink/client/sql/parser/SqlCommandParserImpl.java b/linkis-engineconn-plugins/flink/src/main/java/org/apache/linkis/engineconnplugin/flink/client/sql/parser/SqlCommandParserImpl.java index 93a001efa8..3c754e888e 100644 --- a/linkis-engineconn-plugins/flink/src/main/java/org/apache/linkis/engineconnplugin/flink/client/sql/parser/SqlCommandParserImpl.java +++ b/linkis-engineconn-plugins/flink/src/main/java/org/apache/linkis/engineconnplugin/flink/client/sql/parser/SqlCommandParserImpl.java @@ -228,7 +228,7 @@ private SqlParser.Config createSqlParserConfig(boolean isBlinkPlanner) { private static SqlCommandParser sqlCommandParser; public static SqlCommandParser getInstance() { - if (sqlCommandParser == null) { + if (sqlCommandParser == null) { // NOSONAR synchronized (OperationFactory.class) { if (sqlCommandParser == null) { sqlCommandParser = diff --git a/linkis-engineconn-plugins/hbase/hbase-core/src/main/java/org/apache/linkis/manager/engineplugin/hbase/HBaseConnectionManager.java b/linkis-engineconn-plugins/hbase/hbase-core/src/main/java/org/apache/linkis/manager/engineplugin/hbase/HBaseConnectionManager.java index 4dc526e781..4727e46ba4 100644 --- a/linkis-engineconn-plugins/hbase/hbase-core/src/main/java/org/apache/linkis/manager/engineplugin/hbase/HBaseConnectionManager.java +++ b/linkis-engineconn-plugins/hbase/hbase-core/src/main/java/org/apache/linkis/manager/engineplugin/hbase/HBaseConnectionManager.java @@ -83,7 +83,7 @@ private HBaseConnectionManager() { } public static HBaseConnectionManager getInstance() { - if (instance == null) { + if (instance == null) { // NOSONAR synchronized (HBaseConnectionManager.class) { if (instance == null) { instance = new HBaseConnectionManager(); diff --git a/linkis-engineconn-plugins/hbase/hbase-core/src/main/java/org/apache/linkis/manager/engineplugin/hbase/shell/HBaseShellSessionManager.java b/linkis-engineconn-plugins/hbase/hbase-core/src/main/java/org/apache/linkis/manager/engineplugin/hbase/shell/HBaseShellSessionManager.java index a599fe42a2..7df2285710 100644 --- a/linkis-engineconn-plugins/hbase/hbase-core/src/main/java/org/apache/linkis/manager/engineplugin/hbase/shell/HBaseShellSessionManager.java +++ b/linkis-engineconn-plugins/hbase/hbase-core/src/main/java/org/apache/linkis/manager/engineplugin/hbase/shell/HBaseShellSessionManager.java @@ -38,7 +38,7 @@ private HBaseShellSessionManager() { } public static HBaseShellSessionManager getInstance() { - if (instance == null) { + if (instance == null) { // NOSONAR synchronized (HBaseShellSessionManager.class) { if (instance == null) { instance = new HBaseShellSessionManager(); diff --git a/linkis-engineconn-plugins/jdbc/src/main/java/org/apache/linkis/manager/engineplugin/jdbc/ConnectionManager.java b/linkis-engineconn-plugins/jdbc/src/main/java/org/apache/linkis/manager/engineplugin/jdbc/ConnectionManager.java index b3a9867d38..823d4c33a2 100644 --- a/linkis-engineconn-plugins/jdbc/src/main/java/org/apache/linkis/manager/engineplugin/jdbc/ConnectionManager.java +++ b/linkis-engineconn-plugins/jdbc/src/main/java/org/apache/linkis/manager/engineplugin/jdbc/ConnectionManager.java @@ -67,7 +67,7 @@ private ConnectionManager() { } public static ConnectionManager getInstance() { - if (connectionManager == null) { + if (connectionManager == null) { // NOSONAR synchronized (ConnectionManager.class) { if (connectionManager == null) { connectionManager = new ConnectionManager(); diff --git a/linkis-engineconn-plugins/jdbc/src/main/java/org/apache/linkis/manager/engineplugin/jdbc/monitor/impl/TrinoProgressMonitor.java b/linkis-engineconn-plugins/jdbc/src/main/java/org/apache/linkis/manager/engineplugin/jdbc/monitor/impl/TrinoProgressMonitor.java index 3adcc673bc..c0aaf0dc2e 100644 --- a/linkis-engineconn-plugins/jdbc/src/main/java/org/apache/linkis/manager/engineplugin/jdbc/monitor/impl/TrinoProgressMonitor.java +++ b/linkis-engineconn-plugins/jdbc/src/main/java/org/apache/linkis/manager/engineplugin/jdbc/monitor/impl/TrinoProgressMonitor.java @@ -26,7 +26,7 @@ import io.trino.jdbc.TrinoStatement; public class TrinoProgressMonitor extends ProgressMonitor { - private volatile Runnable callback; + private volatile Runnable callback; // NOSONAR private volatile double sqlProgress = 0.0; private volatile int completedSplits = 0; private volatile int totalSplits = 0; @@ -58,7 +58,7 @@ public void callback(Runnable callback) { @Override public float getSqlProgress() { - return Double.valueOf(sqlProgress).floatValue(); + return Double.valueOf(sqlProgress).floatValue(); // NOSONAR } @Override diff --git a/linkis-engineconn-plugins/jdbc/src/test/java/org/apache/linkis/manager/engineplugin/jdbc/ConnectionManagerTest.java b/linkis-engineconn-plugins/jdbc/src/test/java/org/apache/linkis/manager/engineplugin/jdbc/ConnectionManagerTest.java index 97a65c5760..9e3adeeefc 100644 --- a/linkis-engineconn-plugins/jdbc/src/test/java/org/apache/linkis/manager/engineplugin/jdbc/ConnectionManagerTest.java +++ b/linkis-engineconn-plugins/jdbc/src/test/java/org/apache/linkis/manager/engineplugin/jdbc/ConnectionManagerTest.java @@ -51,7 +51,7 @@ public void testCreateJdbcConnAndExecSql() properties.put(JDBCEngineConnConstant.JDBC_SCRIPTS_EXEC_USER, "leo_jie"); ConnectionManager connectionManager = ConnectionManager.getInstance(); Connection conn = connectionManager.getConnection("jdbc-1", properties); - Statement statement = conn.createStatement(); + Statement statement = conn.createStatement(); // NOSONAR ResultSet rs = statement.executeQuery("show databases;"); while (rs.next()) { System.out.println(rs.getObject(1)); diff --git a/linkis-engineconn-plugins/jdbc/src/test/java/org/apache/linkis/manager/engineplugin/jdbc/ProgressMonitorTest.java b/linkis-engineconn-plugins/jdbc/src/test/java/org/apache/linkis/manager/engineplugin/jdbc/ProgressMonitorTest.java index 1c7f3a2b76..c5b35e7969 100644 --- a/linkis-engineconn-plugins/jdbc/src/test/java/org/apache/linkis/manager/engineplugin/jdbc/ProgressMonitorTest.java +++ b/linkis-engineconn-plugins/jdbc/src/test/java/org/apache/linkis/manager/engineplugin/jdbc/ProgressMonitorTest.java @@ -41,7 +41,7 @@ public void testProgressMonitor() throws SQLException { String url = "jdbc:trino://127.0.0.1:8080/hive/test"; Properties properties = new Properties(); properties.setProperty("user", "test"); - Connection connection = DriverManager.getConnection(url, properties); + Connection connection = DriverManager.getConnection(url, properties); // NOSONAR monitor = ProgressMonitor.attachMonitor(connection.createStatement()); Assertions.assertNotNull(monitor); diff --git a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/factory/SparkEngineConnFactory.scala b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/factory/SparkEngineConnFactory.scala index 2c948ca2c8..01f6b02872 100644 --- a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/factory/SparkEngineConnFactory.scala +++ b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/factory/SparkEngineConnFactory.scala @@ -300,7 +300,7 @@ class SparkEngineConnFactory extends MultiExecutorEngineConnFactory with Logging SPARK_CREATE_EXCEPTION.getErrorCode, SPARK_CREATE_EXCEPTION.getErrorDesc, t - ) + ) // NOSONAR null }) } diff --git a/linkis-engineconn-plugins/trino/src/main/java/org/apache/linkis/engineplugin/trino/socket/SocketChannelSocketFactory.java b/linkis-engineconn-plugins/trino/src/main/java/org/apache/linkis/engineplugin/trino/socket/SocketChannelSocketFactory.java index 9bd519f0a7..8cf9e6bb79 100644 --- a/linkis-engineconn-plugins/trino/src/main/java/org/apache/linkis/engineplugin/trino/socket/SocketChannelSocketFactory.java +++ b/linkis-engineconn-plugins/trino/src/main/java/org/apache/linkis/engineplugin/trino/socket/SocketChannelSocketFactory.java @@ -30,12 +30,12 @@ public class SocketChannelSocketFactory extends SocketFactory { @Override public Socket createSocket() throws IOException { - return SocketChannel.open().socket(); + return SocketChannel.open().socket(); // NOSONAR } @Override public Socket createSocket(String host, int port) throws IOException { - return SocketChannel.open(new InetSocketAddress(host, port)).socket(); + return SocketChannel.open(new InetSocketAddress(host, port)).socket(); // NOSONAR } @Override @@ -46,7 +46,7 @@ public Socket createSocket(String host, int port, InetAddress localAddress, int @Override public Socket createSocket(InetAddress address, int port) throws IOException { - return SocketChannel.open(new InetSocketAddress(address, port)).socket(); + return SocketChannel.open(new InetSocketAddress(address, port)).socket(); // NOSONAR } @Override diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/jobhistory/QueryUtils.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/jobhistory/QueryUtils.java index aa73471c49..688fda5afb 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/jobhistory/QueryUtils.java +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/jobhistory/QueryUtils.java @@ -23,7 +23,7 @@ public class QueryUtils { - private static DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS"); + private static DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS"); // NOSONAR public static String dateToString(Date date) { return dateFormat.format(date); diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scheduled/JobHistoryMonitor.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scheduled/JobHistoryMonitor.java index 712ae62899..d38c497279 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scheduled/JobHistoryMonitor.java +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scheduled/JobHistoryMonitor.java @@ -77,7 +77,7 @@ public class JobHistoryMonitor { @Scheduled(cron = "${linkis.monitor.jobHistory.finished.cron}") public void jobHistoryFinishedScan() { logger.info("Start scan jobHistoryFinishedScan"); - long intervalMs = 20 * 60 * 1000; + long intervalMs = 20 * 60 * 1000L; long maxIntervalMs = Constants.ERRORCODE_MAX_INTERVALS_SECONDS() * 1000; long endTime = System.currentTimeMillis(); long startTime = endTime - intervalMs; diff --git a/linkis-public-enhancements/linkis-bml-server/src/main/java/org/apache/linkis/bml/restful/BmlProjectRestful.java b/linkis-public-enhancements/linkis-bml-server/src/main/java/org/apache/linkis/bml/restful/BmlProjectRestful.java index 900dfca21a..ed9ab45cf6 100644 --- a/linkis-public-enhancements/linkis-bml-server/src/main/java/org/apache/linkis/bml/restful/BmlProjectRestful.java +++ b/linkis-public-enhancements/linkis-bml-server/src/main/java/org/apache/linkis/bml/restful/BmlProjectRestful.java @@ -260,7 +260,7 @@ public Message updateShareResource( Map properties = new HashMap<>(); properties.put("clientIp", clientIp); ResourceTask resourceTask = null; - synchronized (resourceId.intern()) { + synchronized (resourceId.intern()) { // NOSONAR resourceTask = taskService.createUpdateTask(resourceId, DEFAULT_PROXY_USER, file, properties); } diff --git a/linkis-public-enhancements/linkis-bml-server/src/main/java/org/apache/linkis/bml/restful/BmlRestfulApi.java b/linkis-public-enhancements/linkis-bml-server/src/main/java/org/apache/linkis/bml/restful/BmlRestfulApi.java index 8125d706db..363833bf87 100644 --- a/linkis-public-enhancements/linkis-bml-server/src/main/java/org/apache/linkis/bml/restful/BmlRestfulApi.java +++ b/linkis-public-enhancements/linkis-bml-server/src/main/java/org/apache/linkis/bml/restful/BmlRestfulApi.java @@ -748,7 +748,7 @@ public Message updateVersion( Map properties = new HashMap<>(); properties.put("clientIp", clientIp); ResourceTask resourceTask = null; - synchronized (resourceId.intern()) { + synchronized (resourceId.intern()) { // NOSONAR resourceTask = taskService.createUpdateTask(resourceId, user, file, properties); } message = Message.ok("The update resource task was submitted successfully(提交更新资源任务成功)"); @@ -891,7 +891,7 @@ public Message copyResourceToAnotherUser( properties.put("maxVersion", 0); properties.put("system", "dss"); ResourceTask resourceTask = null; - synchronized (resourceId.intern()) { + synchronized (resourceId.intern()) { // NOSONAR resourceTask = taskService.createCopyResourceTask(resourceId, anotherUser, properties); } message = Message.ok(); @@ -930,7 +930,7 @@ public Message rollbackVersion(HttpServletRequest request, @RequestBody JsonNode Map properties = new HashMap<>(); properties.put("clientIp", clientIp); ResourceTask resourceTask = null; - synchronized (resourceId.intern()) { + synchronized (resourceId.intern()) { // NOSONAR resourceTask = taskService.createRollbackVersionTask( resourceId, rollbackVersion, username, properties); diff --git a/linkis-public-enhancements/linkis-bml-server/src/main/java/org/apache/linkis/bml/service/impl/VersionServiceImpl.java b/linkis-public-enhancements/linkis-bml-server/src/main/java/org/apache/linkis/bml/service/impl/VersionServiceImpl.java index b2d9479c5f..f6f4daa8ed 100644 --- a/linkis-public-enhancements/linkis-bml-server/src/main/java/org/apache/linkis/bml/service/impl/VersionServiceImpl.java +++ b/linkis-public-enhancements/linkis-bml-server/src/main/java/org/apache/linkis/bml/service/impl/VersionServiceImpl.java @@ -147,7 +147,7 @@ public boolean downloadResource( Fs fileSystem = FSFactory.getFsByProxyUser(new FsPath(path), user); fileSystem.init(new HashMap()); InputStream inputStream = fileSystem.read(new FsPath(path)); - inputStream.skip(startByte - 1); + inputStream.skip(startByte - 1); // NOSONAR logger.info( "{} downLoad source {} inputStream skipped {} bytes", user, resourceId, (startByte - 1)); byte[] buffer = new byte[1024]; diff --git a/linkis-public-enhancements/linkis-configuration/src/test/java/org/apache/linkis/configuration/restful/api/ConfigurationRestfulApiTest.java b/linkis-public-enhancements/linkis-configuration/src/test/java/org/apache/linkis/configuration/restful/api/ConfigurationRestfulApiTest.java index 41803098d0..027e0393a6 100644 --- a/linkis-public-enhancements/linkis-configuration/src/test/java/org/apache/linkis/configuration/restful/api/ConfigurationRestfulApiTest.java +++ b/linkis-public-enhancements/linkis-configuration/src/test/java/org/apache/linkis/configuration/restful/api/ConfigurationRestfulApiTest.java @@ -161,7 +161,7 @@ public void TestCheckAdmin() throws Exception { public void sendUrl(String url, MultiValueMap paramsMap, String type, String msg) throws Exception { MvcUtils mvcUtils = new MvcUtils(mockMvc); - Message mvcResult = null; + Message mvcResult = null; // NOSONAR if (type.equals("get")) { if (paramsMap != null) { mvcResult = mvcUtils.getMessage(mvcUtils.buildMvcResultGet(url, paramsMap)); @@ -176,7 +176,7 @@ public void sendUrl(String url, MultiValueMap paramsMap, String mvcResult = mvcUtils.getMessage(mvcUtils.buildMvcResultPost(url)); } } - assertEquals(MessageStatus.SUCCESS(), mvcResult.getStatus()); + assertEquals(MessageStatus.SUCCESS(), mvcResult.getStatus()); // NOSONAR logger.info(String.valueOf(mvcResult)); } } diff --git a/linkis-public-enhancements/linkis-cs-server/src/main/java/org/apache/linkis/cs/condition/BinaryLogicCondition.java b/linkis-public-enhancements/linkis-cs-server/src/main/java/org/apache/linkis/cs/condition/BinaryLogicCondition.java index 4e13d04e65..ce221e7361 100644 --- a/linkis-public-enhancements/linkis-cs-server/src/main/java/org/apache/linkis/cs/condition/BinaryLogicCondition.java +++ b/linkis-public-enhancements/linkis-cs-server/src/main/java/org/apache/linkis/cs/condition/BinaryLogicCondition.java @@ -46,6 +46,6 @@ public Condition getRight() { } public void setRight(Condition right) { - right = right; + this.right = right; } } diff --git a/linkis-public-enhancements/linkis-cs-server/src/main/java/org/apache/linkis/cs/condition/construction/ConditionParser.java b/linkis-public-enhancements/linkis-cs-server/src/main/java/org/apache/linkis/cs/condition/construction/ConditionParser.java index 901ef0d5b7..c002a563bd 100644 --- a/linkis-public-enhancements/linkis-cs-server/src/main/java/org/apache/linkis/cs/condition/construction/ConditionParser.java +++ b/linkis-public-enhancements/linkis-cs-server/src/main/java/org/apache/linkis/cs/condition/construction/ConditionParser.java @@ -22,30 +22,32 @@ import java.util.HashMap; import java.util.List; import java.util.Map; - -import com.google.common.collect.Lists; +import java.util.stream.Collectors; +import java.util.stream.Stream; public interface ConditionParser { - public static Map parserMap = - new HashMap() { - { - List conditionParsers = - Lists.newArrayList( - new RegexConditionParser(), - new ContainsConditionParser(), - new ContextTypeConditionParser(), - new ContextScopeConditionParser(), - new AndConditionParser(), - new OrConditionParser(), - new NotConditionParser(), - new NearestConditionParser(), - new ContextValueTypeConditionParser()); - for (ConditionParser conditionParser : conditionParsers) { - put(conditionParser.getName(), conditionParser); - } - } - }; + Map parserMap = initializeParserMap(); + + static Map initializeParserMap() { + List conditionParsers = + Stream.of( + new RegexConditionParser(), + new ContainsConditionParser(), + new ContextTypeConditionParser(), + new ContextScopeConditionParser(), + new AndConditionParser(), + new OrConditionParser(), + new NotConditionParser(), + new NearestConditionParser(), + new ContextValueTypeConditionParser()) + .collect(Collectors.toList()); + Map map = new HashMap<>(); + for (ConditionParser conditionParser : conditionParsers) { + map.put(conditionParser.getName(), conditionParser); + } + return map; + } Condition parse(Map conditionMap); diff --git a/linkis-public-enhancements/linkis-cs-server/src/main/java/org/apache/linkis/cs/contextcache/DefaultContextCacheService.java b/linkis-public-enhancements/linkis-cs-server/src/main/java/org/apache/linkis/cs/contextcache/DefaultContextCacheService.java index 1c7ed0cfd9..58dc805a5c 100644 --- a/linkis-public-enhancements/linkis-cs-server/src/main/java/org/apache/linkis/cs/contextcache/DefaultContextCacheService.java +++ b/linkis-public-enhancements/linkis-cs-server/src/main/java/org/apache/linkis/cs/contextcache/DefaultContextCacheService.java @@ -166,7 +166,7 @@ public List getAllByScope( } catch (Exception e) { logger.error( "Failed to getAllByScope contextID({}) of ContextScope({}) of csType({})", - contextID.getContextId(), + contextID == null ? "NULL" : contextID.getContextId(), scope, csType, e); diff --git a/linkis-public-enhancements/linkis-cs-server/src/main/java/org/apache/linkis/cs/contextcache/cache/cskey/impl/ContextValueMapSetImpl.java b/linkis-public-enhancements/linkis-cs-server/src/main/java/org/apache/linkis/cs/contextcache/cache/cskey/impl/ContextValueMapSetImpl.java index 7ddb75dcaf..3d30225bd8 100644 --- a/linkis-public-enhancements/linkis-cs-server/src/main/java/org/apache/linkis/cs/contextcache/cache/cskey/impl/ContextValueMapSetImpl.java +++ b/linkis-public-enhancements/linkis-cs-server/src/main/java/org/apache/linkis/cs/contextcache/cache/cskey/impl/ContextValueMapSetImpl.java @@ -45,7 +45,7 @@ public Map getContextValueMap(ContextType contextType) } String csType = contextType.name(); if (!contextValueMapSet.containsKey(csType)) { - synchronized (csType.intern()) { + synchronized (csType.intern()) { // NOSONAR if (!contextValueMapSet.containsKey(csType)) { logger.info("For ContextType({}) init ContextValueMap", csType); contextValueMapSet.put(csType, new HashMap(16)); diff --git a/linkis-public-enhancements/linkis-cs-server/src/main/java/org/apache/linkis/cs/contextcache/index/ContextInvertedIndexSetImpl.java b/linkis-public-enhancements/linkis-cs-server/src/main/java/org/apache/linkis/cs/contextcache/index/ContextInvertedIndexSetImpl.java index ebe30bad7d..f812d921ee 100644 --- a/linkis-public-enhancements/linkis-cs-server/src/main/java/org/apache/linkis/cs/contextcache/index/ContextInvertedIndexSetImpl.java +++ b/linkis-public-enhancements/linkis-cs-server/src/main/java/org/apache/linkis/cs/contextcache/index/ContextInvertedIndexSetImpl.java @@ -36,7 +36,7 @@ public class ContextInvertedIndexSetImpl implements ContextInvertedIndexSet { public ContextInvertedIndex getContextInvertedIndex(ContextType contextType) { String csType = contextType.name(); if (!invertedIndexMap.containsKey(csType)) { - synchronized (csType.intern()) { + synchronized (csType.intern()) { // NOSONAR if (!invertedIndexMap.containsKey(csType)) { logger.info("For ContextType({}) init invertedIndex", csType); invertedIndexMap.put(csType, new DefaultContextInvertedIndex()); diff --git a/linkis-public-enhancements/linkis-cs-server/src/main/java/org/apache/linkis/cs/highavailable/ha/impl/BackupInstanceGeneratorImpl.java b/linkis-public-enhancements/linkis-cs-server/src/main/java/org/apache/linkis/cs/highavailable/ha/impl/BackupInstanceGeneratorImpl.java index f7f93c655a..772a80525f 100644 --- a/linkis-public-enhancements/linkis-cs-server/src/main/java/org/apache/linkis/cs/highavailable/ha/impl/BackupInstanceGeneratorImpl.java +++ b/linkis-public-enhancements/linkis-cs-server/src/main/java/org/apache/linkis/cs/highavailable/ha/impl/BackupInstanceGeneratorImpl.java @@ -44,6 +44,8 @@ public class BackupInstanceGeneratorImpl implements BackupInstanceGenerator { @Autowired private ContextHAChecker contextHAChecker; + private Random random = new Random(); + @Override public String getBackupInstance(String haIDKey) throws CSErrorException { @@ -88,6 +90,6 @@ public String chooseBackupInstance(String mainInstanceAlias) throws CSErrorExcep private int getBackupInstanceIndex(List instanceList) { // todo refactor according to load-balance - return new Random().nextInt(instanceList.size()); + return random.nextInt(instanceList.size()); } } diff --git a/linkis-public-enhancements/linkis-cs-server/src/main/java/org/apache/linkis/cs/highavailable/ha/impl/ContextHACheckerImpl.java b/linkis-public-enhancements/linkis-cs-server/src/main/java/org/apache/linkis/cs/highavailable/ha/impl/ContextHACheckerImpl.java index fbaa878d3f..8f1c6b7438 100644 --- a/linkis-public-enhancements/linkis-cs-server/src/main/java/org/apache/linkis/cs/highavailable/ha/impl/ContextHACheckerImpl.java +++ b/linkis-public-enhancements/linkis-cs-server/src/main/java/org/apache/linkis/cs/highavailable/ha/impl/ContextHACheckerImpl.java @@ -99,12 +99,13 @@ public String convertHAIDToHAKey(HAContextID haContextID) throws CSErrorExceptio || StringUtils.isBlank(haContextID.getContextId())) { throw new CSErrorException( CSErrorCode.INVALID_HAID, - "Incomplete HAID Object cannot be encoded. mainInstance : " - + haContextID.getInstance() - + ", backupInstance : " - + haContextID.getBackupInstance() - + ", contextID : " - + haContextID.getContextId()); + "Incomplete HAID Object cannot be encoded. mainInstance : " + haContextID == null + ? "NULL" + : haContextID.getInstance() + ", backupInstance : " + haContextID == null + ? "NULL" + : haContextID.getBackupInstance() + ", contextID : " + haContextID == null + ? "NULL" + : haContextID.getContextId()); } if (StringUtils.isNumeric(haContextID.getContextId())) { return encode(haContextID); diff --git a/linkis-public-enhancements/linkis-cs-server/src/main/java/org/apache/linkis/cs/server/protocol/RestResponseProtocol.java b/linkis-public-enhancements/linkis-cs-server/src/main/java/org/apache/linkis/cs/server/protocol/RestResponseProtocol.java index 304db9a55f..eb7f697fad 100644 --- a/linkis-public-enhancements/linkis-cs-server/src/main/java/org/apache/linkis/cs/server/protocol/RestResponseProtocol.java +++ b/linkis-public-enhancements/linkis-cs-server/src/main/java/org/apache/linkis/cs/server/protocol/RestResponseProtocol.java @@ -55,7 +55,7 @@ public void waitTimeEnd(long mills) throws InterruptedException { public void notifyJob() { logger.info("notify the job"); synchronized (lock) { - lock.notify(); + lock.notify(); // NOSONAR } } diff --git a/linkis-public-enhancements/linkis-cs-server/src/test/java/org/apache/linkis/cs/parser/ApiJsonTest.java b/linkis-public-enhancements/linkis-cs-server/src/test/java/org/apache/linkis/cs/parser/ApiJsonTest.java index 29d5a7ceda..7247ce97b3 100644 --- a/linkis-public-enhancements/linkis-cs-server/src/test/java/org/apache/linkis/cs/parser/ApiJsonTest.java +++ b/linkis-public-enhancements/linkis-cs-server/src/test/java/org/apache/linkis/cs/parser/ApiJsonTest.java @@ -74,7 +74,7 @@ public void temp() { String test = "{\"cols\":[{\"name\":\"birthday\",\"visualType\":\"string\",\"type\":\"category\",\"config\":true,\"field\":{\"alias\":\"\",\"desc\":\"\",\"useExpression\":false},\"format\":{\"formatType\":\"default\"},\"from\":\"cols\"},{\"name\":\"name\",\"visualType\":\"string\",\"type\":\"category\",\"config\":true,\"field\":{\"alias\":\"\",\"desc\":\"\",\"useExpression\":false},\"format\":{\"formatType\":\"default\"},\"from\":\"cols\"}],\"rows\":[],\"metrics\":[{\"name\":\"score@Visualis@6F01974E\",\"visualType\":\"number\",\"type\":\"value\",\"agg\":\"sum\",\"config\":true,\"chart\":{\"id\":1,\"name\":\"table\",\"title\":\"表格\",\"icon\":\"icon-table\",\"coordinate\":\"other\",\"rules\":[{\"dimension\":[0,9999],\"metric\":[0,9999]}],\"data\":{\"cols\":{\"title\":\"列\",\"type\":\"category\"},\"rows\":{\"title\":\"行\",\"type\":\"category\"},\"metrics\":{\"title\":\"指标\",\"type\":\"value\"},\"filters\":{\"title\":\"筛选\",\"type\":\"all\"}},\"style\":{\"table\":{\"fontFamily\":\"PingFang SC\",\"fontSize\":\"12\",\"color\":\"#666\",\"lineStyle\":\"solid\",\"lineColor\":\"#D9D9D9\",\"headerBackgroundColor\":\"#f7f7f7\",\"headerConfig\":[],\"columnsConfig\":[],\"leftFixedColumns\":[],\"rightFixedColumns\":[],\"headerFixed\":true,\"autoMergeCell\":false,\"bordered\":true,\"size\":\"default\",\"withPaging\":true,\"pageSize\":\"20\",\"withNoAggregators\":false},\"spec\":{}}},\"field\":{\"alias\":\"\",\"desc\":\"\",\"useExpression\":false},\"format\":{\"formatType\":\"default\"},\"from\":\"metrics\"}],\"filters\":[],\"color\":{\"title\":\"颜色\",\"type\":\"category\",\"value\":{\"all\":\"#509af2\"},\"items\":[]},\"chartStyles\":{\"richText\":{\"content\":\"

〖@dv_name_dv@〗

〖@dv_birthday_dv@〗

\"},\"spec\":{}},\"selectedChart\":15,\"data\":[],\"pagination\":{\"pageNo\":0,\"pageSize\":0,\"withPaging\":false,\"totalCount\":0},\"dimetionAxis\":\"col\",\"renderType\":\"rerender\",\"orders\":[],\"mode\":\"chart\",\"model\":{\"birthday\":{\"sqlType\":\"STRING\",\"visualType\":\"string\",\"modelType\":\"category\"},\"score\":{\"sqlType\":\"DOUBLE\",\"visualType\":\"number\",\"modelType\":\"value\"},\"teacher\":{\"sqlType\":\"STRING\",\"visualType\":\"string\",\"modelType\":\"category\"},\"city\":{\"sqlType\":\"STRING\",\"visualType\":\"string\",\"modelType\":\"category\"},\"sex\":{\"sqlType\":\"STRING\",\"visualType\":\"string\",\"modelType\":\"category\"},\"fee\":{\"sqlType\":\"DOUBLE\",\"visualType\":\"number\",\"modelType\":\"value\"},\"name\":{\"sqlType\":\"STRING\",\"visualType\":\"string\",\"modelType\":\"category\"},\"lesson\":{\"sqlType\":\"STRING\",\"visualType\":\"string\",\"modelType\":\"category\"},\"id\":{\"sqlType\":\"INT\",\"visualType\":\"number\",\"modelType\":\"value\"},\"class\":{\"sqlType\":\"STRING\",\"visualType\":\"string\",\"modelType\":\"category\"},\"exam_date\":{\"sqlType\":\"STRING\",\"visualType\":\"string\",\"modelType\":\"category\"},\"age\":{\"sqlType\":\"INT\",\"visualType\":\"number\",\"modelType\":\"value\"}},\"controls\":[],\"computed\":[],\"cache\":false,\"expired\":300,\"autoLoadData\":true,\"query\":{\"groups\":[\"birthday\",\"name\"],\"aggregators\":[{\"column\":\"score\",\"func\":\"sum\"}],\"filters\":[],\"orders\":[],\"pageNo\":0,\"pageSize\":0,\"nativeQuery\":false,\"cache\":false,\"expired\":0,\"flush\":false}}"; Set columns = getWidgetUsedColumns(test); - columns.size(); + columns.size(); // NOSONAR } private Set getWidgetUsedColumns(String config) { diff --git a/linkis-public-enhancements/linkis-cs-server/src/test/java/org/apache/linkis/cs/persistence/ContextHistoryTest.java b/linkis-public-enhancements/linkis-cs-server/src/test/java/org/apache/linkis/cs/persistence/ContextHistoryTest.java index 0c771ca9c1..3f1049f24a 100644 --- a/linkis-public-enhancements/linkis-cs-server/src/test/java/org/apache/linkis/cs/persistence/ContextHistoryTest.java +++ b/linkis-public-enhancements/linkis-cs-server/src/test/java/org/apache/linkis/cs/persistence/ContextHistoryTest.java @@ -46,10 +46,10 @@ public void before() { public void testcreateContextHistory() throws CSErrorException, JsonProcessingException { AContextHistory aContextHistory = new AContextHistory(); PersistenceContextID persistenceContextID = new PersistenceContextID(); - persistenceContextID.setContextId(String.valueOf(new Random().nextInt(100000))); + persistenceContextID.setContextId(String.valueOf(new Random().nextInt(100000))); // NOSONAR aContextHistory.setHistoryJson("json"); aContextHistory.setContextType(ContextType.DATA); - aContextHistory.setId(new Random().nextInt(100000)); + aContextHistory.setId(new Random().nextInt(100000)); // NOSONAR aContextHistory.setKeyword("keywords"); aContextHistory.setSource("source"); contextHistoryPersistence.createHistory(persistenceContextID, aContextHistory); diff --git a/linkis-public-enhancements/linkis-cs-server/src/test/java/org/apache/linkis/cs/persistence/ContextMapTest.java b/linkis-public-enhancements/linkis-cs-server/src/test/java/org/apache/linkis/cs/persistence/ContextMapTest.java index 7664c8889d..f715215f49 100644 --- a/linkis-public-enhancements/linkis-cs-server/src/test/java/org/apache/linkis/cs/persistence/ContextMapTest.java +++ b/linkis-public-enhancements/linkis-cs-server/src/test/java/org/apache/linkis/cs/persistence/ContextMapTest.java @@ -45,7 +45,7 @@ public void before() { public void testcreateContextMap() throws CSErrorException, JsonProcessingException { AContextKeyValue aContextKeyValue = new AContextKeyValue(); PersistenceContextID persistenceContextID = new PersistenceContextID(); - persistenceContextID.setContextId(String.valueOf(new Random().nextInt(100000))); + persistenceContextID.setContextId(String.valueOf(new Random().nextInt(100000))); // NOSONAR AContextValue aContextValue = new AContextValue(); CSTable csTable = new CSTable(); csTable.setCreator("hadoop"); diff --git a/linkis-public-enhancements/linkis-cs-server/src/test/java/org/apache/linkis/cs/server/conf/ContextServerConfTest.java b/linkis-public-enhancements/linkis-cs-server/src/test/java/org/apache/linkis/cs/server/conf/ContextServerConfTest.java index 4c5fcb97a8..18c070cfe1 100644 --- a/linkis-public-enhancements/linkis-cs-server/src/test/java/org/apache/linkis/cs/server/conf/ContextServerConfTest.java +++ b/linkis-public-enhancements/linkis-cs-server/src/test/java/org/apache/linkis/cs/server/conf/ContextServerConfTest.java @@ -37,6 +37,6 @@ public void constTest() { Assertions.assertTrue(100 == csSchedulerMaxRunningJobs); Assertions.assertTrue(1000 == csSchedulerMaxAskExecutorTimes); Assertions.assertTrue(10000 == csSchedulerJobWaitMills); - Assertions.assertTrue("cs_1_dev" == confLabel); + Assertions.assertTrue("cs_1_dev".equals(confLabel)); } } diff --git a/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/server/src/test/java/org/apache/linkis/datasourcemanager/core/service/DataSourceInfoServiceTest.java b/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/server/src/test/java/org/apache/linkis/datasourcemanager/core/service/DataSourceInfoServiceTest.java index c9bd12f799..0bddd61d42 100644 --- a/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/server/src/test/java/org/apache/linkis/datasourcemanager/core/service/DataSourceInfoServiceTest.java +++ b/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/server/src/test/java/org/apache/linkis/datasourcemanager/core/service/DataSourceInfoServiceTest.java @@ -360,7 +360,7 @@ void testInsertDataSourceParameter() throws ErrorException { Long res = dataSourceInfoService.insertDataSourceParameter( keyDefinitionList, datasourceId, connectParams, username, comment); - assertTrue(expectedVersion == res); + assertTrue(expectedVersion.equals(res)); } @Test diff --git a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/server/src/main/java/org/apache/linkis/metadata/query/server/utils/MetadataUtils.java b/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/server/src/main/java/org/apache/linkis/metadata/query/server/utils/MetadataUtils.java index 0107347e41..3b9337f4d3 100644 --- a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/server/src/main/java/org/apache/linkis/metadata/query/server/utils/MetadataUtils.java +++ b/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/server/src/main/java/org/apache/linkis/metadata/query/server/utils/MetadataUtils.java @@ -160,8 +160,7 @@ private static List searchMetaServiceClassFormURI( classNameList.add(className); } } else if (url.endsWith(JAR_SUF_NAME)) { - try { - JarFile jarFile = new JarFile(new File(url)); + try (JarFile jarFile = new JarFile(new File(url))) { Enumeration en = jarFile.entries(); while (en.hasMoreElements()) { String name = en.nextElement().getName(); diff --git a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/mongodb/src/main/java/org/apache/linkis/metadata/query/service/MongoDbConnection.java b/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/mongodb/src/main/java/org/apache/linkis/metadata/query/service/MongoDbConnection.java index 7a5292852e..aafd0e53c5 100644 --- a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/mongodb/src/main/java/org/apache/linkis/metadata/query/service/MongoDbConnection.java +++ b/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/mongodb/src/main/java/org/apache/linkis/metadata/query/service/MongoDbConnection.java @@ -249,8 +249,9 @@ public ConnectMessage( this.database = database; if (extraParams != null) { this.extraParams = extraParams; + } else { + this.extraParams = new HashMap<>(); } - this.extraParams = extraParams; this.extraParams.put("connectTimeout", CONNECT_TIMEOUT.getValue()); this.extraParams.put("socketTimeout", SOCKET_TIMEOUT.getValue()); } diff --git a/linkis-public-enhancements/linkis-datasource/linkis-metadata/src/main/java/org/apache/linkis/metadata/service/impl/DataSourceServiceImpl.java b/linkis-public-enhancements/linkis-datasource/linkis-metadata/src/main/java/org/apache/linkis/metadata/service/impl/DataSourceServiceImpl.java index ddcc97277c..2d4a63e327 100644 --- a/linkis-public-enhancements/linkis-datasource/linkis-metadata/src/main/java/org/apache/linkis/metadata/service/impl/DataSourceServiceImpl.java +++ b/linkis-public-enhancements/linkis-datasource/linkis-metadata/src/main/java/org/apache/linkis/metadata/service/impl/DataSourceServiceImpl.java @@ -308,7 +308,7 @@ private void resetRootHdfs() { } private FileSystem getRootHdfs() { - if (rootHdfs == null) { + if (rootHdfs == null) { // NOSONAR synchronized (this) { if (rootHdfs == null) { rootHdfs = HDFSUtils.getHDFSRootUserFileSystem(); diff --git a/linkis-public-enhancements/linkis-datasource/linkis-metadata/src/test/java/org/apache/linkis/metadata/hive/dao/HiveMetaDaoTest.java b/linkis-public-enhancements/linkis-datasource/linkis-metadata/src/test/java/org/apache/linkis/metadata/hive/dao/HiveMetaDaoTest.java index a86cd96010..d15f7829c7 100644 --- a/linkis-public-enhancements/linkis-datasource/linkis-metadata/src/test/java/org/apache/linkis/metadata/hive/dao/HiveMetaDaoTest.java +++ b/linkis-public-enhancements/linkis-datasource/linkis-metadata/src/test/java/org/apache/linkis/metadata/hive/dao/HiveMetaDaoTest.java @@ -113,7 +113,7 @@ public void getPartitionsTest() { queryParam.setTableName("employee"); List partitions = hiveMetaDao.getPartitions(queryParam); - Assertions.assertTrue(partitions.size() >= 0); + Assertions.assertTrue(partitions.size() >= 0); // NOSONAR } @Test @@ -125,7 +125,7 @@ public void getColumnsTest() { queryParam.setTableName("employee"); List> columns = hiveMetaDao.getColumns(queryParam); - Assertions.assertTrue(columns.size() >= 0); + Assertions.assertTrue(columns.size() >= 0); // NOSONAR } @Test @@ -146,7 +146,7 @@ public void getColumnsByStorageDescriptionIDTest() { MetadataQueryParam queryParam = new MetadataQueryParam(); queryParam.setSdId("1"); List> columns = hiveMetaDao.getColumnsByStorageDescriptionID(queryParam); - Assertions.assertTrue(columns.size() >= 0); + Assertions.assertTrue(columns.size() >= 0); // NOSONAR } @Test diff --git a/linkis-public-enhancements/linkis-instance-label-server/src/main/java/org/apache/linkis/instance/label/async/GenericAsyncConsumerQueue.java b/linkis-public-enhancements/linkis-instance-label-server/src/main/java/org/apache/linkis/instance/label/async/GenericAsyncConsumerQueue.java index dbb32e3d0d..deecb6324a 100644 --- a/linkis-public-enhancements/linkis-instance-label-server/src/main/java/org/apache/linkis/instance/label/async/GenericAsyncConsumerQueue.java +++ b/linkis-public-enhancements/linkis-instance-label-server/src/main/java/org/apache/linkis/instance/label/async/GenericAsyncConsumerQueue.java @@ -92,7 +92,9 @@ public synchronized void consumer( LOG.trace("wait until the next time: " + triggerTime); consumeLock.lock(); try { - consumeCondition.await(triggerTime - nowMillsTime, TimeUnit.MILLISECONDS); + if (!consumeCondition.await(triggerTime - nowMillsTime, TimeUnit.MILLISECONDS)) { + LOG.error("Interrupt in awaiting action"); + } } catch (InterruptedException e) { LOG.error("Interrupt in awaiting action, message: [" + e.getMessage() + "]", e); continue; diff --git a/linkis-public-enhancements/linkis-instance-label-server/src/main/java/org/apache/linkis/instance/label/entity/InsPersistenceLabel.java b/linkis-public-enhancements/linkis-instance-label-server/src/main/java/org/apache/linkis/instance/label/entity/InsPersistenceLabel.java index fedc036bba..869b5744bf 100644 --- a/linkis-public-enhancements/linkis-instance-label-server/src/main/java/org/apache/linkis/instance/label/entity/InsPersistenceLabel.java +++ b/linkis-public-enhancements/linkis-instance-label-server/src/main/java/org/apache/linkis/instance/label/entity/InsPersistenceLabel.java @@ -20,6 +20,7 @@ import org.apache.linkis.manager.label.entity.GenericLabel; import java.util.Date; +import java.util.Objects; /** like: PersistenceLabel in label-manager-common */ public class InsPersistenceLabel extends GenericLabel { @@ -93,4 +94,9 @@ public boolean equals(Object other) { } return false; } + + @Override + public int hashCode() { + return Objects.hash(this.getLabelKey(), this.getStringValue()); + } } diff --git a/linkis-public-enhancements/linkis-pes-client/src/main/java/org/apache/linkis/cs/client/http/HttpContextClient.java b/linkis-public-enhancements/linkis-pes-client/src/main/java/org/apache/linkis/cs/client/http/HttpContextClient.java index fffae20f38..335f3fcf6e 100644 --- a/linkis-public-enhancements/linkis-pes-client/src/main/java/org/apache/linkis/cs/client/http/HttpContextClient.java +++ b/linkis-public-enhancements/linkis-pes-client/src/main/java/org/apache/linkis/cs/client/http/HttpContextClient.java @@ -91,7 +91,7 @@ private HttpContextClient(ContextClientConfig contextClientConfig) { } public static HttpContextClient getInstance(ContextClientConfig contextClientConfig) { - if (httpContextClient == null) { + if (httpContextClient == null) { // NOSONAR synchronized (HttpContextClient.class) { if (httpContextClient == null) { httpContextClient = new HttpContextClient(contextClientConfig); @@ -823,11 +823,7 @@ public int batchClearContextByHAID(List idList) throws ErrorException { } if (result instanceof ContextClearByIDResult) { ContextClearByIDResult contextClearByIDResult = (ContextClearByIDResult) result; - if (null != contextClearByIDResult) { - return contextClearByIDResult.num(); - } else { - return 0; - } + return contextClearByIDResult.num(); } else if (null == result) { throw new CSErrorException(80017, "Invalid null result "); } else { @@ -873,11 +869,7 @@ public int batchClearContextByTime( } if (result instanceof ContextClearByTimeResult) { ContextClearByTimeResult contextClearByTimeResult = (ContextClearByTimeResult) result; - if (null != contextClearByTimeResult) { - return contextClearByTimeResult.num(); - } else { - return 0; - } + return contextClearByTimeResult.num(); } else if (null == result) { throw new CSErrorException(80017, "Invalid null result "); } else { diff --git a/linkis-public-enhancements/linkis-pes-client/src/main/java/org/apache/linkis/cs/client/listener/ContextClientListenerManager.java b/linkis-public-enhancements/linkis-pes-client/src/main/java/org/apache/linkis/cs/client/listener/ContextClientListenerManager.java index 3694b38326..0f3120c671 100644 --- a/linkis-public-enhancements/linkis-pes-client/src/main/java/org/apache/linkis/cs/client/listener/ContextClientListenerManager.java +++ b/linkis-public-enhancements/linkis-pes-client/src/main/java/org/apache/linkis/cs/client/listener/ContextClientListenerManager.java @@ -26,7 +26,7 @@ public class ContextClientListenerManager { public static ContextClientListenerBus getContextClientListenerBus() { - if (contextClientListenerBus == null) { + if (contextClientListenerBus == null) { // NOSONAR synchronized (ContextClientListenerManager.class) { if (contextClientListenerBus == null) { contextClientListenerBus = new ContextClientListenerBus(); diff --git a/linkis-public-enhancements/linkis-pes-client/src/main/java/org/apache/linkis/cs/client/service/CSMetaDataService.java b/linkis-public-enhancements/linkis-pes-client/src/main/java/org/apache/linkis/cs/client/service/CSMetaDataService.java index 19e55909ae..75733ec0c7 100644 --- a/linkis-public-enhancements/linkis-pes-client/src/main/java/org/apache/linkis/cs/client/service/CSMetaDataService.java +++ b/linkis-public-enhancements/linkis-pes-client/src/main/java/org/apache/linkis/cs/client/service/CSMetaDataService.java @@ -41,7 +41,7 @@ public class CSMetaDataService implements MetaDataService { private CSMetaDataService() {} public static CSMetaDataService getInstance() { - if (null == csMetaDataService) { + if (null == csMetaDataService) { // NOSONAR synchronized (CSMetaDataService.class) { if (null == csMetaDataService) { csMetaDataService = new CSMetaDataService(); diff --git a/linkis-public-enhancements/linkis-pes-client/src/main/java/org/apache/linkis/cs/client/service/CSNodeServiceImpl.java b/linkis-public-enhancements/linkis-pes-client/src/main/java/org/apache/linkis/cs/client/service/CSNodeServiceImpl.java index 3f79011e0e..9936aad3e9 100644 --- a/linkis-public-enhancements/linkis-pes-client/src/main/java/org/apache/linkis/cs/client/service/CSNodeServiceImpl.java +++ b/linkis-public-enhancements/linkis-pes-client/src/main/java/org/apache/linkis/cs/client/service/CSNodeServiceImpl.java @@ -38,7 +38,7 @@ public class CSNodeServiceImpl implements CSNodeService { private CSNodeServiceImpl() {} public static CSNodeService getInstance() { - if (null == csNodeService) { + if (null == csNodeService) { // NOSONAR synchronized (CSNodeServiceImpl.class) { if (null == csNodeService) { csNodeService = new CSNodeServiceImpl(); diff --git a/linkis-public-enhancements/linkis-pes-client/src/main/java/org/apache/linkis/cs/client/service/CSResourceService.java b/linkis-public-enhancements/linkis-pes-client/src/main/java/org/apache/linkis/cs/client/service/CSResourceService.java index d08b11548e..14920e57f5 100644 --- a/linkis-public-enhancements/linkis-pes-client/src/main/java/org/apache/linkis/cs/client/service/CSResourceService.java +++ b/linkis-public-enhancements/linkis-pes-client/src/main/java/org/apache/linkis/cs/client/service/CSResourceService.java @@ -43,7 +43,7 @@ public class CSResourceService implements ResourceService { private CSResourceService() {} public static CSResourceService getInstance() { - if (null == csResourceService) { + if (null == csResourceService) { // NOSONAR synchronized (CSResourceService.class) { if (null == csResourceService) { csResourceService = new CSResourceService(); diff --git a/linkis-public-enhancements/linkis-pes-client/src/main/java/org/apache/linkis/cs/client/service/CSResultDataServiceImpl.java b/linkis-public-enhancements/linkis-pes-client/src/main/java/org/apache/linkis/cs/client/service/CSResultDataServiceImpl.java index bc682863c2..738641a0e6 100644 --- a/linkis-public-enhancements/linkis-pes-client/src/main/java/org/apache/linkis/cs/client/service/CSResultDataServiceImpl.java +++ b/linkis-public-enhancements/linkis-pes-client/src/main/java/org/apache/linkis/cs/client/service/CSResultDataServiceImpl.java @@ -48,7 +48,7 @@ public class CSResultDataServiceImpl implements CSResultDataService { private CSResultDataServiceImpl() {} public static CSResultDataService getInstance() { - if (null == csResultDataService) { + if (null == csResultDataService) { // NOSONAR synchronized (CSResultDataServiceImpl.class) { if (null == csResultDataService) { csResultDataService = new CSResultDataServiceImpl(); diff --git a/linkis-public-enhancements/linkis-pes-client/src/main/java/org/apache/linkis/cs/client/service/CSTableService.java b/linkis-public-enhancements/linkis-pes-client/src/main/java/org/apache/linkis/cs/client/service/CSTableService.java index 0c89f939de..2d342eec9f 100644 --- a/linkis-public-enhancements/linkis-pes-client/src/main/java/org/apache/linkis/cs/client/service/CSTableService.java +++ b/linkis-public-enhancements/linkis-pes-client/src/main/java/org/apache/linkis/cs/client/service/CSTableService.java @@ -54,7 +54,7 @@ public class CSTableService implements TableService { private CSTableService() {} public static CSTableService getInstance() { - if (null == csTableService) { + if (null == csTableService) { // NOSONAR synchronized (CSTableService.class) { if (null == csTableService) { csTableService = new CSTableService(); @@ -104,7 +104,7 @@ public List getUpstreamTables(String contextIDStr, String nodeName) if (null != rsList) logger.info( "contextID: {} and nodeName: {} succeed to get tables size {}", - contextID.getContextId(), + contextID == null ? "NULL" : contextID.getContextId(), nodeName, rsList.size()); return rsList; diff --git a/linkis-public-enhancements/linkis-pes-client/src/main/java/org/apache/linkis/cs/client/service/CSVariableService.java b/linkis-public-enhancements/linkis-pes-client/src/main/java/org/apache/linkis/cs/client/service/CSVariableService.java index bd45a990c5..afd424c4c6 100644 --- a/linkis-public-enhancements/linkis-pes-client/src/main/java/org/apache/linkis/cs/client/service/CSVariableService.java +++ b/linkis-public-enhancements/linkis-pes-client/src/main/java/org/apache/linkis/cs/client/service/CSVariableService.java @@ -99,7 +99,7 @@ public void putVariable(String contextIDStr, String contextKeyStr, LinkisVariabl } public static CSVariableService getInstance() { - if (null == csVariableService) { + if (null == csVariableService) { // NOSONAR synchronized (CSVariableService.class) { if (null == csVariableService) { csVariableService = new CSVariableService(); diff --git a/linkis-public-enhancements/linkis-pes-client/src/main/java/org/apache/linkis/cs/client/service/CSWorkServiceImpl.java b/linkis-public-enhancements/linkis-pes-client/src/main/java/org/apache/linkis/cs/client/service/CSWorkServiceImpl.java index 5059e68faa..37ce4eb9f0 100644 --- a/linkis-public-enhancements/linkis-pes-client/src/main/java/org/apache/linkis/cs/client/service/CSWorkServiceImpl.java +++ b/linkis-public-enhancements/linkis-pes-client/src/main/java/org/apache/linkis/cs/client/service/CSWorkServiceImpl.java @@ -41,7 +41,7 @@ private CSWorkServiceImpl() {} private static CSWorkService csWorkService = null; public static CSWorkService getInstance() { - if (null == csWorkService) { + if (null == csWorkService) { // NOSONAR synchronized (CSWorkServiceImpl.class) { if (null == csWorkService) { csWorkService = new CSWorkServiceImpl(); diff --git a/linkis-public-enhancements/linkis-pes-client/src/main/java/org/apache/linkis/cs/client/service/ContextHistoryClientServiceImpl.java b/linkis-public-enhancements/linkis-pes-client/src/main/java/org/apache/linkis/cs/client/service/ContextHistoryClientServiceImpl.java index c0ff85f96a..1776e2d80c 100644 --- a/linkis-public-enhancements/linkis-pes-client/src/main/java/org/apache/linkis/cs/client/service/ContextHistoryClientServiceImpl.java +++ b/linkis-public-enhancements/linkis-pes-client/src/main/java/org/apache/linkis/cs/client/service/ContextHistoryClientServiceImpl.java @@ -44,7 +44,7 @@ public class ContextHistoryClientServiceImpl implements ContextHistoryClientServ private ContextHistoryClientServiceImpl() {} public static ContextHistoryClientService getInstance() { - if (null == contextHistoryClientService) { + if (null == contextHistoryClientService) { // NOSONAR synchronized (ContextHistoryClientServiceImpl.class) { if (null == contextHistoryClientService) { contextHistoryClientService = new ContextHistoryClientServiceImpl(); diff --git a/linkis-public-enhancements/linkis-pes-client/src/main/java/org/apache/linkis/cs/client/service/DefaultSearchService.java b/linkis-public-enhancements/linkis-pes-client/src/main/java/org/apache/linkis/cs/client/service/DefaultSearchService.java index 6b60a50dba..1f0149bc22 100644 --- a/linkis-public-enhancements/linkis-pes-client/src/main/java/org/apache/linkis/cs/client/service/DefaultSearchService.java +++ b/linkis-public-enhancements/linkis-pes-client/src/main/java/org/apache/linkis/cs/client/service/DefaultSearchService.java @@ -186,7 +186,7 @@ public List searchUpstreamKeyValue( } public static SearchService getInstance() { - if (null == searchService) { + if (null == searchService) { // SONAR synchronized (DefaultSearchService.class) { if (null == searchService) { searchService = new DefaultSearchService(); diff --git a/linkis-public-enhancements/linkis-pes-client/src/main/java/org/apache/linkis/cs/client/service/LinkisJobDataServiceImpl.java b/linkis-public-enhancements/linkis-pes-client/src/main/java/org/apache/linkis/cs/client/service/LinkisJobDataServiceImpl.java index 9be63b07d6..fe8298b4d0 100644 --- a/linkis-public-enhancements/linkis-pes-client/src/main/java/org/apache/linkis/cs/client/service/LinkisJobDataServiceImpl.java +++ b/linkis-public-enhancements/linkis-pes-client/src/main/java/org/apache/linkis/cs/client/service/LinkisJobDataServiceImpl.java @@ -45,7 +45,7 @@ public class LinkisJobDataServiceImpl implements LinkisJobDataService { private LinkisJobDataServiceImpl() {} public static LinkisJobDataService getInstance() { - if (null == linkisJobDataService) { + if (null == linkisJobDataService) { // NOSONAR synchronized (LinkisJobDataServiceImpl.class) { if (null == linkisJobDataService) { linkisJobDataService = new LinkisJobDataServiceImpl(); diff --git a/linkis-public-enhancements/linkis-pes-client/src/main/java/org/apache/linkis/errorcode/client/handler/LinkisErrorCodeHandler.java b/linkis-public-enhancements/linkis-pes-client/src/main/java/org/apache/linkis/errorcode/client/handler/LinkisErrorCodeHandler.java index aa444ac55d..3fe661d735 100644 --- a/linkis-public-enhancements/linkis-pes-client/src/main/java/org/apache/linkis/errorcode/client/handler/LinkisErrorCodeHandler.java +++ b/linkis-public-enhancements/linkis-pes-client/src/main/java/org/apache/linkis/errorcode/client/handler/LinkisErrorCodeHandler.java @@ -63,7 +63,7 @@ public class LinkisErrorCodeHandler new ThreadPoolExecutor.AbortPolicy()); public static LinkisErrorCodeHandler getInstance() { - if (null == linkisErrorCodeHandler) { + if (null == linkisErrorCodeHandler) { // NOSONAR synchronized (LinkisErrorCodeHandler.class) { if (null == linkisErrorCodeHandler) { linkisErrorCodeHandler = new LinkisErrorCodeHandler(); @@ -121,8 +121,9 @@ public void handle(String logFilePath, int type) { LOGGER.info("begin to handle logFilePath {}", logFilePath); // At the end of the file, write "error code information is being generated for you". - try { - writeToFile(logFilePath, ERROR_CODE_PRE); + try (BufferedWriter bufferedWriter = + new BufferedWriter(new OutputStreamWriter(new FileOutputStream(logFilePath, true)))) { + writeToFile(bufferedWriter, ERROR_CODE_PRE); } catch (Exception e) { // If there is a write exception, skip this question directly. LOGGER.error("Failed to append error code to log file {}", logFilePath, e); @@ -146,13 +147,14 @@ public void handle(String logFilePath, int type) { LOGGER.error("failed to handle log file {} ", logFilePath, e); return; } - try { + try (BufferedWriter bufferedWriter = + new BufferedWriter(new OutputStreamWriter(new FileOutputStream(logFilePath, true)))) { if (errorCodeSet.size() == 0) { - writeToFile(logFilePath, ERROR_CODE_FAILED); + writeToFile(bufferedWriter, ERROR_CODE_FAILED); } else { - writeToFile(logFilePath, ERROR_CODE_OK); + writeToFile(bufferedWriter, ERROR_CODE_OK); List retErrorCodes = new ArrayList<>(errorCodeSet); - writeToFile(logFilePath, retErrorCodes.toString()); + writeToFile(bufferedWriter, retErrorCodes.toString()); } } catch (Exception e) { LOGGER.error("failed to write to errorcodes to {} ", logFilePath, e); @@ -162,12 +164,9 @@ public void handle(String logFilePath, int type) { LOGGER.info("put handle into threadPool"); } - private void writeToFile(String filePath, String content) throws Exception { - BufferedWriter bufferedWriter = - new BufferedWriter(new OutputStreamWriter(new FileOutputStream(filePath, true))); + private void writeToFile(BufferedWriter bufferedWriter, String content) throws Exception { bufferedWriter.write(content); bufferedWriter.write(NEW_LINE); - bufferedWriter.close(); } /** diff --git a/linkis-public-enhancements/linkis-pes-client/src/main/java/org/apache/linkis/errorcode/client/manager/LinkisErrorCodeManager.java b/linkis-public-enhancements/linkis-pes-client/src/main/java/org/apache/linkis/errorcode/client/manager/LinkisErrorCodeManager.java index a401beda72..6d05676a4b 100644 --- a/linkis-public-enhancements/linkis-pes-client/src/main/java/org/apache/linkis/errorcode/client/manager/LinkisErrorCodeManager.java +++ b/linkis-public-enhancements/linkis-pes-client/src/main/java/org/apache/linkis/errorcode/client/manager/LinkisErrorCodeManager.java @@ -32,7 +32,7 @@ public class LinkisErrorCodeManager { private LinkisErrorCodeManager() {} public static LinkisErrorCodeManager getInstance() { - if (linkisErrorCodeManager == null) { + if (linkisErrorCodeManager == null) { // NOSONAR synchronized (LinkisErrorCodeManager.class) { if (linkisErrorCodeManager == null) { linkisErrorCodeManager = new LinkisErrorCodeManager(); diff --git a/linkis-public-enhancements/linkis-pes-client/src/main/java/org/apache/linkis/errorcode/client/synchronizer/LinkisErrorCodeSynchronizer.java b/linkis-public-enhancements/linkis-pes-client/src/main/java/org/apache/linkis/errorcode/client/synchronizer/LinkisErrorCodeSynchronizer.java index 2b3a3f9fed..7861a40432 100644 --- a/linkis-public-enhancements/linkis-pes-client/src/main/java/org/apache/linkis/errorcode/client/synchronizer/LinkisErrorCodeSynchronizer.java +++ b/linkis-public-enhancements/linkis-pes-client/src/main/java/org/apache/linkis/errorcode/client/synchronizer/LinkisErrorCodeSynchronizer.java @@ -83,7 +83,7 @@ private void init() { } public static LinkisErrorCodeSynchronizer getInstance() { - if (linkisErrorCodeSynchronizer == null) { + if (linkisErrorCodeSynchronizer == null) { // NOSONAR synchronized (LinkisErrorCodeSynchronizer.class) { if (linkisErrorCodeSynchronizer == null) { linkisErrorCodeSynchronizer = new LinkisErrorCodeSynchronizer(); diff --git a/linkis-public-enhancements/linkis-pes-common/src/main/java/org/apache/linkis/cs/common/serialize/helper/ContextSerializationHelper.java b/linkis-public-enhancements/linkis-pes-common/src/main/java/org/apache/linkis/cs/common/serialize/helper/ContextSerializationHelper.java index 6c22c075c0..7d523d815b 100644 --- a/linkis-public-enhancements/linkis-pes-common/src/main/java/org/apache/linkis/cs/common/serialize/helper/ContextSerializationHelper.java +++ b/linkis-public-enhancements/linkis-pes-common/src/main/java/org/apache/linkis/cs/common/serialize/helper/ContextSerializationHelper.java @@ -71,7 +71,7 @@ private void init() throws CSErrorException { private static ContextSerializationHelper contextSerializationHelper = null; public static ContextSerializationHelper getInstance() { - if (contextSerializationHelper == null) { + if (contextSerializationHelper == null) { // NOSONAR synchronized (ContextSerializationHelper.class) { if (contextSerializationHelper == null) { contextSerializationHelper = new ContextSerializationHelper(); diff --git a/linkis-public-enhancements/linkis-pes-common/src/main/java/org/apache/linkis/cs/common/serialize/impl/context/LinkisHAWorkFlowContextIDSerializer.java b/linkis-public-enhancements/linkis-pes-common/src/main/java/org/apache/linkis/cs/common/serialize/impl/context/LinkisHAWorkFlowContextIDSerializer.java index 7a967af9f9..2e752df100 100644 --- a/linkis-public-enhancements/linkis-pes-common/src/main/java/org/apache/linkis/cs/common/serialize/impl/context/LinkisHAWorkFlowContextIDSerializer.java +++ b/linkis-public-enhancements/linkis-pes-common/src/main/java/org/apache/linkis/cs/common/serialize/impl/context/LinkisHAWorkFlowContextIDSerializer.java @@ -37,7 +37,7 @@ public LinkisHAWorkFlowContextID fromJson(String json) throws CSErrorException { @Override public boolean accepts(Object obj) { - if (null != obj && obj.getClass().getName().equals(LinkisHAWorkFlowContextID.class.getName())) { + if (null != obj && obj.getClass().isAssignableFrom(LinkisHAWorkFlowContextID.class)) { return true; } return false; diff --git a/linkis-public-enhancements/linkis-pes-common/src/main/java/org/apache/linkis/cs/common/serialize/impl/history/metadata/CSTableLineageSerializer.java b/linkis-public-enhancements/linkis-pes-common/src/main/java/org/apache/linkis/cs/common/serialize/impl/history/metadata/CSTableLineageSerializer.java index 7b83f7cf5a..5fda484c92 100644 --- a/linkis-public-enhancements/linkis-pes-common/src/main/java/org/apache/linkis/cs/common/serialize/impl/history/metadata/CSTableLineageSerializer.java +++ b/linkis-public-enhancements/linkis-pes-common/src/main/java/org/apache/linkis/cs/common/serialize/impl/history/metadata/CSTableLineageSerializer.java @@ -64,6 +64,6 @@ public String getType() { @Override public boolean accepts(Object obj) { - return null != obj && obj.getClass().getName().equals(CSTableLineageHistory.class.getName()); + return null != obj && obj.getClass().isAssignableFrom(CSTableLineageHistory.class); } } diff --git a/linkis-public-enhancements/linkis-pes-common/src/main/java/org/apache/linkis/cs/common/serialize/impl/history/metadata/CSTableMetadataHistorySerializer.java b/linkis-public-enhancements/linkis-pes-common/src/main/java/org/apache/linkis/cs/common/serialize/impl/history/metadata/CSTableMetadataHistorySerializer.java index d6aafb0634..6c97a2840f 100644 --- a/linkis-public-enhancements/linkis-pes-common/src/main/java/org/apache/linkis/cs/common/serialize/impl/history/metadata/CSTableMetadataHistorySerializer.java +++ b/linkis-public-enhancements/linkis-pes-common/src/main/java/org/apache/linkis/cs/common/serialize/impl/history/metadata/CSTableMetadataHistorySerializer.java @@ -57,7 +57,6 @@ public String getType() { @Override public boolean accepts(Object obj) { - return null != obj - && obj.getClass().getName().equals(CSTableMetadataContextHistory.class.getName()); + return null != obj && obj.getClass().isAssignableFrom(CSTableMetadataContextHistory.class); } } diff --git a/linkis-public-enhancements/linkis-pes-common/src/main/java/org/apache/linkis/cs/listener/ListenerBus/ContextAsyncListenerBus.java b/linkis-public-enhancements/linkis-pes-common/src/main/java/org/apache/linkis/cs/listener/ListenerBus/ContextAsyncListenerBus.java index c6e4116c1d..f880c048c2 100644 --- a/linkis-public-enhancements/linkis-pes-common/src/main/java/org/apache/linkis/cs/listener/ListenerBus/ContextAsyncListenerBus.java +++ b/linkis-public-enhancements/linkis-pes-common/src/main/java/org/apache/linkis/cs/listener/ListenerBus/ContextAsyncListenerBus.java @@ -43,7 +43,7 @@ public void doPostEvent(L listener, E event) { private static ContextAsyncListenerBus contextAsyncListenerBus = null; public static ContextAsyncListenerBus getInstance() { - if (contextAsyncListenerBus == null) { + if (contextAsyncListenerBus == null) { // NOSONAR synchronized (ContextAsyncListenerBus.class) { if (contextAsyncListenerBus == null) { contextAsyncListenerBus = new ContextAsyncListenerBus(); diff --git a/linkis-public-enhancements/linkis-pes-common/src/main/java/org/apache/linkis/cs/listener/callback/imp/DefaultContextIDCallbackEngine.java b/linkis-public-enhancements/linkis-pes-common/src/main/java/org/apache/linkis/cs/listener/callback/imp/DefaultContextIDCallbackEngine.java index 3910deef77..17ab7cf746 100644 --- a/linkis-public-enhancements/linkis-pes-common/src/main/java/org/apache/linkis/cs/listener/callback/imp/DefaultContextIDCallbackEngine.java +++ b/linkis-public-enhancements/linkis-pes-common/src/main/java/org/apache/linkis/cs/listener/callback/imp/DefaultContextIDCallbackEngine.java @@ -147,7 +147,7 @@ public void onEventError(Event event, Throwable t) {} private DefaultContextIDCallbackEngine() {} public static DefaultContextIDCallbackEngine getInstance() { - if (singleDefaultContextIDCallbackEngine == null) { + if (singleDefaultContextIDCallbackEngine == null) { // NOSONAR synchronized (DefaultContextIDCallbackEngine.class) { if (singleDefaultContextIDCallbackEngine == null) { singleDefaultContextIDCallbackEngine = new DefaultContextIDCallbackEngine(); diff --git a/linkis-public-enhancements/linkis-pes-common/src/main/java/org/apache/linkis/cs/listener/callback/imp/DefaultContextKeyCallbackEngine.java b/linkis-public-enhancements/linkis-pes-common/src/main/java/org/apache/linkis/cs/listener/callback/imp/DefaultContextKeyCallbackEngine.java index c78fdc58d2..7ce58f172a 100644 --- a/linkis-public-enhancements/linkis-pes-common/src/main/java/org/apache/linkis/cs/listener/callback/imp/DefaultContextKeyCallbackEngine.java +++ b/linkis-public-enhancements/linkis-pes-common/src/main/java/org/apache/linkis/cs/listener/callback/imp/DefaultContextKeyCallbackEngine.java @@ -61,7 +61,7 @@ public void registerClient(ListenerDomain listenerDomain) { } } // 针对cskey生成一个bean，cskey对应的value值目前为空 - if (contextKey != null) { + if (contextKey != null && contextID != null) { ContextKeyValueBean contextKeyValueBean = new ContextKeyValueBean(); contextKeyValueBean.setCsKey(contextKey); contextKeyValueBean.setCsID(contextID); @@ -160,7 +160,7 @@ public void onEventError(Event event, Throwable t) {} private DefaultContextKeyCallbackEngine() {} public static DefaultContextKeyCallbackEngine getInstance() { - if (singleDefaultContextKeyCallbackEngine == null) { + if (singleDefaultContextKeyCallbackEngine == null) { // NOSONAR synchronized (DefaultContextKeyCallbackEngine.class) { if (singleDefaultContextKeyCallbackEngine == null) { singleDefaultContextKeyCallbackEngine = new DefaultContextKeyCallbackEngine(); diff --git a/linkis-public-enhancements/linkis-pes-common/src/main/java/org/apache/linkis/cs/listener/manager/imp/DefaultContextListenerManager.java b/linkis-public-enhancements/linkis-pes-common/src/main/java/org/apache/linkis/cs/listener/manager/imp/DefaultContextListenerManager.java index 51b9e78d08..38295441d0 100644 --- a/linkis-public-enhancements/linkis-pes-common/src/main/java/org/apache/linkis/cs/listener/manager/imp/DefaultContextListenerManager.java +++ b/linkis-public-enhancements/linkis-pes-common/src/main/java/org/apache/linkis/cs/listener/manager/imp/DefaultContextListenerManager.java @@ -48,7 +48,7 @@ public DefaultContextKeyCallbackEngine getContextKeyCallbackEngine() { private DefaultContextListenerManager() {} public static DefaultContextListenerManager getInstance() { - if (singleDefaultContextListenerManager == null) { + if (singleDefaultContextListenerManager == null) { // NOSONAR synchronized (DefaultContextListenerManager.class) { if (singleDefaultContextListenerManager == null) { singleDefaultContextListenerManager = new DefaultContextListenerManager(); diff --git a/linkis-public-enhancements/linkis-pes-common/src/main/java/org/apache/linkis/metadata/query/common/cache/ConnCacheManager.java b/linkis-public-enhancements/linkis-pes-common/src/main/java/org/apache/linkis/metadata/query/common/cache/ConnCacheManager.java index 65fbcdf373..3f2863758d 100644 --- a/linkis-public-enhancements/linkis-pes-common/src/main/java/org/apache/linkis/metadata/query/common/cache/ConnCacheManager.java +++ b/linkis-public-enhancements/linkis-pes-common/src/main/java/org/apache/linkis/metadata/query/common/cache/ConnCacheManager.java @@ -29,7 +29,7 @@ public class ConnCacheManager implements CacheManager { private ConnCacheManager() {} public static CacheManager custom() { - if (null == manager) { + if (null == manager) { // NOSONAR synchronized (ConnCacheManager.class) { if (null == manager) { manager = new ConnCacheManager(); diff --git a/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/filesystem/exception/WorkspaceExceptionManager.java b/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/filesystem/exception/WorkspaceExceptionManager.java index 3b664f86fc..ee1c8bece1 100644 --- a/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/filesystem/exception/WorkspaceExceptionManager.java +++ b/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/filesystem/exception/WorkspaceExceptionManager.java @@ -23,78 +23,78 @@ public class WorkspaceExceptionManager { - private static final Map desc = - new HashMap(32) { - { - put( - "80001", - "Requesting IO-Engine to initialize fileSystem failed(请求IO-Engine初始化fileSystem失败)!"); - put( - "80002", - "The user has obtained the filesystem for more than {0} ms. Please contact the administrator(用户获取filesystem的时间超过{0} ms，请联系管理员)"); - put( - "80003", - "User local root directory:{0} does not exist, please contact administrator to add(用户本地根目录:{0}不存在,请联系管理员添加)"); - put("80004", "The path:{0} is empty(路径:{0} 为空)!"); - put("80005", "The created folder name:{0} is duplicated(创建的文件夹名:{0} 重复)"); - put("80006", "The file name:{0} created is duplicated(创建的文件名:{0} 重复)"); - put("80007", "The renamed name:{0} is repeated,(重命名的名字:{0} 重复)"); - put("80008", "The deleted file or folder does not exist(删除的文件or文件夹不存在)!"); - put( - "80009", - "This user does not have permission to delete this file or folder(该用户无权删除此文件或文件夹)!"); - put( - "80010", - "The user: {0} has no permission to view the contents of the directory:{1}(该用户:{0}无权限查看该目录:{1}的内容)."); - put("80011", "Downloaded file: {0} does not exist(下载的文件:{0}不存在)"); - put("80012", "This user has no permission to read this file(该用户无权读取该文件)!"); - put("80013", "File: {0} does not exist(文件:{0}不存在)"); - put( - "80014", - "The user has no permission to modify the contents of this file and cannot save it(该用户无权限对此文件内容进行修改，无法保存)!"); - put("80015", "Unsupported resultset output type(不支持的结果集输出类型)"); - put("80016", "The file content is empty and cannot be imported(文件内容为空，不能进行导入操作)!"); - put( - "80017", - "The header of the file has no qualifiers. Do not check the first behavior header or set no qualifier(该文件的表头没有限定符，请勿勾选首行为表头或者设置无限定符)!"); - put("80018", "This user has no permission to read this log(该用户无权限读取此日志)!"); - put("80019", "ScriptContent is empty,this is normal(scriptContent 为空，这是正常的)!"); - put("80021", "Upload failed(上传失败)"); - put("80022", "Update failed(更新失败)"); - put("80023", "Download failed(下载失败)"); - put( - "80024", - "Non-tabular result sets cannot be downloaded as excel(非table类型的结果集不能下载为excel)"); - put( - "80028", - "The path exist special char,only support numbers, uppercase letters, underscores, Chinese(路径存在特殊字符,只支持数字,字母大小写,下划线,中文)"); - put("80029", "Empty dir(空目录)!"); - put("80030", "Creating user path: {0} failed(创建用户路径:{0}失败)"); - put("80031", "User: {0} not initialized(用户:{0}未初始化)"); - put( - "80032", - "The file size exceeds 30M and page viewing is currently not supported. Please download to view or view in a shared directory(文件大小超过30M，暂不支持页面查看。请下载查看或在共享目录中查看)"); - put( - "80033", - "The log file exceeds 30MB and is too large and cannot be opened, path : {0} (日志文件超过30M，文件太大暂不支持打开查看，文件地址：{0})"); - put( - "80034", - "The result set exceeds {0} rows and page viewing is currently not supported. Please download to view or view in the shared directory(结果集行数超过{0}行，暂不支持页面查看。请下载查看或在共享目录中查看)"); - put( - "80035", - "Parameter error, column index order is incorrect, please pass parameters in ascending order (参数错误，列索引顺序不正确或范围错误，请传入非复数并按升序传参)"); - put( - "80036", - "Parameter error, page size is incorrect, please pass in a number within [1-500] (分页参数错误，页码从1开始，页大小需在[1-500]范围内，获取的列索引需在实际结果集列数范围内)"); - put( - "80037", - "Parameter error, page size is incorrect, please pass in a number within [1-500] (参数错误，列筛选最多支持筛选50列)"); - put( - "80038", - "The name directory {0} specified by PKG-INFO does not exist. Please confirm that the {0} specified by PKG-INFO in the package matches the actual folder name (PKG-INFO指定Name目录{0}不存在，请确认包中PKG-INFO指定{0}和实际文件夹名称一致)"); - put("80039", "File upload failed, error message: {0} (文件上传失败，错误信息：{0})"); - } - }; + private static final Map desc = initialDescMap(); + + private static HashMap initialDescMap() { + HashMap map = new HashMap<>(); + map.put( + "80001", + "Requesting IO-Engine to initialize fileSystem failed(请求IO-Engine初始化fileSystem失败)!"); + map.put( + "80002", + "The user has obtained the filesystem for more than {0} ms. Please contact the administrator(用户获取filesystem的时间超过{0} ms，请联系管理员)"); + map.put( + "80003", + "User local root directory:{0} does not exist, please contact administrator to add(用户本地根目录:{0}不存在,请联系管理员添加)"); + map.put("80004", "The path:{0} is empty(路径:{0} 为空)!"); + map.put("80005", "The created folder name:{0} is duplicated(创建的文件夹名:{0} 重复)"); + map.put("80006", "The file name:{0} created is duplicated(创建的文件名:{0} 重复)"); + map.put("80007", "The renamed name:{0} is repeated,(重命名的名字:{0} 重复)"); + map.put("80008", "The deleted file or folder does not exist(删除的文件or文件夹不存在)!"); + map.put( + "80009", + "This user does not have permission to delete this file or folder(该用户无权删除此文件或文件夹)!"); + map.put( + "80010", + "The user: {0} has no permission to view the contents of the directory:{1}(该用户:{0}无权限查看该目录:{1}的内容)."); + map.put("80011", "Downloaded file: {0} does not exist(下载的文件:{0}不存在)"); + map.put("80012", "This user has no permission to read this file(该用户无权读取该文件)!"); + map.put("80013", "File: {0} does not exist(文件:{0}不存在)"); + map.put( + "80014", + "The user has no permission to modify the contents of this file and cannot save it(该用户无权限对此文件内容进行修改，无法保存)!"); + map.put("80015", "Unsupported resultset output type(不支持的结果集输出类型)"); + map.put("80016", "The file content is empty and cannot be imported(文件内容为空，不能进行导入操作)!"); + map.put( + "80017", + "The header of the file has no qualifiers. Do not check the first behavior header or set no qualifier(该文件的表头没有限定符，请勿勾选首行为表头或者设置无限定符)!"); + map.put("80018", "This user has no permission to read this log(该用户无权限读取此日志)!"); + map.put("80019", "ScriptContent is empty,this is normal(scriptContent 为空，这是正常的)!"); + map.put("80021", "Upload failed(上传失败)"); + map.put("80022", "Update failed(更新失败)"); + map.put("80023", "Download failed(下载失败)"); + map.put( + "80024", "Non-tabular result sets cannot be downloaded as excel(非table类型的结果集不能下载为excel)"); + map.put( + "80028", + "The path exist special char,only support numbers, uppercase letters, underscores, Chinese(路径存在特殊字符,只支持数字,字母大小写,下划线,中文)"); + map.put("80029", "Empty dir(空目录)!"); + map.put("80030", "Creating user path: {0} failed(创建用户路径:{0}失败)"); + map.put("80031", "User: {0} not initialized(用户:{0}未初始化)"); + map.put( + "80032", + "The file size exceeds 30M and page viewing is currently not supported. Please download to view or view in a shared directory(文件大小超过30M，暂不支持页面查看。请下载查看或在共享目录中查看)"); + map.put( + "80033", + "The log file exceeds 30MB and is too large and cannot be opened, path : {0} (日志文件超过30M，文件太大暂不支持打开查看，文件地址：{0})"); + map.put( + "80034", + "The result set exceeds {0} rows and page viewing is currently not supported. Please download to view or view in the shared directory(结果集行数超过{0}行，暂不支持页面查看。请下载查看或在共享目录中查看)"); + map.put( + "80035", + "Parameter error, column index order is incorrect, please pass parameters in ascending order (参数错误，列索引顺序不正确或范围错误，请传入非复数并按升序传参)"); + map.put( + "80036", + "Parameter error, page size is incorrect, please pass in a number within [1-500] (分页参数错误，页码从1开始，页大小需在[1-500]范围内，获取的列索引需在实际结果集列数范围内)"); + map.put( + "80037", + "Parameter error, page size is incorrect, please pass in a number within [1-500] (参数错误，列筛选最多支持筛选50列)"); + map.put( + "80038", + "The name directory {0} specified by PKG-INFO does not exist. Please confirm that the {0} specified by PKG-INFO in the package matches the actual folder name (PKG-INFO指定Name目录{0}不存在，请确认包中PKG-INFO指定{0}和实际文件夹名称一致)"); + map.put("80039", "File upload failed, error message: {0} (文件上传失败，错误信息：{0})"); + return map; + } public static WorkSpaceException createException(int errorCode, Object... format) { return new WorkSpaceException( diff --git a/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/filesystem/restful/api/FsRestfulApi.java b/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/filesystem/restful/api/FsRestfulApi.java index 74aeec15b7..8ba6f78690 100644 --- a/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/filesystem/restful/api/FsRestfulApi.java +++ b/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/filesystem/restful/api/FsRestfulApi.java @@ -959,7 +959,7 @@ public void resultsetToExcel( } break; default: - WorkspaceExceptionManager.createException(80015); + throw WorkspaceExceptionManager.createException(80015); } fileSource.write(fsWriter); fsWriter.flush(); @@ -1147,31 +1147,31 @@ public Message formate( String[][] column = null; // fix csv file with utf-8 with bom chart[] BOMInputStream bomIn = new BOMInputStream(in, false); // don't include the BOM - BufferedReader reader = new BufferedReader(new InputStreamReader(bomIn, encoding)); - - String header = reader.readLine(); - if (StringUtils.isEmpty(header)) { - throw WorkspaceExceptionManager.createException(80016); - } - String[] line = header.split(fieldDelimiter, -1); - int colNum = line.length; - column = new String[2][colNum]; - if (hasHeader) { - for (int i = 0; i < colNum; i++) { - column[0][i] = line[i]; - if (escapeQuotes) { - try { - column[0][i] = column[0][i].substring(1, column[0][i].length() - 1); - } catch (StringIndexOutOfBoundsException e) { - throw WorkspaceExceptionManager.createException(80017); + try (BufferedReader reader = new BufferedReader(new InputStreamReader(bomIn, encoding))) { + String header = reader.readLine(); + if (StringUtils.isEmpty(header)) { + throw WorkspaceExceptionManager.createException(80016); + } + String[] line = header.split(fieldDelimiter, -1); + int colNum = line.length; + column = new String[2][colNum]; + if (hasHeader) { + for (int i = 0; i < colNum; i++) { + column[0][i] = line[i]; + if (escapeQuotes) { + try { + column[0][i] = column[0][i].substring(1, column[0][i].length() - 1); + } catch (StringIndexOutOfBoundsException e) { + throw WorkspaceExceptionManager.createException(80017); + } } + column[1][i] = "string"; + } + } else { + for (int i = 0; i < colNum; i++) { + column[0][i] = "col_" + (i + 1); + column[1][i] = "string"; } - column[1][i] = "string"; - } - } else { - for (int i = 0; i < colNum; i++) { - column[0][i] = "col_" + (i + 1); - column[1][i] = "string"; } } res.put("columnName", column[0]); @@ -1241,35 +1241,35 @@ public Message getSheetInfo( String[][] column = null; // fix csv file with utf-8 with bom chart[] BOMInputStream bomIn = new BOMInputStream(in, false); // don't include the BOM - BufferedReader reader = new BufferedReader(new InputStreamReader(bomIn, encoding)); - - String header = reader.readLine(); - if (StringUtils.isEmpty(header)) { - throw WorkspaceExceptionManager.createException(80016); - } - String[] line = header.split(fieldDelimiter, -1); - int colNum = line.length; - column = new String[2][colNum]; - if (hasHeader) { - for (int i = 0; i < colNum; i++) { - HashMap csvMap = new HashMap<>(); - column[0][i] = line[i]; - if (escapeQuotes) { - try { - csvMap.put(column[0][i].substring(1, column[0][i].length() - 1), "string"); - } catch (StringIndexOutOfBoundsException e) { - throw WorkspaceExceptionManager.createException(80017); + try (BufferedReader reader = new BufferedReader(new InputStreamReader(bomIn, encoding))) { + String header = reader.readLine(); + if (StringUtils.isEmpty(header)) { + throw WorkspaceExceptionManager.createException(80016); + } + String[] line = header.split(fieldDelimiter, -1); + int colNum = line.length; + column = new String[2][colNum]; + if (hasHeader) { + for (int i = 0; i < colNum; i++) { + HashMap csvMap = new HashMap<>(); + column[0][i] = line[i]; + if (escapeQuotes) { + try { + csvMap.put(column[0][i].substring(1, column[0][i].length() - 1), "string"); + } catch (StringIndexOutOfBoundsException e) { + throw WorkspaceExceptionManager.createException(80017); + } + } else { + csvMap.put(column[0][i], "string"); } - } else { - csvMap.put(column[0][i], "string"); + csvMapList.add(csvMap); + } + } else { + for (int i = 0; i < colNum; i++) { + HashMap csvMap = new HashMap<>(); + csvMap.put("col_" + (i + 1), "string"); + csvMapList.add(csvMap); } - csvMapList.add(csvMap); - } - } else { - for (int i = 0; i < colNum; i++) { - HashMap csvMap = new HashMap<>(); - csvMap.put("col_" + (i + 1), "string"); - csvMapList.add(csvMap); } } sheetInfo = new HashMap<>(1); diff --git a/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/api/UDFRestfulApi.java b/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/api/UDFRestfulApi.java index 9ecde360d1..0440e664f0 100644 --- a/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/api/UDFRestfulApi.java +++ b/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/api/UDFRestfulApi.java @@ -870,7 +870,7 @@ public void downloadToLocal( // filename表示文件的默认名称，因为网络传输只支持URL编码的相关支付，因此需要将文件名URL编码后进行传输,前端收到后需要反编码才能获取到真正的名称 response.addHeader("Content-Disposition", "attachment;filename=" + downloadVo.getFileName()); // response.addHeader("Content-Length", "" + file.length()); - outputStream = new BufferedOutputStream(response.getOutputStream()); + outputStream = new BufferedOutputStream(response.getOutputStream()); // NOSONAR response.setContentType("application/octet-stream"); byte[] buffer = new byte[1024]; int hasRead = 0; diff --git a/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/service/impl/UDFServiceImpl.java b/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/service/impl/UDFServiceImpl.java index e5155c8ddb..ed65ee8573 100644 --- a/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/service/impl/UDFServiceImpl.java +++ b/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/service/impl/UDFServiceImpl.java @@ -407,28 +407,20 @@ public UDFInfo createSharedUdfInfo(UDFInfo udfInfo, Long shareParentId, FsPath s private UDFTree getOrCreateTree(String userName, String category, String treeName) throws UDFException { + Map selfTreemap = new HashMap<>(); + selfTreemap.put("parent", -1); + selfTreemap.put("userName", userName); + selfTreemap.put("category", category); // 个人函数目录 - List selfTree = - udfTreeDao.getTreesByParentId( - new HashMap() { - { - put("parent", -1); - put("userName", userName); - put("category", category); - } - }); + List selfTree = udfTreeDao.getTreesByParentId(selfTreemap); if (selfTree == null || selfTree.size() == 0) { throw new UDFException("该用户没有个人函数目录!"); } - List selfTreeChildren = - udfTreeDao.getTreesByParentId( - new HashMap() { - { - put("parent", selfTree.get(0).getId()); - put("userName", userName); - put("category", category); - } - }); + Map selfTreeChildrenMap = new HashMap<>(); + selfTreeChildrenMap.put("parent", selfTree.get(0).getId()); + selfTreeChildrenMap.put("userName", userName); + selfTreeChildrenMap.put("category", category); + List selfTreeChildren = udfTreeDao.getTreesByParentId(selfTreeChildrenMap); for (UDFTree tree : selfTreeChildren) { if (tree.getName().equals(treeName)) { return tree; @@ -612,17 +604,15 @@ public PageInfo getManagerPages( } } boolean finalCanExpire = canExpire; - l.setOperationStatus( - new HashMap() { - { - put("canUpdate", true); - put("canDelete", !finalCanExpire); - put("canExpire", finalCanExpire); - put("canShare", ismanager); - put("canPublish", ismanager && Boolean.TRUE.equals(l.getShared())); - put("canHandover", true); - } - }); + Map operationStatusMap = new HashMap<>(); + operationStatusMap.put("canUpdate", true); + operationStatusMap.put("canDelete", !finalCanExpire); + operationStatusMap.put("canExpire", finalCanExpire); + operationStatusMap.put("canShare", ismanager); + operationStatusMap.put( + "canPublish", ismanager && Boolean.TRUE.equals(l.getShared())); + operationStatusMap.put("canHandover", true); + l.setOperationStatus(operationStatusMap); }); } logger.info("end to get managerPages."); @@ -1079,17 +1069,15 @@ public List getUdfByNameList(List udfNameList, String creator) } } boolean finalCanExpire = canExpire; - udfInfo.setOperationStatus( - new HashMap() { - { - put("canUpdate", true); - put("canDelete", !finalCanExpire); - put("canExpire", finalCanExpire); - put("canShare", ismanager); - put("canPublish", ismanager && Boolean.TRUE.equals(udfInfo.getShared())); - put("canHandover", true); - } - }); + Map operationStatusMap = new HashMap<>(); + operationStatusMap.put("canUpdate", true); + operationStatusMap.put("canDelete", !finalCanExpire); + operationStatusMap.put("canExpire", finalCanExpire); + operationStatusMap.put("canShare", ismanager); + operationStatusMap.put( + "canPublish", ismanager && Boolean.TRUE.equals(udfInfo.getShared())); + operationStatusMap.put("canHandover", true); + udfInfo.setOperationStatus(operationStatusMap); }); return retList; } diff --git a/linkis-spring-cloud-services/linkis-service-gateway/linkis-spring-cloud-gateway/src/main/java/org/apache/linkis/gateway/springcloud/http/GatewayAuthorizationFilter.java b/linkis-spring-cloud-services/linkis-service-gateway/linkis-spring-cloud-gateway/src/main/java/org/apache/linkis/gateway/springcloud/http/GatewayAuthorizationFilter.java index edeee8f170..cef85b843e 100644 --- a/linkis-spring-cloud-services/linkis-service-gateway/linkis-spring-cloud-gateway/src/main/java/org/apache/linkis/gateway/springcloud/http/GatewayAuthorizationFilter.java +++ b/linkis-spring-cloud-services/linkis-service-gateway/linkis-spring-cloud-gateway/src/main/java/org/apache/linkis/gateway/springcloud/http/GatewayAuthorizationFilter.java @@ -210,7 +210,7 @@ private Mono gatewayDeal( realRd = rd; } } - String uri = realRd.getUri().toString(); + String uri = realRd.getUri().toString(); // NOSONAR if (uri != null) { uri = uri From cb4e395a004e85457a2758e051a6f3ef75e6c980 Mon Sep 17 00:00:00 2001 From: Yonghao Mei <73584269+mayinrain@users.noreply.github.com> Date: Tue, 19 Nov 2024 16:16:25 +0800 Subject: [PATCH 64/90] chore: 1.9.0 (#653) --- .../apps/PythonModule/public/tutorial.html | 87 ++++++++++++++ .../src/apps/PythonModule/src/global.less | 3 + .../apps/PythonModule/src/locales/en-US.js | 12 +- .../apps/PythonModule/src/locales/zh-CN.js | 11 +- .../PythonModule/src/pages/index/index.jsx | 112 ++++++++++++------ .../apps/PythonModule/src/pages/index/main.js | 74 ++++++++---- .../datasource/datasourceForm/index.vue | 8 ++ .../apps/linkis/module/datasource/index.vue | 10 +- .../globalHistoryManagement/viewHistory.vue | 18 ++- .../module/resourceManagement/index.vue | 14 +-- .../src/apps/linkis/view/linkis/index.vue | 1 + 11 files changed, 277 insertions(+), 73 deletions(-) create mode 100644 linkis-web/src/apps/PythonModule/public/tutorial.html diff --git a/linkis-web/src/apps/PythonModule/public/tutorial.html b/linkis-web/src/apps/PythonModule/public/tutorial.html new file mode 100644 index 0000000000..794b2bc0ed --- /dev/null +++ b/linkis-web/src/apps/PythonModule/public/tutorial.html @@ -0,0 +1,87 @@ + + + + + + Python 模块使用文档 + + + +
+
+ 注意事项：当前linkis支持的python版本为2.7.13和3.6.4两个版本，用户上传的python文件或模块必须兼容对应的python版本才能正常使用。 +
+

1. 首先你需要准备好对应的模块物料，支持.py文件、.tar.gz和.zip文件

+

a. 对于py 文件你的模块名一定要和文件名一致，如hello_world.py 模块名就是hello_world,您可以在里面定义你的python方法和class等，如下：

+ 
+            
#文件名一定是 hello_world.py

+def add_hello_world(name):

+  return "hello world {}".format(name)
+            
+
+

b. 对于zip文件需要稍微复杂点，你得遵循Python模块包的文件格式，如下：

+ 
+            
#zip文件名hello_world.zip

+hello_world.zip

+├── hello_world # 包目录，必须为模块名

+├── __init__.py # 必须有该文件

+├── hello_world.py # 其它python文件

+
#hello_world.py文件内容如下：

+def add_hello_world(name):

+  return "hello world {}".format(name)
+            
+
+

需要有 __init__.py 文件才能让 Python 将包含该文件的目录当作包来处理，可以只是一个空文件。你也可以直接去pip网站上面下载对应的包，然后通过zip压缩即可

+

2. 在linkis 管理台建立对应的模块：

+

第一步点击新增

+

第二步选择对应的引擎，分别对于Spark（.py脚本），python（.python脚本），通用（.py脚本和.python脚本）

+

第三步上传之前准备的模块物料，如果要更新模块内容需要先将旧的模块删除后再重新上传

+

第四步选择加载才可以进行使用

+

第五步如果你的模块不需要了，可以选择过期，等同于删除

+ +

3. 在scritpis进行使用

+

1. 如果你的模块已经加好了，可以直接在脚本中通过import进行使用如下：

+ 
+            
from hello_world import add_hello_world

+add_hello_world("peace")
+            
+
+

2. 可以将对应的模块注册成Spark的UDF进行使用

+ 
+            
from pyspark.sql.functions import udf

+from pyspark.sql.types import StringType

+from hello_world import add_hello_world

+add_hello_world_udf = udf(add_hello_world, StringType())

+df=spark.sql("select * from dws_demo.demo_user_info limit 100")

+df_transformed = df.withColumn("username", add_hello_world_udf(df["username"]))

+df_transformed.show()
+            
+
+
+ + \ No newline at end of file diff --git a/linkis-web/src/apps/PythonModule/src/global.less b/linkis-web/src/apps/PythonModule/src/global.less index 67cc7b0090..94e665c23b 100644 --- a/linkis-web/src/apps/PythonModule/src/global.less +++ b/linkis-web/src/apps/PythonModule/src/global.less @@ -9,3 +9,6 @@ body { #app { height: 100vh; } +.fes-form .fes-form-item-label { + justify-content: flex-end; +} diff --git a/linkis-web/src/apps/PythonModule/src/locales/en-US.js b/linkis-web/src/apps/PythonModule/src/locales/en-US.js index fa159589f3..d714976e4f 100644 --- a/linkis-web/src/apps/PythonModule/src/locales/en-US.js +++ b/linkis-web/src/apps/PythonModule/src/locales/en-US.js @@ -25,7 +25,7 @@ export default { cancel: 'Cancel', moduleResource: 'Module Resource', uploadResource: 'Please upload module resource', - onlyPyAndZip: 'Only .py and .zip files are supported', + onlyPyAndZip: 'Only .py, .zip and tar.gz files are supported', selectIsLoaded: 'Please select if loaded', selectIsExpire: 'Please select if expired', moduleDescription: 'Module Description', @@ -46,12 +46,18 @@ export default { confirmStatusChange: 'Confirm Load Status Change', confirmStatusChangeContent: 'Are you sure you want to change the load status of module {name}?', moduleNameTooLong: 'Module name cannot exceed 50 characters', - moduleNameNotFormat: 'Module name can only contain letters, numbers, and underscores, and must start with a letter', + // 支持横线 + moduleNameNotFormat: 'Module name can only contain letters, numbers, dash, and underscores, and must start with a letter', moduleNameExist: 'Module name {name} already exists, please delete the old module before re-uploading', moduleSizeExceed: 'Module size cannot exceed 50MB', saveSuccess: 'Saved successfully', normal: 'Normal', expire: 'Expired', known: 'Got it', - uploadFile: 'Upload File' + uploadFile: 'Upload File', + fullScreen: 'Full Screen View', + pythonModule: 'Module Dependencies', + placeholderPyModules: 'Please enter the module dependencies.', + noDeps: 'There is no module dependencies', + messageForPyModules: 'Please enter the module dependencies, separated by commas, Module name cannot exceed 50 characters and can only contain letters, numbers, dash, and underscores, and must start with a letter' }; diff --git a/linkis-web/src/apps/PythonModule/src/locales/zh-CN.js b/linkis-web/src/apps/PythonModule/src/locales/zh-CN.js index 7607a233da..a555064503 100644 --- a/linkis-web/src/apps/PythonModule/src/locales/zh-CN.js +++ b/linkis-web/src/apps/PythonModule/src/locales/zh-CN.js @@ -25,7 +25,7 @@ export default { cancel: '取消', moduleResource: '模块物料', uploadResource: '请上传模块物料', - onlyPyAndZip: '仅支持.py和.zip格式文件', + onlyPyAndZip: '仅支持.py、.zip和.tar.gz格式文件', selectIsLoaded: '请选择是否加载', selectIsExpire: '请选择是否过期', moduleDescription: '模块描述', @@ -46,12 +46,17 @@ export default { confirmStatusChange: '确认加载状态变更', confirmStatusChangeContent: '您确定要修改模块 {name} 的加载状态吗？', moduleNameTooLong: '模块名称长度不能超过50个字符', - moduleNameNotFormat: '模块名称只支持数字字母下划线，且以字母开头', + moduleNameNotFormat: '模块名称只支持数字字母横线下划线，且以字母开头', moduleNameExist: '模块名称{name}已存在，如需重新上传请先删除旧的模块', moduleSizeExceed: '模块大小不能超过50MB', saveSuccess: '保存成功', normal: '正常', expire: '过期', known: '知道了', - uploadFile: '上传文件' + uploadFile: '上传文件', + fullScreen: '全屏查看', + pythonModule: '模块依赖', + placeholderPyModules: '请输入模块依赖', + noDeps: '无依赖模块', + messageForPyModules: '模块名长度最大为50，支持数字字母横线下划线，且以英文开头', }; diff --git a/linkis-web/src/apps/PythonModule/src/pages/index/index.jsx b/linkis-web/src/apps/PythonModule/src/pages/index/index.jsx index 672375d8b8..5ec93dec9e 100644 --- a/linkis-web/src/apps/PythonModule/src/pages/index/index.jsx +++ b/linkis-web/src/apps/PythonModule/src/pages/index/index.jsx @@ -51,10 +51,11 @@ export default defineComponent({ id: 'apiPythonuploadFilesystem', query(params) { return letgoRequest( - '/api/rest_j/v1/filesystem/python-upload', + '/api/rest_j/v1/udf/python-upload', params, { - method: 'POST' + method: 'POST', + timeout: 60000 }, ); }, @@ -204,7 +205,7 @@ export default defineComponent({ _label={$t('moduleName')} placeholder={$t('moduleNamePlaceholder')} v-model={$$.pythonModuleName} - span={5} + span={6} style={{ width: '260px', }} @@ -214,7 +215,7 @@ export default defineComponent({ _label={$t('userName')} placeholder={$t('userNamePlaceholder')} v-model={$$.userName} - span={5} + span={6} style={{ width: '260px', }} @@ -223,7 +224,7 @@ export default defineComponent({ - + /> */} + {$t('onlyPyAndZip')} @@ -463,7 +470,7 @@ export default defineComponent({ if($$.selectedModule.name && $$.selectedModule.path) { return (
- {$$.selectedModule.name + '.' + $$.selectedModule.path?.split('.')[1] || ''} + {$$.selectedModule?.path?.split('/')[$$.selectedModule?.path?.split('/').length - 1] || ''}
); } @@ -474,7 +481,7 @@ export default defineComponent({ } }} beforeUpload={(...args) => $$.validateModuleFile(...args)} - accept={['.zip', '.py']} + accept={['.zip', '.py', '.tar.gz']} httpRequest={(...args) => $$.handleUploadHttpRequest(...args)} > }} + loading={$$.isUploading} > {$t('uploadFile')} @@ -507,7 +515,7 @@ export default defineComponent({ }, ]} /> - + /> */} + {$t('onlyPyAndZip')} @@ -638,7 +652,7 @@ export default defineComponent({ if($$.selectedModule.name && $$.selectedModule.path) { return (
- {$$.selectedModule.name + '.' + $$.selectedModule.path?.split('.')[1] || ''} + {$$.selectedModule?.path?.split('/')[$$.selectedModule?.path?.split('/').length - 1] || ''}
); } @@ -649,7 +663,7 @@ export default defineComponent({ } }} beforeUpload={(...args) => $$.validateModuleFile(...args)} - accept={['.zip', '.py']} + accept={['.zip', '.py', '.tar.gz']} httpRequest={(...args) => $$.handleUploadHttpRequest(...args)} > }} + loading={$$.isUploading} > {$t('uploadFile')} @@ -682,7 +697,7 @@ export default defineComponent({ }, ]} /> - + /> */} document.body} v-slots={{ title: () => { - return(

{$t('useTutorial')}

) - } + return( +
+

{$t('useTutorial')}

+ $$.openNewTab()}>{$t('fullScreen')} +
+ ) + }, + }} onUpdate:show={[ () => { @@ -754,7 +775,10 @@ export default defineComponent({ ]} >
-

1. 首先你需要准备好对应的模块物料，支持.py文件和.zip文件

+
+ 注意事项：当前linkis支持的python版本为2.7.13和3.6.4两个版本，用户上传的python文件或模块必须兼容对应的python版本才能正常使用。 +
+

1. 首先你需要准备好对应的模块物料，支持.py文件, tar.gz文件, .zip文件

a. 对于py 文件你的模块名一定要和文件名一致，如hello_world.py 模块名就是hello_world,您可以在里面定义你的python方法和class等，如下：

                   #文件名一定是 hello_world.py

@@ -823,28 +847,42 @@ df_transformed.show()
                         : 'Python';
                   },
                 },
-                {
-                  prop: 'status',
-                  label: $t('status'),
-                  formatter: ({
-                    row,
-                    column,
-                    rowIndex,
-                    coloumIndex,
-                    cellValue,
-                  }) => {
-                    return row.isExpire === 0
-                      ? $t('normal')
-                      : $t('expire');
-                  },
-                },
+                // {
+                //   prop: 'status',
+                //   label: $t('status'),
+                //   formatter: ({
+                //     row,
+                //     column,
+                //     rowIndex,
+                //     coloumIndex,
+                //     cellValue,
+                //   }) => {
+                //     return row.isExpire === 0
+                //       ? $t('normal')
+                //       : $t('expire');
+                //   },
+                // },
                 {
                   prop: 'path',
                   label: $t('pathInfo'),
                 },
+                {
+                  prop: 'pythonModule',
+                  label: $t('pythonModule'),
+                  formatter: ({row, column ,rowIndex, columnIndex, cellValue}) => {
+                    if(!row.pythonModule) {
+                      return '- -'
+                    }
+                  }
+                },
                 {
                   prop: 'description',
                   label: $t('moduleDescription'),
+                  formatter: ({row, column, rowIndex, columnIndex, cellValue}) => {
+                    if(!row.description) {
+                      return '- -'
+                    }
+                  }
                 },
                 {
                   prop: 'createTime',
diff --git a/linkis-web/src/apps/PythonModule/src/pages/index/main.js b/linkis-web/src/apps/PythonModule/src/pages/index/main.js
index 14a8942948..c829101d01 100644
--- a/linkis-web/src/apps/PythonModule/src/pages/index/main.js
+++ b/linkis-web/src/apps/PythonModule/src/pages/index/main.js
@@ -6,7 +6,7 @@ export class Main extends LetgoPageBase {
     this.pythonModuleName = '';
     this.userName = '';
     this.engineType = '';
-    this.isExpired = 0;
+    // this.isExpired = 0;
     this.isLoaded = null;
     this.currentPage = 1;
     this.pageSize = 10;
@@ -17,9 +17,10 @@ export class Main extends LetgoPageBase {
     this.newModuleName = '';
     this.selectedEngineType = 'spark';
     this.selectedModuleDescription = '';
+    this.selectedModulePythonModule = '';
     this.selectedModulePath = '';
     this.selectedModuleIsLoad = 1;
-    this.selectedModuleIsExpire = 0;
+    // this.selectedModuleIsExpire = 0;
     this.selectedModuleId = null;
     this.selectedModuleFile = null;
     this.selectedModuleFileError = '';
@@ -33,6 +34,8 @@ export class Main extends LetgoPageBase {
     this.addFormRef = null;
     this.editFormRef = null;
     this.tutorialVisible = false;
+    this.isUploading = false;
+    this.getDep = false;
   }
 
   onMounted () {
@@ -49,7 +52,7 @@ export class Main extends LetgoPageBase {
         engineType: this.engineType,
         username: this.userName,
         // isLoad: this.isLoaded,
-        isExpire: this.isExpired,
+        isExpire: 0,
         pageNow: this.currentPage,
         pageSize: this.pageSize
       };
@@ -58,7 +61,7 @@ export class Main extends LetgoPageBase {
       }
       const response =
                 await this.$pageCode.apiPythonlistUdf.trigger(params);
-      this.pythonModuleList = response.data.pythonList;
+      this.pythonModuleList = response.data.pythonList
       this.totalRecords = response.data.totalPage;
       return response;
     } catch (error) {
@@ -76,7 +79,7 @@ export class Main extends LetgoPageBase {
     this.pythonModuleName = '';
     this.userName = '';
     this.engineType = '';
-    this.isExpired = 0;
+    // this.isExpired = 0;
     this.isLoaded = null;
     this.currentPage = 1;
     this.pageSize = 10;
@@ -87,11 +90,12 @@ export class Main extends LetgoPageBase {
     this.addPythonModuleVisible = true;
     this.selectedModule.name = '';
     this.selectedModule.engineType = 'spark';
-    this.selectedModule.isExpire = 0;
+    // this.selectedModule.isExpire = 0;
     this.selectedModule.isLoad = 1;
     this.selectedModule.path = '';
     this.selectedModule.fileList = [];
     this.selectedModule.description = '';
+    this.selectedModule.pythonModule = '';
   }
 
   showTutorial () {
@@ -169,7 +173,7 @@ export class Main extends LetgoPageBase {
       throw new Error(this.$pageCode.$t('moduleNameTooLong'));
     }
     // 名称只支持数字字母下划线，且以字母开头
-    if (!/^[a-zA-Z][a-zA-Z0-9_]*$/.test(newModuleName.split('.')[0])) {
+    if (!/^[a-zA-Z][a-zA-Z0-9_-]*$/.test(newModuleName.split('.')[0])) {
       this.$utils.FMessage.error(this.$pageCode.$t('moduleNameNotFormat'));
       throw new Error(this.$pageCode.$t('moduleNameNotFormat'));
     }
@@ -209,18 +213,32 @@ export class Main extends LetgoPageBase {
     }
   }
 
+  cutExtension(fileName) {
+    const pyIndex = fileName.indexOf('.py');
+    const zipIndex = fileName.indexOf('.zip');
+    const targzIndex = fileName.indexOf('.tar.gz');
+    if(Math.max(pyIndex, zipIndex, targzIndex) !== -1) {
+      return fileName.substring(0, Math.max(pyIndex, zipIndex, targzIndex));
+    } else {
+      return fileName;
+    }
+  }
   async handleUploadHttpRequest (options) {
     try {
+      this.isUploading = true;
       const formData = new FormData();
-      window.console.log('options:', options);
       formData.append('file', options.file);
       formData.append('fileName', options.file.name);
       const response = await this.$pageCode.apiPythonuploadFilesystem.trigger(formData);
       this.selectedModule.path = response.data.filePath;
-      this.selectedModule.name = options.file.name.split('.')[0];
+      this.selectedModule.name = this.cutExtension(response.data.fileName);
+      this.selectedModule.pythonModule = response.data.dependencies;
+      this.getDep = true;
       this.selectedModule.fileList = [options.file];
+      this.isUploading = false;
     } catch (err) {
       window.console.error(err);
+      this.isUploading = false;
       // this.$utils.FMessage.error('上传失败');
     }
   }
@@ -231,8 +249,9 @@ export class Main extends LetgoPageBase {
     selectedModuleDescription,
     selectedModulePath,
     selectedModuleIsLoad,
-    selectedModuleIsExpire,
-    selectedModuleId
+    // selectedModuleIsExpire,
+    selectedModulePythonModule,
+    selectedModuleId,
   ) {
     const params = {
       name: newModuleName,
@@ -240,7 +259,8 @@ export class Main extends LetgoPageBase {
       path: selectedModulePath,
       engineType: selectedEngineType,
       isLoad: selectedModuleIsLoad,
-      isExpire: selectedModuleIsExpire
+      pythonModule: selectedModulePythonModule,
+      isExpire: 0
     };
     if (selectedModuleId) {
       params.id = selectedModuleId;
@@ -284,7 +304,9 @@ export class Main extends LetgoPageBase {
         this.selectedModule.description,
         this.selectedModule.path,
         this.selectedModule.isLoad,
-        this.selectedModule.isExpire
+        this.selectedModule.pythonModule,
+
+        // this.selectedModule.isExpire
       );
       this.addPythonModuleVisible = false;
       this.loadPythonModuleList();
@@ -302,8 +324,9 @@ export class Main extends LetgoPageBase {
         this.selectedModule.description,
         this.selectedModule.path,
         this.selectedModule.isLoad,
-        this.selectedModule.isExpire,
-        this.selectedModule.id
+        // this.selectedModule.isExpire,
+        this.selectedModule.pythonModule,
+        this.selectedModule.id,
       );
       this.closeEditModuleModal();
       this.loadPythonModuleList();
@@ -338,16 +361,19 @@ export class Main extends LetgoPageBase {
   }
 
   async handleLoadStatusChange () {
-    const { id, name, path, isExpire, isLoad, engineType, description } =
+    const { id, name, path, 
+      // isExpire, 
+      isLoad, engineType, description, pythonModule } =
             this.selectedModule;
     window.console.log({
       id,
       name,
       path,
-      isExpire,
+      // isExpire,
       isLoad,
       engineType,
-      description
+      description,
+      pythonModule
     });
     const targetLoadStatus = isLoad === 1 ? 0 : 1;
     if (id === null) {
@@ -363,8 +389,9 @@ export class Main extends LetgoPageBase {
         description,
         path,
         targetLoadStatus,
-        isExpire,
-        id
+        // isExpire,
+        pythonModule,
+        id,
       );
       await this.loadPythonModuleList();
     } catch (error) {
@@ -377,6 +404,7 @@ export class Main extends LetgoPageBase {
 
   closeAddModuleModal () {
     this.addPythonModuleVisible = false;
+    this.getDep = false;
   }
 
   handleFileListChange ({ file, fileList }) {
@@ -385,6 +413,7 @@ export class Main extends LetgoPageBase {
 
   closeEditModuleModal () {
     this.editPythonModuleVisible = false;
+    this.getDep = false;
   }
 
   closeDeleteConfirmation () {
@@ -394,4 +423,9 @@ export class Main extends LetgoPageBase {
   closeLoadStatusChangeConfirmation () {
     this.loadStatusChangeConfirmationVisible = false;
   }
+
+  openNewTab() {
+    window.open('./tutorial.html', '_blank');
+    this.tutorialVisible = false;
+  }
 }
diff --git a/linkis-web/src/apps/linkis/module/datasource/datasourceForm/index.vue b/linkis-web/src/apps/linkis/module/datasource/datasourceForm/index.vue
index c19af012d3..14f749c6ab 100644
--- a/linkis-web/src/apps/linkis/module/datasource/datasourceForm/index.vue
+++ b/linkis-web/src/apps/linkis/module/datasource/datasourceForm/index.vue
@@ -22,6 +22,7 @@
       v-model="fApi"
       :option="options"
       :value.sync="formData"
+      @change="handleFormChange"
     />
@@ -160,6 +161,7 @@ export default { options: { submitBtn: false, }, + isEncrypt: '', rule: [ { type: 'input', @@ -219,11 +221,17 @@ export default { }, }, methods: { + handleFormChange(field, value, rule, api, setFlag) { + if(field === 'password' && !setFlag) { + this.isEncrypt = ''; + } + }, getDataSource(newV) { if (this.data.id) { getDataSourceByIdAndVersion(newV.id, newV.versionId || 0).then( (result) => { const mConnect = result.info.connectParams + this.isEncrypt = result.info.connectParams?.isEncrypt || ''; this.sourceConnectData = mConnect delete result.info.connectParams this.dataSrc = { ...result.info, ...mConnect } diff --git a/linkis-web/src/apps/linkis/module/datasource/index.vue b/linkis-web/src/apps/linkis/module/datasource/index.vue index 1ad41841bc..82d6fd524a 100644 --- a/linkis-web/src/apps/linkis/module/datasource/index.vue +++ b/linkis-web/src/apps/linkis/module/datasource/index.vue @@ -588,7 +588,9 @@ export default { realFormData.connectParams = formData realFormData.createSystem = 'Linkis' realFormData.dataSourceTypeId = this.currentSourceData.dataSourceTypeId - + if(this.$refs.datasourceForm.isEncrypt) { + realFormData.connectParams.isEncrypt = this.$refs.datasourceForm.isEncrypt; + } let postDataSource = createDataSource let commentMsg = this.$t('message.linkis.datasource.initVersion') if (!this.currentSourceData.id) { @@ -694,8 +696,12 @@ export default { realFormData[key] = formData[key] delete formData[key] }) - realFormData.connectParams = formData + // console.log(this.$refs.datasourceForm.isEncrypt); + if(this.$refs.datasourceForm.isEncrypt) { + realFormData.connectParams.isEncrypt = this.$refs.datasourceForm.isEncrypt; + } + realFormData.createSystem = 'Linkis' realFormData.dataSourceTypeId = this.currentSourceData.dataSourceTypeId diff --git a/linkis-web/src/apps/linkis/module/globalHistoryManagement/viewHistory.vue b/linkis-web/src/apps/linkis/module/globalHistoryManagement/viewHistory.vue index d52fd6b65f..56725cbf15 100644 --- a/linkis-web/src/apps/linkis/module/globalHistoryManagement/viewHistory.vue +++ b/linkis-web/src/apps/linkis/module/globalHistoryManagement/viewHistory.vue @@ -17,7 +17,7 @@