From ade3a1c41a41913239182636879c0da7aa947ac8 Mon Sep 17 00:00:00 2001 From: v-kkhuang <62878639+v-kkhuang@users.noreply.github.com> Date: Fri, 27 Dec 2024 10:54:21 +0800 Subject: [PATCH] Dev 1.10.0 bug fix (#694) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Code Optimization * Code Optimization * Code Optimization * Code Optimization * fix miss code * New interface: Retrieve data source list based on type name * New interface: Retrieve data source list based on type name * bug fix --------- Co-authored-by: “v_kkhuang” <“420895376@qq.com”> --- .../manager/am/conf/AMConfiguration.java | 2 +- .../restful/DataSourceCoreRestfulApi.java | 70 ++++++++++----- .../GetDataSourceByTypeNameAction.scala | 85 +++++++++++++++++++ .../GetDataSourceKeyDefineByTypeAction.scala | 65 ++++++++++++++ .../GetDataSourceByTypeNameResult.scala | 44 ++++++++++ .../GetDataSourceKeyDefineByTypeResult.scala | 47 ++++++++++ 6 files changed, 293 insertions(+), 20 deletions(-) create mode 100644 linkis-public-enhancements/linkis-pes-client/src/main/scala/org/apache/linkis/datasource/client/request/GetDataSourceByTypeNameAction.scala create mode 100644 linkis-public-enhancements/linkis-pes-client/src/main/scala/org/apache/linkis/datasource/client/request/GetDataSourceKeyDefineByTypeAction.scala create mode 100644 linkis-public-enhancements/linkis-pes-client/src/main/scala/org/apache/linkis/datasource/client/response/GetDataSourceByTypeNameResult.scala create mode 100644 linkis-public-enhancements/linkis-pes-client/src/main/scala/org/apache/linkis/datasource/client/response/GetDataSourceKeyDefineByTypeResult.scala diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/conf/AMConfiguration.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/conf/AMConfiguration.java index 903c58ada7..c1d96fb931 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/conf/AMConfiguration.java +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/conf/AMConfiguration.java @@ -101,7 +101,7 @@ public class AMConfiguration { CommonVars.apply("wds.linkis.allow.batch.kill.engine.types", "spark,hive,python"); public static final CommonVars UNALLOW_BATCH_KILL_ENGINE_TYPES = - CommonVars.apply("wds.linkis.allow.batch.kill.engine.types", "trino,appconn,io_file,jdbc"); + CommonVars.apply("wds.linkis.unallow.batch.kill.engine.types", "trino,appconn,io_file,jdbc"); public static final CommonVars MULTI_USER_ENGINE_USER = CommonVars.apply("wds.linkis.multi.user.engine.user", getDefaultMultiEngineUser()); public static final String UDF_KILL_ENGINE_TYPE = diff --git a/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/server/src/main/java/org/apache/linkis/datasourcemanager/core/restful/DataSourceCoreRestfulApi.java b/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/server/src/main/java/org/apache/linkis/datasourcemanager/core/restful/DataSourceCoreRestfulApi.java index c6d99e4061..bbb43d94df 100644 --- a/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/server/src/main/java/org/apache/linkis/datasourcemanager/core/restful/DataSourceCoreRestfulApi.java +++ b/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/server/src/main/java/org/apache/linkis/datasourcemanager/core/restful/DataSourceCoreRestfulApi.java @@ -140,6 +140,37 @@ public Message getKeyDefinitionsByType( "Fail to get key definitions of data source type[查询数据源参数键值对失败]"); } + @ApiOperation( + value = "getKeyDefinitionsByTypeName", + notes = "get key definitions by typeName", + response = Message.class) + @ApiImplicitParams({@ApiImplicitParam(name = "typeName", required = true, dataType = "String")}) + @RequestMapping(value = "/key-define/{typeName}", method = RequestMethod.GET) + public Message getKeyDefinitionsByTypeName( + @PathVariable("typeName") String typeName, HttpServletRequest request) { + return RestfulApiHelper.doAndResponse( + () -> { + String userName = ModuleUserUtils.getOperationUser(request, "getKeyDefinitionsByType"); + List dataSourceTypes = + dataSourceRelateService.getAllDataSourceTypes(request.getHeader("Content-Language")); + DataSourceType targetDataSourceType = + dataSourceTypes.stream() + .filter(type -> type.getName().equals(typeName)) + .findFirst() + .orElse(null); + if (targetDataSourceType != null) { + List keyDefinitions = + dataSourceRelateService.getKeyDefinitionsByType( + Long.valueOf(targetDataSourceType.getId()), + request.getHeader("Content-Language")); + return Message.ok().data("keyDefine", keyDefinitions); + } else { + return Message.error("No data source type found with name: " + typeName); + } + }, + "Fail to get key definitions of data source type[查询数据源参数键值对失败]"); + } + @ApiOperation(value = "insertJsonInfo", notes = "insert json info", response = Message.class) @ApiOperationSupport(ignoreParameters = {"dataSource"}) @ApiImplicitParams({ @@ -898,34 +929,35 @@ public Message encryptDatasourcePassword( @RequestMapping(value = "/info-by-type", method = RequestMethod.GET) public Message getDataSourceListByTypes( HttpServletRequest request, - @RequestParam(value = "typeName", required = false) String typeName, - @RequestParam(value = "currentPage", required = false) Integer currentPage, - @RequestParam(value = "pageSize", required = false) Integer pageSize) { + @RequestParam String typeName, + @RequestParam(required = false, defaultValue = "1") Integer currentPage, + @RequestParam(required = false, defaultValue = "10") Integer pageSize) { return RestfulApiHelper.doAndResponse( () -> { String userName = ModuleUserUtils.getOperationUser(request, "getDataSourceByTypeName"); if (AuthContext.isAdministrator(userName)) { userName = null; } - List queryList = new ArrayList<>(); - Message message = Message.ok(); List dataSourceTypes = dataSourceRelateService.getAllDataSourceTypes(request.getHeader("Content-Language")); - // 从dataSourceTypes过滤出typeName为typeName的数据源类型 - for (DataSourceType dataSourceType : dataSourceTypes) { - if (dataSourceType.getName().equals(typeName)) { - DataSourceVo dataSourceVo = - new DataSourceVo(null, Long.valueOf(dataSourceType.getId()), null, null); - dataSourceVo.setCurrentPage(null != currentPage ? currentPage : 1); - dataSourceVo.setPageSize(null != pageSize ? pageSize : 10); - dataSourceVo.setPermissionUser(userName); - PageInfo pageInfo = - dataSourceInfoService.queryDataSourceInfoPage(dataSourceVo); - queryList = pageInfo.getList(); - message.data("totalPage", pageInfo.getTotal()); - } + DataSourceType targetDataSourceType = + dataSourceTypes.stream() + .filter(type -> type.getName().equals(typeName)) + .findFirst() + .orElse(null); + if (targetDataSourceType != null) { + DataSourceVo dataSourceVo = new DataSourceVo(); + dataSourceVo.setDataSourceTypeId(Long.valueOf(targetDataSourceType.getId())); + dataSourceVo.setPermissionUser(userName); + dataSourceVo.setCurrentPage(currentPage); + dataSourceVo.setPageSize(pageSize); + PageInfo pageInfo = + dataSourceInfoService.queryDataSourceInfoPage(dataSourceVo); + List queryList = pageInfo.getList(); + return Message.ok().data("queryList", queryList).data("totalPage", pageInfo.getTotal()); + } else { + return Message.error("No data source type found with name: " + typeName); } - return message.data("queryList", queryList); }, "Fail to get all types of data source[获取数据源列表失败]"); } diff --git a/linkis-public-enhancements/linkis-pes-client/src/main/scala/org/apache/linkis/datasource/client/request/GetDataSourceByTypeNameAction.scala b/linkis-public-enhancements/linkis-pes-client/src/main/scala/org/apache/linkis/datasource/client/request/GetDataSourceByTypeNameAction.scala new file mode 100644 index 0000000000..25022b2252 --- /dev/null +++ b/linkis-public-enhancements/linkis-pes-client/src/main/scala/org/apache/linkis/datasource/client/request/GetDataSourceByTypeNameAction.scala @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.datasource.client.request + +import org.apache.linkis.datasource.client.config.DatasourceClientConfig.DATA_SOURCE_SERVICE_MODULE +import org.apache.linkis.datasource.client.errorcode.DatasourceClientErrorCodeSummary._ +import org.apache.linkis.datasource.client.exception.DataSourceClientBuilderException +import org.apache.linkis.datasource.client.request.QueryDataSourceAction.Builder +import org.apache.linkis.httpclient.request.GetAction + +class GetDataSourceByTypeNameAction extends GetAction with DataSourceAction { + private var user: String = _ + + override def setUser(user: String): Unit = this.user = user + + override def getUser: String = this.user + + override def suffixURLs: Array[String] = + Array(DATA_SOURCE_SERVICE_MODULE.getValue, "info-by-type") + +} + +object GetDataSourceByTypeNameAction { + def builder(): Builder = new Builder + + class Builder private[GetDataSourceByTypeNameAction] () { + private var user: String = _ + private var typeName: String = _ + private var currentPage: Integer = _ + private var pageSize: Integer = _ + + def setUser(user: String): Builder = { + this.user = user + this + } + + def setTypeName(typeName: String): Builder = { + this.typeName = typeName + this + } + + def setCurrentPage(currentPage: Integer): Builder = { + this.currentPage = currentPage + this + } + + def setPageSize(pageSize: Integer): Builder = { + this.pageSize = pageSize + this + } + + def build(): GetDataSourceByTypeNameAction = { + if (user == null) throw new DataSourceClientBuilderException(USER_NEEDED.getErrorDesc) + val action = new GetDataSourceByTypeNameAction + if (typeName != null) { + action.setParameter("typeName", typeName) + } + if (currentPage != null) { + action.setParameter("currentPage", currentPage) + } + if (pageSize != null) { + action.setParameter("pageSize", pageSize) + } + action.user = user + action + } + + } + +} diff --git a/linkis-public-enhancements/linkis-pes-client/src/main/scala/org/apache/linkis/datasource/client/request/GetDataSourceKeyDefineByTypeAction.scala b/linkis-public-enhancements/linkis-pes-client/src/main/scala/org/apache/linkis/datasource/client/request/GetDataSourceKeyDefineByTypeAction.scala new file mode 100644 index 0000000000..854351a5c1 --- /dev/null +++ b/linkis-public-enhancements/linkis-pes-client/src/main/scala/org/apache/linkis/datasource/client/request/GetDataSourceKeyDefineByTypeAction.scala @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.datasource.client.request + +import org.apache.linkis.datasource.client.config.DatasourceClientConfig.DATA_SOURCE_SERVICE_MODULE +import org.apache.linkis.datasource.client.errorcode.DatasourceClientErrorCodeSummary._ +import org.apache.linkis.datasource.client.exception.DataSourceClientBuilderException +import org.apache.linkis.httpclient.request.GetAction + +class GetDataSourceKeyDefineByTypeAction extends GetAction with DataSourceAction { + private var user: String = _ + private var typeName: String = _ + + override def setUser(user: String): Unit = this.user = user + + override def getUser: String = this.user + + override def suffixURLs: Array[String] = + Array(DATA_SOURCE_SERVICE_MODULE.getValue, "key-define", typeName) + +} + +object GetDataSourceKeyDefineByTypeAction { + def builder(): Builder = new Builder + + class Builder private[GetDataSourceKeyDefineByTypeAction] () { + private var user: String = _ + private var typeName: String = _ + + def setUser(user: String): Builder = { + this.user = user + this + } + + def setTypeName(typeName: String): Builder = { + this.typeName = typeName + this + } + + def build(): GetDataSourceKeyDefineByTypeAction = { + if (user == null) throw new DataSourceClientBuilderException(USER_NEEDED.getErrorDesc) + val action = new GetDataSourceKeyDefineByTypeAction + action.user = user + action.typeName = typeName + action + } + + } + +} diff --git a/linkis-public-enhancements/linkis-pes-client/src/main/scala/org/apache/linkis/datasource/client/response/GetDataSourceByTypeNameResult.scala b/linkis-public-enhancements/linkis-pes-client/src/main/scala/org/apache/linkis/datasource/client/response/GetDataSourceByTypeNameResult.scala new file mode 100644 index 0000000000..5165664301 --- /dev/null +++ b/linkis-public-enhancements/linkis-pes-client/src/main/scala/org/apache/linkis/datasource/client/response/GetDataSourceByTypeNameResult.scala @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.datasource.client.response + +import org.apache.linkis.datasourcemanager.common.domain.DataSource +import org.apache.linkis.httpclient.dws.DWSHttpClient +import org.apache.linkis.httpclient.dws.annotation.DWSHttpMessageResult +import org.apache.linkis.httpclient.dws.response.DWSResult + +import java.util + +import scala.beans.BeanProperty + +@DWSHttpMessageResult("/api/rest_j/v\\d+/data-source-manager/info-by-type") +class GetDataSourceByTypeNameResult extends DWSResult { + @BeanProperty var queryList: util.List[java.util.Map[String, Any]] = _ + @BeanProperty var totalPage: Int = _ + + def getAllDataSource: util.List[DataSource] = { + import scala.collection.JavaConverters._ + queryList.asScala + .map(x => { + val str = DWSHttpClient.jacksonJson.writeValueAsString(x) + DWSHttpClient.jacksonJson.readValue(str, classOf[DataSource]) + }) + .asJava + } + +} diff --git a/linkis-public-enhancements/linkis-pes-client/src/main/scala/org/apache/linkis/datasource/client/response/GetDataSourceKeyDefineByTypeResult.scala b/linkis-public-enhancements/linkis-pes-client/src/main/scala/org/apache/linkis/datasource/client/response/GetDataSourceKeyDefineByTypeResult.scala new file mode 100644 index 0000000000..16e3b2c063 --- /dev/null +++ b/linkis-public-enhancements/linkis-pes-client/src/main/scala/org/apache/linkis/datasource/client/response/GetDataSourceKeyDefineByTypeResult.scala @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.datasource.client.response + +import org.apache.linkis.datasourcemanager.common.domain.{ + DataSourceParamKeyDefinition, + DatasourceVersion +} +import org.apache.linkis.httpclient.dws.DWSHttpClient +import org.apache.linkis.httpclient.dws.annotation.DWSHttpMessageResult +import org.apache.linkis.httpclient.dws.response.DWSResult + +import java.util + +import scala.beans.BeanProperty + +@DWSHttpMessageResult("/api/rest_j/v\\d+/data-source-manager/key-define/(\\S+)") +class GetDataSourceKeyDefineByTypeResult extends DWSResult { + + @BeanProperty var keyDefine: util.List[java.util.Map[String, Any]] = _ + + def getDatasourceVersion: util.List[DataSourceParamKeyDefinition] = { + import scala.collection.JavaConverters._ + keyDefine.asScala + .map(x => { + val str = DWSHttpClient.jacksonJson.writeValueAsString(x) + DWSHttpClient.jacksonJson.readValue(str, classOf[DataSourceParamKeyDefinition]) + }) + .asJava + } + +}