Merge pull request #293 from WeDataSphere/dev-1.1.15-webank

update 1.1.15 to 1.1.16

linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/service/TaskExecutionServiceImpl.scala

@@ -429,14 +429,36 @@ class TaskExecutionServiceImpl
         while (null != taskFuture && !taskFuture.isDone) {
           if (!ExecutionNodeStatus.isCompleted(task.getStatus)) {
             Utils.tryAndWarn {
-              val progressResponse = Utils.tryAndWarn(taskProgress(task.getTaskId))
-              val resourceResponse = Utils.tryAndWarn(buildResourceMap(task))
-              val extraInfoMap = Utils.tryAndWarn(buildExtraInfoMap(task))
+              val progressResponse = Utils.tryCatch(taskProgress(task.getTaskId)) {
+                case e: Exception =>
+                  logger.info("Failed to get progress", e)
+                  null
+              }
+              val resourceResponse = Utils.tryCatch(buildResourceMap(task)) { case e: Exception =>
+                logger.info("Failed to get resource", e)
+                null
+              }
+              val extraInfoMap = Utils.tryCatch(buildExtraInfoMap(task)) { case e: Exception =>
+                logger.info("Failed to get extra info ", e)
+                null
+              }
               val resourceMap = if (null != resourceResponse) resourceResponse.resourceMap else null
+
+              /**
+               * It is guaranteed that there must be progress the progress must be greater than or
+               * equal to 0.1
+               */
+              val newProgressResponse = if (null == progressResponse) {
+                ResponseTaskProgress(task.getTaskId, 0.1f, null)
+              } else if (progressResponse.progress < 0.1f) {
+                ResponseTaskProgress(task.getTaskId, 0.1f, progressResponse.progressInfo)
+              } else {
+                progressResponse
+              }
               val respRunningInfo: ResponseTaskRunningInfo = ResponseTaskRunningInfo(
-                progressResponse.execId,
-                progressResponse.progress,
-                progressResponse.progressInfo,
+                newProgressResponse.execId,
+                newProgressResponse.progress,
+                newProgressResponse.progressInfo,
                 resourceMap,
                 extraInfoMap
               )
@@ -498,7 +520,7 @@ class TaskExecutionServiceImpl
   }
 
   override def taskProgress(taskID: String): ResponseTaskProgress = {
-    var response = ResponseTaskProgress(taskID, 0, null)
+    var response = ResponseTaskProgress(taskID, 0.01f, null)
     if (StringUtils.isBlank(taskID)) return response
     val executor = taskIdCache.getIfPresent(taskID)
     if (null != executor) {
@@ -513,11 +535,9 @@ class TaskExecutionServiceImpl
             ResponseTaskProgress(taskID, progress, executor.getProgressInfo(taskID))
           )
         }
-      } else {
-        response = ResponseTaskProgress(taskID, -1, null)
       }
     } else {
-      logger.error(s"Executor of taskId : $taskID is not cached.")
+      logger.info(s"Executor of taskId : $taskID is not cached.")
     }
     response
   }

linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/scheduler/EntranceFIFOUserConsumer.java

@@ -40,23 +40,45 @@ public EntranceFIFOUserConsumer(
   public boolean runScheduleIntercept() {
     Consumer[] consumers = getSchedulerContext().getOrCreateConsumerManager().listConsumers();
     int creatorRunningJobNum = 0;
-    String[] groupNames = getGroup().getGroupName().split("_");
-    if (groupNames.length < 3) {
+
+    // APP_TEST_hadoop_hive or IDE_hadoop_hive
+    String groupNameStr = getGroup().getGroupName();
+    String[] groupNames = groupNameStr.split("_");
+    int length = groupNames.length;
+    if (length < 3) {
       return true;
     }
-    String creatorName = groupNames[0];
-    String ecType = groupNames[2];
+
+    // APP_TEST
+    int lastIndex = groupNameStr.lastIndexOf("_");
+    int secondLastIndex = groupNameStr.lastIndexOf("_", lastIndex - 1);
+    String creatorName = groupNameStr.substring(0, secondLastIndex);
+
+    // hive
+    String ecType = groupNames[length - 1];
+
     for (Consumer consumer : consumers) {
       String groupName = consumer.getGroup().getGroupName();
       if (groupName.startsWith(creatorName) && groupName.endsWith(ecType)) {
         creatorRunningJobNum += consumer.getRunningEvents().length;
       }
     }
+
     int creatorECTypeMaxRunningJobs =
         CreatorECTypeDefaultConf.getCreatorECTypeMaxRunningJobs(creatorName, ecType);
+
+    if (logger.isDebugEnabled()) {
+      logger.debug(
+          "Creator: {} EC: {} there are currently:{} jobs running and maximum limit: {}",
+          creatorName,
+          ecType,
+          creatorRunningJobNum,
+          creatorECTypeMaxRunningJobs);
+    }
+
     if (creatorRunningJobNum > creatorECTypeMaxRunningJobs) {
       logger.error(
-          "Creator: {} EC: {} there are currently {} jobs running that exceed the maximum limit: {}",
+          "Creator: {} EC: {} there are currently:{} jobs running that exceed the maximum limit: {}",
           creatorName,
           ecType,
           creatorRunningJobNum,

linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/conf/EntranceConfiguration.scala

@@ -243,6 +243,6 @@ object EntranceConfiguration {
     CommonVars("wds.linkis.entrance.task.timeout.scan", new TimeType("12h"))
 
   val ENABLE_HDFS_JVM_USER =
-    CommonVars[Boolean]("linkis.entrance.enable.hdfs.jvm.user", true).getValue
+    CommonVars[Boolean]("linkis.entrance.enable.hdfs.jvm.user", false).getValue
 
 }

linkis-dist/package/db/linkis_dml.sql

@@ -90,7 +90,7 @@ INSERT INTO `linkis_ps_configuration_config_key` (`key`, `description`, `name`,
 INSERT INTO `linkis_ps_configuration_config_key` (`key`, `description`, `name`, `default_value`, `validate_type`, `validate_range`, `is_hidden`, `is_advanced`, `level`, `treeName`, `engine_conn_type`) VALUES ('spark.tispark.tidb.user', NULL, NULL, 'root', 'None', NULL, '0', '0', '1', 'tidb设置', 'spark');
 INSERT INTO `linkis_ps_configuration_config_key` (`key`, `description`, `name`, `default_value`, `validate_type`, `validate_range`, `is_hidden`, `is_advanced`, `level`, `treeName`, `engine_conn_type`) VALUES ('spark.python.version', '取值范围：python2,python3', 'python版本','python2', 'OFT', '[\"python3\",\"python2\"]', '0', '0', '1', 'spark引擎设置', 'spark');
 INSERT INTO `linkis_ps_configuration_config_key` (`key`, `description`, `name`, `default_value`, `validate_type`, `validate_range`, `engine_conn_type`, `is_hidden`, `is_advanced`, `level`, `treeName`, `boundary_type`, `en_treeName`, `en_description`, `en_name`) VALUES ('spark.conf', '多个参数使用分号[;]分隔 例如spark.shuffle.spill=true;', 'spark自定义配置参数',null, 'None', NULL, 'spark',0, 1, 1,'spark资源设置', 0, 'Spark Resource Settings','Multiple parameters are separated by semicolons [;] For example, spark.sql.shuffle.partitions=10;', 'Spark Custom Configuration Parameters');
-INSERT INTO `linkis_ps_configuration_config_key` (`key`, `description`, `name`, `default_value`, `validate_type`, `validate_range`, `engine_conn_type`, `is_hidden`, `is_advanced`, `level`, `treeName`, `boundary_type`, `en_treeName`, `en_description`, `en_name`) VALUES ('spark.locality.wait', '范围：0-3000，单位：毫秒', '任务调度本地等待时间', '3000', 'OFT', '[\"0\",\"1000\",\"2000\",\"3000\"]', 'spark', 0, 1, 1, 'spark资源设置', 0, 'Spark Resource Settings', 'Range: 0-3000, Unit: millisecond', 'Task Scheduling Local Waiting Time');
+INSERT INTO `linkis_ps_configuration_config_key` (`key`, `description`, `name`, `default_value`, `validate_type`, `validate_range`, `engine_conn_type`, `is_hidden`, `is_advanced`, `level`, `treeName`, `boundary_type`, `en_treeName`, `en_description`, `en_name`) VALUES ('spark.locality.wait', '范围：0-3，单位：秒', '任务调度本地等待时间', '3s', 'OFT', '[\"0s\",\"1s\",\"2s\",\"3s\"]', 'spark', 0, 1, 1, 'spark资源设置', 0, 'Spark Resource Settings', 'Range: 0-3, Unit: second', 'Task Scheduling Local Waiting Time');
 INSERT INTO `linkis_ps_configuration_config_key` (`key`, `description`, `name`, `default_value`, `validate_type`, `validate_range`, `engine_conn_type`, `is_hidden`, `is_advanced`, `level`, `treeName`, `boundary_type`, `en_treeName`, `en_description`, `en_name`) VALUES ('spark.memory.fraction', '范围：0.4,0.5,0.6，单位：百分比', '执行内存和存储内存的百分比', '0.6', 'OFT', '[\"0.4\",\"0.5\",\"0.6\"]', 'spark', 0, 1, 1, 'spark资源设置', 0, 'Spark Resource Settings', 'Range: 0.4, 0.5, 0.6, in percentage', 'Percentage Of Execution Memory And Storage Memory');
 
 -- hive

linkis-dist/package/db/upgrade/1.4.1_schema/mysql/linkis_dml.sql

@@ -69,7 +69,7 @@ INSERT INTO `linkis_ps_configuration_config_value` (`config_key_id`, `config_val
 INSERT INTO `linkis_ps_configuration_config_key`
 (`key`, `description`, `name`, `default_value`, `validate_type`, `validate_range`, `engine_conn_type`, `is_hidden`, `is_advanced`, `level`, `treeName`, `boundary_type`, `en_treeName`, `en_description`, `en_name`)
 VALUES
-('spark.locality.wait', '范围：0-3000，单位：毫秒', '任务调度本地等待时间', '3000', 'OFT', '[\"0\",\"1000\",\"2000\",\"3000\"]', 'spark', 0, 1, 1, 'spark资源设置', 0, 'Spark Resource Settings', 'Range: 0-3000, Unit: millisecond', 'Task Scheduling Local Waiting Time');
+('spark.locality.wait', '范围：0-3，单位：秒', '任务调度本地等待时间', '3s', 'OFT', '[\"0s\",\"1s\",\"2s\",\"3s\"]', 'spark', 0, 1, 1, 'spark资源设置', 0, 'Spark Resource Settings', 'Range: 0-3, Unit: second', 'Task Scheduling Local Waiting Time');
 
 
 -- all 默认

linkis-engineconn-plugins/hive/src/main/scala/org/apache/linkis/engineplugin/hive/executor/HiveEngineConnExecutor.scala

@@ -470,7 +470,7 @@ class HiveEngineConnExecutor(
       val currentBegin = (currentSQL - 1) / totalSQLs.asInstanceOf[Float]
       val finishedStage =
         if (null != driver && null != driver.getPlan() && !driver.getPlan().getRootTasks.isEmpty) {
-          Utils.tryAndWarn(
+          Utils.tryQuietly(
             Utilities
               .getMRTasks(driver.getPlan().getRootTasks)
               .asScala

linkis-web/src/apps/linkis/module/acrossClusterRule/index.vue

@@ -22,34 +22,37 @@
       size="large"
       fix/>
     <Row class="search-bar">
-      <Col span="4" class="search-item">
-        <span class="search-label">{{$t('message.linkis.ipListManagement.userName')}}</span>
+      <Col span="6" class="search-item">
+        <span :title="$t('message.linkis.ipListManagement.userName')" class="search-label">{{$t('message.linkis.ipListManagement.userName')}}</span>
         <Input
           v-model="queryData.username"
           class="input"
           :placeholder="$t('message.linkis.ipListManagement.inputUser')"
           @on-enter="search"
+          style="width: 290px"
         ></Input>
       </Col>
-      <Col span="4" class="search-item">
+      <Col span="6" class="search-item">
         <span :title="$t('message.linkis.ipListManagement.appName')" class="search-label">{{$t('message.linkis.ipListManagement.appName')}}</span>
         <Input
           v-model="queryData.creator"
           class="input"
           :placeholder="$t('message.linkis.ipListManagement.inputApp')"
           @on-enter="search"
+          style="width: 290px"
         ></Input>
       </Col>
-      <Col span="4" class="search-item">
-        <span class="search-label">{{$t('message.linkis.ipListManagement.cluster')}}</span>
+      <Col span="6" class="search-item">
+        <span :title="$t('message.linkis.ipListManagement.cluster')" class="search-label">{{$t('message.linkis.ipListManagement.cluster')}}</span>
         <Input
           v-model="queryData.clusterName"
           class="input"
           :placeholder="$t('message.linkis.ipListManagement.inputCluster')"
           @on-enter="search"
+          style="width: 290px"
         ></Input>
       </Col>
-      <Col span="9">
+      <Col span="6">
         <Button type="primary" class="button" @click="search">{{
           $t('message.linkis.ipListManagement.search')
         }}</Button>

linkis-web/src/apps/linkis/module/configManagement/index.vue

@@ -23,13 +23,13 @@
       fix/>
     <Row class="search-bar">
       <Col span="6" class="search-item">
-        <span class="search-label">{{$t('message.linkis.ipListManagement.engineType')}}</span>
+        <span :title="$t('message.linkis.ipListManagement.engineType')" class="search-label">{{$t('message.linkis.ipListManagement.engineType')}}</span>
         <Select v-model="queryData.engineType" style="width: 290px">
           <Option v-for="(item) in getEngineTypes" :label="item === 'all' ? $t('message.linkis.engineTypes.all'): item" :value="item" :key="item" />
         </Select>
       </Col>
       <Col span="6" class="search-item">
-        <span class="search-label">{{$t('message.linkis.ipListManagement.key')}}</span>
+        <span :title="$t('message.linkis.ipListManagement.key')" class="search-label">{{$t('message.linkis.ipListManagement.key')}}</span>
         <Input
           v-model="queryData.key"
           style="width: 290px"
@@ -370,7 +370,8 @@ export default {
 
         this.datalist = res.configKeyList.map((item) => {
           item.formatBoundaryType = this.boundaryTypeMap[item.boundaryType]
-          item.formatValidateType = this.validateTypeList.find(p => p.value === item.validateType).label
+          item.formatValidateType = this.validateTypeList.find(p => p.value === item.validateType)?.label || ''
+          item.templateRequired = item.templateRequired ? '1' : '0'
           return item
         });
         this.page.totalPage = res.totalPage;
@@ -425,16 +426,17 @@ export default {
       const target = '/configuration/baseKeyValue'
       this.$refs.createTenantForm.validate(async (valid) => {
         if(valid) {
-          this.clearSearch();
+
           try {
             if(this.mode !== 'edit') {
               this.page.pageNow = 1;
             }
             this.isRequesting = true
             const body = cloneDeep(this.modalData);
             if(!body.engineType || body.engineType === 'all') delete body.engineType
+            body.templateRequired = body.templateRequired === '1'
             await api.fetch(target, body, "post")
-            await this.getTableData();
+            await this.clearSearch();
             this.cancel();
             this.$Message.success(this.$t('message.linkis.udf.success'));
 

linkis-web/src/apps/linkis/module/userConfig/index.vue

@@ -113,11 +113,11 @@
             </Select>
           </FormItem>
           <FormItem :label="$t('message.linkis.userConfig.configVKey')" prop="configKey">
-            <Select v-model="modalData.configKey" :disabled="noSelectConfig || isEdit" filterable clearable @on-open-change="fetchConfigOption" style="width: 319px;">
-              <Option v-for="item in configOption" :value="item" :key="item" >{{item}}</Option>
+            <Select v-model="modalData.configKey" :disabled="noSelectConfig || isEdit" filterable clearable @on-open-change="fetchConfigOption" @on-select="onSelect" style="width: 319px;">
+              <Option v-for="item in configOption" :key="item.value" :value="item.value" >{{item.label}}</Option>
             </Select>
           </FormItem>
-          <FormItem :label="$t('message.linkis.userConfig.configValue')" prop="configValue">
+          <FormItem :label="$t('message.linkis.userConfig.configValue')" :rules="{required: !!defaultValue , message: $t('message.linkis.ipListManagement.notEmpty'), trigger: 'blur'}" prop="configValue">
             <Input class="input" v-model="modalData.configValue" clearable  style="width: 319px" :placeholder="$t('message.linkis.userConfig.configValuePro')"></Input>
           </FormItem>
         </Form>
@@ -259,9 +259,6 @@ export default {
         ],
         configKey: [
           {required: true, message: this.$t('message.linkis.tenantTagManagement.notEmpty'), trigger: 'blur'}
-        ],
-        configValue: [
-          {required: true, message: this.$t('message.linkis.tenantTagManagement.notEmpty'), trigger: 'blur'}
         ]
       },
       btnDisabled: false,
@@ -276,7 +273,8 @@ export default {
       engineTypeOptions: [],
       allEngineTypeOptions: [],
       configOption: [],
-      versionOption: []
+      versionOption: [],
+      defaultValue: '',
     }
   },
   computed: {
@@ -329,7 +327,6 @@ export default {
       try {
         api.fetch("/configuration/engineType", {}, "get")
           .then((res) => {
-            window.console.log(res);
             res.engineType = res.engineType.map((item) => {
               return {label: item, value: item};
             });
@@ -360,7 +357,7 @@ export default {
     },
     async createTenant () {
       this.showCreateModal = true;
-      this.mode = 'create'
+      this.mode = 'create';
     },
     cancel() {
       this.showCreateModal = false;
@@ -402,11 +399,12 @@ export default {
     },
     edit(data) {
       const {
-        user, creator, engineType, version, key, configValue
+        user, creator, engineType, version, key, configValue, defaultValue
       } = data;
-      this.configOption.push(key);
+      this.configOption.push({value: key, label: key});
+      this.defaultValue = defaultValue
       this.modalData = {
-        user, creator, engineType, version, configKey: key, configValue
+        user, creator, engineType, version, configKey: key, configValue,
       };
       this.versionOption = [{label: version, value: version}];
       this.showCreateModal = true;
@@ -473,11 +471,16 @@ export default {
         let keyArr = [];
         res.itemList.forEach((item) => {
           if (!keyArr.includes(item.key)) {
-            keyArr.push(item.key);
+            keyArr.push({label: item.key, value: item.key, defaultValue: item.defaultValue});
           }
         });
         this.configOption = keyArr;
       })
+    },
+    onSelect(val) {
+      this.defaultValue = this.configOption.find((item) => {
+        return item.value === val
+      })?.defaultValue
     }
   },
   created() {

linkis-web/src/components/virtualTable/historyTable/historyTable.vue

@@ -51,23 +51,23 @@
             v-for="(th, index) in columns"
             :key="th.key"
             :style="{'text-align': th.align}"
-            class="we-table-row-cell">
-            <Tooltip :content="td[th.key]" :disabled="!th.ellipsis" placement="bottom-end">
-              <div
-                class="we-table-row-label"
-                :style="{'width': th.width ? th.width + 'px' : getComputedWidth(th)}"
-                :class="{'ellipsis': th.ellipsis, [th.className]: true}">
-                <table-expand
-                  v-if="th.renderType"
-                  :row="td"
-                  :column="th"
-                  :index="index"
-                  :render="renderComponent({type: th.renderType, cell: td, key: th.key, params: th.renderParams, trIndex})"></table-expand>
-                <span
-                  v-else
-                  :class="th.className">{{ td[th.key] }}</span>
-              </div>
-            </Tooltip>
+            class="we-table-row-cell"
+            :aria-label="td[th.key]"
+            :class="[!th.ellipsis ? '' : 'hint--bottom hint--rounded']">
+            <div
+              class="we-table-row-label"
+              :style="{'width': th.width ? th.width + 'px' : getComputedWidth(th)}"
+              :class="{'ellipsis': th.ellipsis, [th.className]: true}">
+              <table-expand
+                v-if="th.renderType"
+                :row="td"
+                :column="th"
+                :index="index"
+                :render="renderComponent({type: th.renderType, cell: td, key: th.key, params: th.renderParams, trIndex})"></table-expand>
+              <span
+                v-else
+                :class="th.className">{{ td[th.key] }}</span>
+            </div>
           </td>
         </tr>
       </tbody>

linkis-web/src/dss/module/resourceSimple/engine.vue

@@ -248,6 +248,8 @@ export default {
           return 'Openlookeng';
         case 'elasticsearch':
           return 'Elasticsearch';
+        case 'trino':
+          return 'Trino';
         case 'Unlock':
           return this.$t('message.common.resourceSimple.KX')
         case 'Idle':