Skip to content

Commit

Permalink
code optimize
Browse files Browse the repository at this point in the history
  • Loading branch information
peacewong committed Sep 11, 2023
1 parent 5e52f90 commit 703eb50
Show file tree
Hide file tree
Showing 104 changed files with 620 additions and 574 deletions.
143 changes: 143 additions & 0 deletions linkis-dist/package/conf/linkis-et-monitor-ims.properties
Original file line number Diff line number Diff line change
@@ -0,0 +1,143 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

jobhistory.errorcode.01002={"alert_title":"Linkis Service load is too high, please contact Linkis owner","alert_info":"Linkis Linkis Service load is too high, please contact Linkis owner","alert_way":"1,2,3","alert_reciver":"hadoop","alert_level":"2","alert_obj":"linkis-alert-obj","can_recover":"0"}
jobhistory.errorcode.01003={"alert_title":"Linkis Service load is too high, please contact Linkis owner","alert_info":"Linkis Service load is too high, please contact Linkis owner","alert_way":"1,2,3","alert_reciver":"hadoop","alert_level":"2","alert_obj":"linkis-alert-obj","can_recover":"0"}
jobhistory.errorcode.01101={"alert_title":"Linkis Service load is too high, please contact Linkis owner","alert_info":" Service load is too high, please contact Linkis owner","alert_way":"1,2,3","alert_reciver":"hadoop","alert_level":"2","alert_obj":"linkis-alert-obj","can_recover":"0"}
jobhistory.errorcode.01102={"alert_title":"Linkis ECM memory Service load is too high, please contact Linkis owner","alert_info":" Service load is too high, please contact Linkis owner","alert_way":"1,2,3","alert_reciver":"hadoop","alert_level":"2","alert_obj":"linkis-alert-obj","can_recover":"0"}
jobhistory.errorcode.01103={"alert_title":"Linkis ECM CPU Service load is too high, please contact Linkis owner","alert_info":" Service load is too high, please contact Linkis owner","alert_way":"1,2,3","alert_reciver":"hadoop","alert_level":"2","alert_obj":"linkis-alert-obj","can_recover":"0"}
jobhistory.errorcode.01104={"alert_title":"Linkis ECM instances Service load is too high, please contact Linkis owner","alert_info":" Service load is too high, please contact Linkis owner","alert_way":"1,2,3","alert_reciver":"hadoop","alert_level":"2","alert_obj":"linkis-alert-obj","can_recover":"0"}
jobhistory.errorcode.01105={"alert_title":"Linkis Memory Service load is too high, please contact Linkis owner","alert_info":"The machine has insufficient memory. Please contact the administrator to expand the memory.","alert_way":"1,2,3","alert_reciver":"hadoop","alert_level":"2","alert_obj":"linkis-alert-obj","can_recover":"0"}
jobhistory.unfinished.time.exceed.sec.43200={"alert_title":"Linkis task execute timeout","alert_info":"Linkis task execute timeout 12h","alert_way":"1,2,3","alert_reciver":"","alert_level":"4","alert_obj":"Linkis","can_recover":"0"}

ecm.resource.monitor.im.12003=\
{"alert_title":"Linkis Monitor Resource Alert",\
"alert_info":" $instance ECM Memory/cpu insufficient resource < 0.2 please contact Linkis owner: $name",\
"alert_way":"1,2,3",\
"alert_reciver":"hadoop",\
"alert_level":"3",\
"alert_obj":"linkis_alert",\
"can_recover":"0"}
ecm.resource.monitor.im.12004=\
{"alert_title":"Linkis Monitor Resource Alert",\
"alert_info":" $instance ECM Memory/cpu insufficient resource < 0.2 please contact Linkis owner: $name",\
"alert_way":"1,2,3",\
"alert_reciver":"hadoop",\
"alert_level":"2",\
"alert_obj":"linkis_alert",\
"can_recover":"0"}
ecm.resource.monitor.im.12005=\
{"alert_title":"Linkis entrance user running task monitor",\
"alert_info":"User $username runninng task at linkis ( $url ) > $runningtask ,please attention",\
"alert_way":"1,2,3",\
"alert_reciver":"hadoop",\
"alert_level":"3",\
"alert_obj":"linkis_alert",\
"can_recover":"0"}
ecm.resource.monitor.im.12006=\
{"alert_title":"Linkis entrance user queued task monitor",\
"alert_info":"User $username queued task at linkis ( $url ) > $queuedtask ,please attention",\
"alert_way":"1,2,3",\
"alert_reciver":"hadoop",\
"alert_level":"3",\
"alert_obj":"linkis_alert",\
"can_recover":"0"}
ecm.resource.monitor.im.12007=\
{"alert_title":"Linkis entrance user total task monitor",\
"alert_info":"User $username queued task at linkis ( $url ) > $tasktotal ,please attention",\
"alert_way":"1,2,3",\
"alert_reciver":"hadoop",\
"alert_level":"3",\
"alert_obj":"linkis_alert",\
"can_recover":"0"}
ecm.resource.monitor.im.12008=\
{"alert_title":"Linkis entrance all running task monitor",\
"alert_info":"linkis ( $url ) running task > $taskminor,please attention",\
"alert_way":"1,2,3",\
"alert_reciver":"hadoop",\
"alert_level":"3",\
"alert_obj":"linkis_alert",\
"can_recover":"0"}
ecm.resource.monitor.im.12009=\
{"alert_title":"Linkis entrance all running task monitor",\
"alert_info":"linkis ( $url ) all task > $taskmajor,please attention",\
"alert_way":"1,2,3",\
"alert_reciver":"hadoop",\
"alert_level":"2",\
"alert_obj":"linkis_alert",\
"can_recover":"0"}



user.mode.monitor.im.12011=\
{"alert_title":"User mode execution timeout alarm",\
"alert_info":"User mode execution timeout alarm Linkis url: $url engineType:$engineType Task ID: $jobId please attention $name",\
"alert_way":"1,2,3",\
"alert_reciver":"alexyang,hadoop",\
"alert_level":"2",\
"alert_obj":"linkis_alert",\
"can_recover":"0"}
user.mode.monitor.im.12012=\
{"alert_title":"User mode execution failure alarm",\
"alert_info":"User mode execution failure alarm Linkis url: $url Engine: $engineType TaskID: $jobId ER=rrorCode?$errorCode errorMsg: $errorMsg please attention $name",\
"alert_way":"1,2,3",\
"alert_reciver":"alexyang,hadoop",\
"alert_level":"2",\
"alert_obj":"linkis_alert",\
"can_recover":"0"}

user.mode.monitor.im.12017=\
{"alert_title":"@alert_title",\
"alert_info":"task execute failed, reason $msg",\
"alert_way":"1,2,3",\
"alert_reciver":"@alert_reciver",\
"alert_level":"@alert_level",\
"alert_obj":"linkis_alert",\
"can_recover":"0"}
user.mode.monitor.im.12018=\
{"alert_title":"@alert_title",\
"alert_info":"task execute time out $timeout",\
"alert_way":"1,2,3",\
"alert_reciver":"@alert_reciver",\
"alert_level":"@alert_level",\
"alert_obj":"linkis_alert",\
"can_recover":"0"}

thread.monitor.timeout.im.12014=\
{"alert_title":" Linkis Shell Timeout Alert ",\
"alert_info":"Monitor Shell execute time out $shellName",\
"alert_way":"1,2,3",\
"alert_reciver":"hadoop",\
"alert_level":"3",\
"alert_obj":"linkis_alert",\
"can_recover":"0"}
jobhistory.result.monitor.im.12015=\
{"alert_title":"Linkis \u4EFB\u52A1\u72B6\u6001\u901A\u77E5",\
"alert_info":"Task ID:$id final status: $status",\
"alert_way":"1,2,3",\
"alert_reciver":"",\
"alert_level":"4",\
"alert_obj":"linkis_alert",\
"can_recover":"0"}
jobhistory.result.monitor.im.12016=\
{"alert_title":"@alert_title",\
"alert_info":"[Alarm time]$date\n[Subsystem]$sysid\n[Alarm IP]$ip\n[Alarm object]$object\n[Alarm information]$detail",\
"alert_way":"1,2,3",\
"alert_reciver":"@alert_reciver",\
"alert_level":"@alert_level",\
"alert_obj":"linkis_alert",\
"can_recover":"0"}


86 changes: 86 additions & 0 deletions linkis-dist/package/conf/linkis-et-monitor.properties
Original file line number Diff line number Diff line change
@@ -0,0 +1,86 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

wds.linkis.server.mybatis.typeAliasesPackage=org.apache.linkis.monitor.scan.app.instance.entity,org.apache.linkis.monitor.scan.app.jobhistory.entity,org.apache.linkis.bml.cleaner.dao
wds.linkis.server.mybatis.BasePackage=org.apache.linkis.monitor.scan.app.instance.dao,org.apache.linkis.monitor.scan.app.jobhistory.dao,org.apache.linkis.bml.cleaner.dao


# alert server url
linkis.alert.url=http://127.0.0.1/

# alert receiver
linkis.alert.receiver.default=hadoop

# monitor ecm resource
linkis.monitor.ecm.resource.cron=0 0/10 * * * ?

# Resource remaining ratio
linkis.monitor.ecmResourceTask.major=0.005
linkis.monitor.ecmResourceTask.minor=0.01

# entrance task metrics cron
linkis.monitor.entrance.task.cron=0 0/10 * * * ?

# Timeout task cron
linkis.monitor.jobHistory.timeout.cron=0 0/20 * * * ?

# time out interval 24h
linkis.monitor.scanner.timeout.interval.seconds=86400


# Finished task cron
linkis.monitor.jobHistory.finished.cron=0 0/20 * * * ?

# linkis user mode cron
linkis.monitor.user.mode.cron=0 0/20 * * * ?

# user mode for engine
linkis.monitor.user.enginelist=[{"engineType":"hive-2.3.3","code":"show tables","runType":"hql","executeUser":"hadoop"},\
{"engineType":"spark-2.4.3","code":"show tables","runType":"sql","executeUser":"hadoop"},\
{"engineType":"shell-1","code":"pwd","runType":"sh","executeUser":"hadoop"}]

linkis.monitor.user.authToken=BML-AUTH

linkis.monitor.user.mode.timeOut=300


# bml clear cron
linkis.monitor.bml.clear.history.version.cron=0 0 12 * * ?
# bml max version
linkis.bml.cleaner.version.max.num=50
# keep verssion
linkis.bml.cleaner.version.keep.num=20
# clear max interval
linkis.bml.cleaner.previous.interval.days=30
# once scan limit
linkis.bml.cleaner.once.limit.num=100

# clear db ec record cron
linkis.monitor.clear.ecRecord.cron=0 10 12 * * ?

# clear task log cron
linkis.monitor.clear.taskLog.cron=0 10 12 * * ?

# clear db task history cron
linkis.monitor.clear.history.task.cron=0 0 13 * * ?

# poll size
linkis.monitor.scheduled.pool.cores.num=20

# shell time out conf minute
linkis.monitor.shell.time.out.minute=60

##Spring
spring.server.port=9119
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
* limitations under the License.
*/

package org.apache.linkis.monitor.scan.app;
package org.apache.linkis.monitor;

import org.apache.linkis.LinkisBaseServerApp;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,11 +15,12 @@
* limitations under the License.
*/

package org.apache.linkis.monitor.scan.app.bml.cleaner.dao;
package org.apache.linkis.monitor.bml.cleaner.dao;

import org.apache.linkis.monitor.scan.app.bml.cleaner.entity.CleanedResourceVersion;
import org.apache.linkis.monitor.scan.app.bml.cleaner.entity.ResourceVersion;
import org.apache.linkis.monitor.scan.app.bml.cleaner.vo.CleanResourceVo;
import org.apache.linkis.monitor.bml.cleaner.vo.CleanResourceVo;
import org.apache.linkis.monitor.bml.cleaner.entity.CleanedResourceVersion;
import org.apache.linkis.monitor.bml.cleaner.entity.ResourceVersion;
import org.apache.linkis.monitor.bml.cleaner.vo.CleanResourceVo;

import org.apache.ibatis.annotations.*;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
* limitations under the License.
*/

package org.apache.linkis.monitor.scan.app.bml.cleaner.entity;
package org.apache.linkis.monitor.bml.cleaner.entity;

import java.util.Date;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
* limitations under the License.
*/

package org.apache.linkis.monitor.scan.app.bml.cleaner.entity;
package org.apache.linkis.monitor.bml.cleaner.entity;

import java.util.Date;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
* limitations under the License.
*/

package org.apache.linkis.monitor.scan.app.bml.cleaner.service;
package org.apache.linkis.monitor.bml.cleaner.service;

public interface CleanerService {

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,10 @@
* limitations under the License.
*/

package org.apache.linkis.monitor.scan.app.bml.cleaner.service;
package org.apache.linkis.monitor.bml.cleaner.service;

import org.apache.linkis.monitor.scan.app.bml.cleaner.entity.CleanedResourceVersion;
import org.apache.linkis.monitor.bml.cleaner.entity.CleanedResourceVersion;
import org.apache.linkis.monitor.bml.cleaner.entity.CleanedResourceVersion;
import org.apache.linkis.common.io.FsPath;
import org.apache.linkis.storage.fs.FileSystem;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,16 +15,16 @@
* limitations under the License.
*/

package org.apache.linkis.monitor.scan.app.bml.cleaner.service.impl;

import org.apache.linkis.monitor.scan.app.bml.cleaner.dao.VersionDao;
import org.apache.linkis.monitor.scan.app.bml.cleaner.entity.CleanedResourceVersion;
import org.apache.linkis.monitor.scan.app.bml.cleaner.entity.ResourceVersion;
import org.apache.linkis.monitor.scan.app.bml.cleaner.service.CleanerService;
import org.apache.linkis.monitor.scan.app.bml.cleaner.service.VersionService;
import org.apache.linkis.monitor.scan.app.bml.cleaner.vo.CleanResourceVo;
package org.apache.linkis.monitor.bml.cleaner.service.impl;

import org.apache.linkis.monitor.bml.cleaner.dao.VersionDao;
import org.apache.linkis.monitor.bml.cleaner.entity.CleanedResourceVersion;
import org.apache.linkis.monitor.bml.cleaner.entity.ResourceVersion;
import org.apache.linkis.monitor.bml.cleaner.service.CleanerService;
import org.apache.linkis.monitor.bml.cleaner.vo.CleanResourceVo;
import org.apache.linkis.monitor.bml.cleaner.service.VersionService;
import org.apache.linkis.common.io.FsPath;
import org.apache.linkis.monitor.scan.constants.Constants;
import org.apache.linkis.monitor.constants.Constants;
import org.apache.linkis.storage.FSFactory;
import org.apache.linkis.storage.fs.FileSystem;
import org.apache.linkis.storage.utils.StorageConfiguration;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,11 +15,13 @@
* limitations under the License.
*/

package org.apache.linkis.monitor.scan.app.bml.cleaner.service.impl;
package org.apache.linkis.monitor.bml.cleaner.service.impl;

import org.apache.linkis.monitor.scan.app.bml.cleaner.dao.VersionDao;
import org.apache.linkis.monitor.scan.app.bml.cleaner.entity.CleanedResourceVersion;
import org.apache.linkis.monitor.scan.app.bml.cleaner.service.VersionService;
import org.apache.linkis.monitor.bml.cleaner.dao.VersionDao;
import org.apache.linkis.monitor.bml.cleaner.entity.CleanedResourceVersion;
import org.apache.linkis.monitor.bml.cleaner.dao.VersionDao;
import org.apache.linkis.monitor.bml.cleaner.entity.CleanedResourceVersion;
import org.apache.linkis.monitor.bml.cleaner.service.VersionService;
import org.apache.linkis.common.io.FsPath;
import org.apache.linkis.storage.fs.FileSystem;

Expand All @@ -32,7 +34,8 @@
@Service
public class VersionServiceImpl implements VersionService {

@Autowired VersionDao versionDao;
@Autowired
VersionDao versionDao;

public void setVersionDao(VersionDao versionDao) {
this.versionDao = versionDao;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
* limitations under the License.
*/

package org.apache.linkis.monitor.scan.app.bml.cleaner.vo;
package org.apache.linkis.monitor.bml.cleaner.vo;

public class CleanResourceVo {
private String resourceId;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,11 +15,11 @@
* limitations under the License.
*/

package org.apache.linkis.monitor.scan.app.monitor.config;
package org.apache.linkis.monitor.config;

import org.apache.linkis.monitor.scan.app.monitor.until.HttpsUntils;
import org.apache.linkis.monitor.scan.app.monitor.until.ThreadUtils;
import org.apache.linkis.monitor.scan.utils.log.LogUtils;
import org.apache.linkis.monitor.until.HttpsUntils;
import org.apache.linkis.monitor.until.ThreadUtils;
import org.apache.linkis.monitor.utils.log.LogUtils;

import org.springframework.context.annotation.Configuration;
import org.springframework.context.event.ContextClosedEvent;
Expand Down
Loading

0 comments on commit 703eb50

Please sign in to comment.